Merge pull request #247 from kobotoolbox/243-support-new-pyxform-feat…

…ures

Add support for pyxform v1.5.0

src/formpack/schema/fields.py

@@ -160,23 +160,47 @@ def from_json_definition(cls, definition, hierarchy=None,
             choice = field_choices[choice_id]
 
         data_type_classes = {
-            "select_one": FormChoiceField,
-            "select_multiple": FormChoiceFieldWithMultipleSelect,
-            "geopoint": FormGPSField,
-            "date": DateField,
-            "text": TextField,
-            "barcode": TextField,
-
-            # calculate is usually not text but for our purpose it's good
-            # enough
-            "calculate": TextField,
-            "acknowledge": TextField,
-            "integer": NumField,
+            # selects
+            'select_one': FormChoiceField,
+            'select_one_from_file': FormChoiceField,
+            'select_multiple': FormChoiceFieldWithMultipleSelect,
+            # TODO: Get this to work with FormChoiceFieldWithMultipleSelect
+            'select_multiple_from_file': TextField,
+            'rank': TextField,
+
+            # date and time
+            'date': DateField,
+            'today': DateField,
+            'datetime': TextField,
+            'time': TextField,
+            'start': TextField,
+            'end': TextField,
+
+            # general
+            'text': TextField,
+            'barcode': TextField,
+            'acknowledge': TextField,
+
+            # geo
+            'geopoint': FormGPSField,
+            'start-geopoint': FormGPSField,
+
+            # media
+            'video': TextField,
+            'image': TextField,
+            'audio': TextField,
+            'file': TextField,
+            'background-audio': TextField,
+
+            # numeric
+            'calculate': TextField,
+            'integer': NumField,
             'decimal': NumField,
+            'range': NumField,
 
             # legacy type, treat them as text
-            "select_one_external": partial(TextField, data_type=data_type),
-            "cascading_select": partial(TextField, data_type=data_type),
+            'select_one_external': partial(TextField, data_type=data_type),
+            'cascading_select': partial(TextField, data_type=data_type),
         }
 
         args = {

src/formpack/utils/expand_content.py

@@ -11,7 +11,7 @@
 from .array_to_xpath import EXPANDABLE_FIELD_TYPES
 from .future import iteritems, OrderedDict
 from .iterator import get_first_occurrence
-from .replace_aliases import META_TYPES
+from .replace_aliases import META_TYPES, SELECT_TYPES
 from .string import str_types
 from ..constants import (UNTRANSLATED, OR_OTHER_COLUMN,
                          TAG_COLUMNS_AND_SEPARATORS)
@@ -231,6 +231,7 @@ def _mark_special(**kwargs):
 
 
 def _expand_type_to_dict(type_str):
+    SELECT_PATTERN = r'^({select_type})\s+(\S+)$'
     out = {}
     match = re.search('( or.other)$', type_str)
     if match:
@@ -243,12 +244,10 @@ def _expand_type_to_dict(type_str):
     if type_str in ['select_one', 'select_multiple']:
         out['type'] = type_str
         return out
-    for _re in [
-        r'^(select_one)\s+(\S+)$',
-        r'^(select_multiple)\s+(\S+)$',
-        r'^(select_one_external)\s+(\S+)$',
-    ]:
-        match = re.match(_re, type_str)
+    for select_type in SELECT_TYPES:
+        match = re.match(
+            SELECT_PATTERN.format(select_type=select_type), type_str
+        )
         if match:
             (type_, list_name) = match.groups()
             out['type'] = type_

src/formpack/utils/flatten_content.py

@@ -10,6 +10,7 @@
 from .array_to_xpath import array_to_xpath
 from .future import range
 from .string import str_types
+from .replace_aliases import SELECT_TYPES
 from ..constants import (UNTRANSLATED, OR_OTHER_COLUMN,
                          TAG_COLUMNS_AND_SEPARATORS)
 
@@ -72,11 +73,10 @@ def _stringify_type__depr(json_qtype):
     {'select_one': 'xyz'} -> 'select_one xyz'
     {'select_multiple': 'xyz'} -> 'select_mutliple xyz'
     """
-    _type_keys = ['select_one', 'select_multiple']
     if len(json_qtype.keys()) != 1:
         raise ValueError('Type object must have exactly one key: %s' %
-                         ', '.join(_type_keys))
-    for try_key in _type_keys:
+                         ', '.join(SELECT_TYPES))
+    for try_key in SELECT_TYPES:
         if try_key in json_qtype:
             return '{} {}'.format(try_key, json_qtype[try_key])
     if 'select_one_or_other' in json_qtype:

src/formpack/utils/replace_aliases.py

@@ -66,25 +66,35 @@ def aliases_to_ordered_dict(_d):
     'geopoint': ['gps'],
 })
 
-selects = aliases_to_ordered_dict({
-    'select_multiple': [
-        'select all that apply',
-        'select multiple',
-        'select many',
-        'select_many',
-        'select all that apply from',
-        'add select multiple prompt using',
-    ],
-    'select_one_external': [
-        'select one external',
-    ],
-    'select_one': [
-        'select one',
-        'select one from',
-        'add select one prompt using',
-        'select1',
-    ],
-})
+# keys used in `_expand_type_to_dict()` to handle choices argument
+selects = aliases_to_ordered_dict(
+    {
+        'select_multiple': [
+            'select all that apply',
+            'select multiple',
+            'select many',
+            'select_many',
+            'select all that apply from',
+            'add select multiple prompt using',
+        ],
+        'select_multiple_from_file': [
+            'select multiple from file',
+        ],
+        'select_one_external': [
+            'select one external',
+        ],
+        'select_one': [
+            'select one',
+            'select one from',
+            'add select one prompt using',
+            'select1',
+        ],
+        'select_one_from_file': [
+            'select one from file',
+        ],
+        'rank': [],
+    }
+)
 # Python3: Cast to a list because it's merged into other dicts
 # (i.e `SELECT_SCHEMA` in validators.py)
 SELECT_TYPES = list(selects.keys())
@@ -96,12 +106,15 @@ def aliases_to_ordered_dict(_d):
     'deviceid',
     'phone_number',
     'simserial',
+    'audit',
     # meta values
     'username',
     # reconsider:
     'phonenumber',
     'imei',
     'subscriberid',
+    # geo
+    'start-geopoint',
 ]
 
 LABEL_OPTIONAL_TYPES = [
@@ -128,17 +141,23 @@ def aliases_to_ordered_dict(_d):
     'video',
     'image',
     'audio',
+    'file',
+    'background-audio',
     # enter time values
     'date',
     'datetime',
     'time',
-
     # prompt to collect geo data
     'location',
-
     # no response
     'acknowledge',
     'note',
+    # external data source
+    'xml-external',
+    'csv-external',
+    # other
+    'range',
+    'hidden',
 ] + GEO_TYPES
 formpack_preferred_types = set(MAIN_TYPES + LABEL_OPTIONAL_TYPES + SELECT_TYPES)
 

src/formpack/utils/xform_tools.py

@@ -15,10 +15,12 @@
     ("select one from", 'select_one'),
     ("select1", 'select_one'),
     ("select one", 'select_one'),
+    ('select one from file', 'select_one_from_file'),
     ("add select multiple prompt using", 'select_multiple'),
     ("select all that apply from", 'select_multiple'),
     ("select multiple", 'select_multiple'),
     ("select all that apply", 'select_multiple'),
+    ('select multiple from file', 'select_multiple_from_file'),
     ("select_one_external", "select one external"),
     ('cascading select', 'cascading_select'),
     ('location', 'geopoint'),

tests/test_replace_aliases.py

@@ -21,6 +21,12 @@ def test_select_one_aliases_replaced():
     assert dealias_type('select_one dogs') == 'select_one dogs'
 
 
+def test_replace_select_one_from_file():
+    s1 = {'survey': [{'type': 'select one from file dogs.csv'}]}
+    replace_aliases(s1, in_place=True)
+    assert s1['survey'][0]['type'] == 'select_one_from_file dogs.csv'
+
+
 def test_true_false_value_replaced():
     # only replaced on columns with TF_COLUMNS
     s1 = {'survey': [
@@ -43,6 +49,12 @@ def test_select_multiple_aliases_replaced():
     assert dealias_type('select_multiple dogs') == 'select_multiple dogs'
 
 
+def test_replace_select_multiple_from_file():
+    s1 = {'survey': [{'type': 'select multiple from file dogs.csv'}]}
+    replace_aliases(s1, in_place=True)
+    assert s1['survey'][0]['type'] == 'select_multiple_from_file dogs.csv'
+
+
 def test_misc_types():
     assert dealias_type('begin group') == 'begin_group'
     assert dealias_type('end group') == 'end_group'

tests/test_utils_flatten_content.py

@@ -133,6 +133,27 @@ def test_flatten_select_multiple_type():
     assert ss_struct[0]['type'] == 'select_multiple yn'
 
 
+def test_flatten_select_one_from_file_type():
+    a1 = _wrap_type({'select_one_from_file': 'yn.csv'})
+    flatten_content(a1, in_place=True)
+    ss_struct = a1['survey']
+    assert ss_struct[0]['type'] == 'select_one_from_file yn.csv'
+
+
+def test_flatten_select_multiple_from_file_type():
+    a1 = _wrap_type({'select_multiple_from_file': 'yn.csv'})
+    flatten_content(a1, in_place=True)
+    ss_struct = a1['survey']
+    assert ss_struct[0]['type'] == 'select_multiple_from_file yn.csv'
+
+
+def test_flatten_rank_type():
+    a1 = _wrap_type({'rank': 'yn'})
+    flatten_content(a1, in_place=True)
+    ss_struct = a1['survey']
+    assert ss_struct[0]['type'] == 'rank yn'
+
+
 def test_json_hash():
     # consistent output
     assert json_hash({'a': 'z', 'b': 'y', 'c': 'x'}) == 'f6117d60'