Merge 303a304a28 into 4dfc66e45d

rest_framework/fields.py

@@ -1423,7 +1423,7 @@ class ListField(Field):
             if len(val) > 0:
                 # Support QueryDict lists in HTML input.
                 return val
-            return html.parse_html_list(dictionary, prefix=self.field_name)
+            return html.parse_json_form(dictionary, prefix=self.field_name)
         return dictionary.get(self.field_name, empty)
 
     def to_internal_value(self, data):
@@ -1431,7 +1431,7 @@ class ListField(Field):
         List of dicts of native values <- List of dicts of primitive datatypes.
         """
         if html.is_html_input(data):
-            data = html.parse_html_list(data)
+            data = html.parse_json_form(data)
         if isinstance(data, type('')) or not hasattr(data, '__iter__'):
             self.fail('not_a_list', input_type=type(data).__name__)
         if not self.allow_empty and len(data) == 0:
@@ -1468,7 +1468,7 @@ class DictField(Field):
         # We override the default field access in order to support
         # dictionaries in HTML forms.
         if html.is_html_input(dictionary):
-            return html.parse_html_dict(dictionary, prefix=self.field_name)
+            return html.parse_json_form(dictionary, prefix=self.field_name)
         return dictionary.get(self.field_name, empty)
 
     def to_internal_value(self, data):
@@ -1476,7 +1476,7 @@ class DictField(Field):
         Dicts of native values <- Dicts of primitive datatypes.
         """
         if html.is_html_input(data):
-            data = html.parse_html_dict(data)
+            data = html.parse_json_form(data)
         if not isinstance(data, dict):
             self.fail('not_a_dict', input_type=type(data).__name__)
         return dict([

rest_framework/serializers.py

@@ -375,7 +375,7 @@ class Serializer(BaseSerializer):
         # We override the default field access in order to support
         # nested HTML forms.
         if html.is_html_input(dictionary):
-            return html.parse_html_dict(dictionary, prefix=self.field_name) or empty
+            return html.parse_json_form(dictionary, prefix=self.field_name) or empty
         return dictionary.get(self.field_name, empty)
 
     def run_validation(self, data=empty):
@@ -525,7 +525,7 @@ class ListSerializer(BaseSerializer):
         # We override the default field access in order to support
         # lists in HTML forms.
         if html.is_html_input(dictionary):
-            return html.parse_html_list(dictionary, prefix=self.field_name)
+            return html.parse_json_form(dictionary, prefix=self.field_name)
         return dictionary.get(self.field_name, empty)
 
     def run_validation(self, data=empty):
@@ -553,7 +553,7 @@ class ListSerializer(BaseSerializer):
         List of dicts of native values <- List of dicts of primitive datatypes.
         """
         if html.is_html_input(data):
-            data = html.parse_html_list(data)
+            data = html.parse_json_form(data)
 
         if not isinstance(data, list):
             message = self.error_messages['not_a_list'].format(

rest_framework/utils/html.py

@@ -3,8 +3,6 @@ Helpers for dealing with HTML input.
 """
 import re
 
-from django.utils.datastructures import MultiValueDict
-
 
 def is_html_input(dictionary):
     # MultiDict type datastructures are used to represent HTML form input,
@@ -12,78 +10,287 @@ def is_html_input(dictionary):
     return hasattr(dictionary, 'getlist')
 
 
-def parse_html_list(dictionary, prefix=''):
+def parse_json_form(dictionary, prefix=''):
     """
-    Used to suport list values in HTML forms.
-    Supports lists of primitives and/or dictionaries.
+    Parse an HTML JSON form submission as per the W3C Draft spec
+    An implementation of "The application/json encoding algorithm"
+    http://www.w3.org/TR/html-json-forms/
+    """
+    # Step 1: Initialize output object
+    output = {}
+    for name, value in get_all_items(dictionary):
+        # TODO: implement is_file flag
 
-    * List of primitives.
+        # Step 2: Compute steps array
+        steps = parse_json_path(name)
 
-    {
-        '[0]': 'abc',
-        '[1]': 'def',
-        '[2]': 'hij'
-    }
-        -->
-    [
-        'abc',
-        'def',
-        'hij'
+        # Step 3: Initialize context
+        context = output
+
+        # Step 4: Iterate through steps
+        for step in steps:
+            # Step 4.1 Retrieve current value from context
+            current_value = get_value(context, step.key, Undefined())
+
+            # Steps 4.2, 4.3: Set JSON value on context
+            context = set_json_value(
+                context=context,
+                step=step,
+                current_value=current_value,
+                entry_value=value,
+                is_file=False,
+            )
+    # Convert any remaining Undefined array entries to None
+    output = clean_undefined(output)
+
+    # Account for DRF prefix (not part of JSON form spec)
+    result = get_value(output, prefix, Undefined())
+    if isinstance(result, Undefined):
+        return output
+    else:
+        return result
+
+
+def parse_json_path(path):
+    """
+    Parse a string as a JSON path
+    An implementation of "steps to parse a JSON encoding path"
+    http://www.w3.org/TR/html-json-forms/#dfn-steps-to-parse-a-json-encoding-path
+    """
+
+    # Steps 1, 2, 3
+    original_path = path
+    steps = []
+
+    # Step 11 (Failure)
+    failed = [
+        JsonStep(
+            type="object",
+            key=original_path,
+            last=True,
+            failed=True,
+        )
     ]
 
-    * List of dictionaries.
+    # Other variables for later use
+    digit_re = re.compile(r'^\[([0-9]+)\]')
+    key_re = re.compile(r'^\[([^\]]+)\]')
 
-    {
-        '[0]foo': 'abc',
-        '[0]bar': 'def',
-        '[1]foo': 'hij',
-        '[1]bar': 'klm',
-    }
-        -->
-    [
-        {'foo': 'abc', 'bar': 'def'},
-        {'foo': 'hij', 'bar': 'klm'}
-    ]
-    """
-    ret = {}
-    regex = re.compile(r'^%s\[([0-9]+)\](.*)$' % re.escape(prefix))
-    for field, value in dictionary.items():
-        match = regex.match(field)
-        if not match:
+    # Step 4 - Find characters before first [ (if any)
+    parts = path.split("[")
+    first_key = parts[0]
+    if parts[1:]:
+        path = "[" + "[".join(parts[1:])
+    else:
+        path = ""
+
+    # Step 5 - According to spec, keys cannot start with [
+    # NOTE: This was allowed in older DRF versions, so disabling rule for now
+    # if not first_key:
+    #     return failed
+
+    # Step 6 - Save initial step
+    steps.append(JsonStep(
+        type="object",
+        key=first_key,
+    ))
+
+    # Step 7 - Simple single-step case (no [ found)
+    if not path:
+        steps[-1].last = True
+        return steps
+
+    # Step 8 - Loop
+    while path:
+        # Step 8.1 - Check for single-item array
+        if path[:2] == "[]":
+            steps[-1].append = True
+            path = path[2:]
+            if path:
+                return failed
             continue
-        index, key = match.groups()
-        index = int(index)
-        if not key:
-            ret[index] = value
-        elif isinstance(ret.get(index), dict):
-            ret[index][key] = value
+
+        # Step 8.2 - Check for array[index]
+        digit_match = digit_re.match(path)
+        if digit_match:
+            path = digit_re.sub("", path)
+            steps.append(JsonStep(
+                type="array",
+                key=int(digit_match.group(1)),
+            ))
+            continue
+
+        # Step 8.3 - Check for object[key]
+        key_match = key_re.match(path)
+        if key_match:
+            path = key_re.sub("", path)
+            steps.append(JsonStep(
+                type="object",
+                key=key_match.group(1),
+            ))
+            continue
+
+        # Step 8.4 - Invalid key format
+        return failed
+
+    # Step 9
+    next_step = None
+    for step in reversed(steps):
+        if next_step:
+            step.next_type = next_step.type
         else:
-            ret[index] = MultiValueDict({key: [value]})
-    return [ret[item] for item in sorted(ret.keys())]
+            step.last = True
+        next_step = step
+
+    return steps
 
 
-def parse_html_dict(dictionary, prefix=''):
+def set_json_value(context, step, current_value, entry_value, is_file):
     """
-    Used to support dictionary values in HTML forms.
-
-    {
-        'profile.username': 'example',
-        'profile.email': 'example@example.com',
-    }
-        -->
-    {
-        'profile': {
-            'username': 'example',
-            'email': 'example@example.com'
-        }
-    }
+    Apply a JSON value to a context object
+    An implementation of "steps to set a JSON encoding value"
+    http://www.w3.org/TR/html-json-forms/#dfn-steps-to-set-a-json-encoding-value
     """
-    ret = {}
-    regex = re.compile(r'^%s\.(.+)$' % re.escape(prefix))
-    for field, value in dictionary.items():
-        match = regex.match(field)
-        if not match:
-            continue
-        key = match.groups()[0]
-        ret[key] = value
-    return ret
+
+    # TODO: handle is_file
+
+    # Add empty values to array so indexing works like JavaScript
+    if isinstance(context, list) and isinstance(step.key, int):
+        while len(context) <= step.key:
+            context.append(Undefined())
+
+    # Step 7: Handle last step
+    if step.last:
+        if isinstance(current_value, Undefined):
+            # Step 7.1: No existing value
+            key = step.key
+            if isinstance(context, dict) and isinstance(key, int):
+                key = str(key)
+            if step.append:
+                context[key] = [entry_value]
+            else:
+                context[key] = entry_value
+        elif isinstance(current_value, list):
+            # Step 7.2: Existing value is an Array, assume multi-valued field
+            # and add entry to end.
+
+            # FIXME: What if the other items in the array had explicit keys and
+            # this one is supposed to be the "" value?
+            # (See step 8.4 and Example 7)
+            context[step.key].append(entry_value)
+
+        elif isinstance(current_value, dict) and not is_file:
+            # Step 7.3: Existing value is an Object
+            return set_json_value(
+                context=current_value,
+                step=JsonStep(type="object", key="", last=True),
+                current_value=current_value.get("", Undefined()),
+                entry_value=entry_value,
+                is_file=is_file,
+            )
+        else:
+            # Step 7.4: Existing value is a scalar; preserve both values
+            context[step.key] = [current_value, entry_value]
+
+        # Step 7.5
+        return context
+
+    # Step 8: Handle intermediate steps
+    if isinstance(current_value, Undefined):
+        # 8.1: No existing value
+        if step.next_type == "array":
+            context[step.key] = []
+        else:
+            context[step.key] = {}
+        return context[step.key]
+    elif isinstance(current_value, dict):
+        # Step 8.2: Existing value is an Object
+        return get_value(context, step.key, Undefined())
+    elif isinstance(current_value, list):
+        # Step 8.3: Existing value is an Array
+        if step.next_type == "array":
+            return current_value
+        # Convert array to object to facilitate mixed keys
+        obj = {}
+        for i, item in enumerate(current_value):
+            if not isinstance(item, Undefined):
+                obj[str(i)] = item
+        context[step.key] = obj
+        return obj
+    else:
+        # 8.4: Existing value is a scalar; convert to Object, preserving
+        # current value via an empty key
+        obj = {'': current_value}
+        context[step.key] = obj
+        return obj
+
+
+def get_value(obj, key, default=None):
+    """
+    Mimic JavaScript Object/Array behavior by allowing access to nonexistent
+    indexes.
+    """
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    elif isinstance(obj, list):
+        try:
+            return obj[key]
+        except IndexError:
+            return default
+
+
+class Undefined(object):
+    """
+    Use Undefined() rather than None to distinguish from null.
+    """
+    pass
+
+
+class JsonStep(object):
+    """
+    Struct to represent "step" as described in HTML JSON form algorithm
+    """
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+    def __repr__(self):
+        vals = ",".join(
+            "%s=%s" % (key, val) for key, val in self.__dict__.items()
+        )
+        return "JsonStep(%s)" % vals
+
+    type = None
+    next_type = None
+    key = None
+    append = None
+    last = None
+    failed = None
+
+
+def clean_undefined(obj):
+    """
+    Convert Undefined array entries to None (null)
+    """
+    if isinstance(obj, list):
+        return [
+            None if isinstance(item, Undefined) else item
+            for item in obj
+        ]
+    if isinstance(obj, dict):
+        for key in obj:
+            obj[key] = clean_undefined(obj[key])
+    return obj
+
+
+def get_all_items(obj):
+    """
+    dict.items() but with a separate row for each value in a MultiValueDict
+    """
+    if hasattr(obj, 'getlist'):
+        items = []
+        for key in obj:
+            for value in obj.getlist(key):
+                items.append((key, value))
+        return items
+    else:
+        return obj.items()

tests/test_json_forms.py

@@ -0,0 +1,179 @@
+from django.utils.datastructures import MultiValueDict
+
+from rest_framework.utils import html
+
+
+class TestHtmlJsonExamples:
+    """
+    Tests for the HTML JSON form algorithm
+    Examples 1-10 from http://www.w3.org/TR/html-json-forms/
+    """
+
+    def test_basic_keys(self):
+        """
+        Example 1: Basic Keys
+        """
+        input_data = {
+            'name': "Bender",
+            'hind': "Bitable",
+            'shiny': True
+        }
+        expected_output = input_data
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_multiple_values(self):
+        """
+        Example 2: Multiple Values
+        """
+        input_data = MultiValueDict()
+        input_data.setlist('bottle-on-wall', [1, 2, 3])
+        expected_output = {
+            'bottle-on-wall': [1, 2, 3]
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_deeper_structure(self):
+        """
+        Example 3: Deeper Structure
+        """
+        input_data = {
+            'pet[species]': "Dahut",
+            'pet[name]': "Hypatia",
+            'kids[1]': "Thelma",
+            'kids[0]': "Ashley",
+        }
+        expected_output = {
+            'pet': {
+                'species': "Dahut",
+                'name': "Hypatia",
+            },
+            'kids': ["Ashley", "Thelma"],
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_sparse_arrays(self):
+        """
+        Example 4: Sparse Arrays
+        """
+        input_data = {
+            "hearbeat[0]": "thunk",
+            "hearbeat[2]": "thunk",
+        }
+        expected_output = {
+            "hearbeat": ["thunk", None, "thunk"]
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_even_deeper(self):
+        """
+        Example 5: Even Deeper
+        """
+        input_data = {
+            'pet[0][species]': "Dahut",
+            'pet[0][name]': "Hypatia",
+            'pet[1][species]': "Felis Stultus",
+            'pet[1][name]': "Billie",
+        }
+        expected_output = {
+            'pet': [
+                {
+                    'species': "Dahut",
+                    'name': "Hypatia",
+                },
+                {
+                    'species': "Felis Stultus",
+                    'name': "Billie",
+                }
+            ]
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_such_deep(self):
+        """
+        Example 6: Such Deep
+        """
+        input_data = {
+            'wow[such][deep][3][much][power][!]': "Amaze",
+        }
+        expected_output = {
+            'wow': {
+                'such': {
+                    'deep': [
+                        None,
+                        None,
+                        None,
+                        {
+                            'much': {
+                                'power': {
+                                    '!': "Amaze",
+                                }
+                            }
+                        }
+                    ]
+                }
+            }
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_merge_behaviour(self):
+        """
+        Example 7: Merge Behaviour
+        """
+        # FIXME: Shouldn't this work regardless of key order?
+        from rest_framework.compat import OrderedDict
+        input_data = OrderedDict([
+            ('mix', "scalar"),
+            ('mix[0]', "array 1"),
+            ('mix[2]', "array 2"),
+            ('mix[key]', "key key"),
+            ('mix[car]', "car key"),
+        ])
+        expected_output = {
+            'mix': {
+                '': "scalar",
+                '0': "array 1",
+                '2': "array 2",
+                'key': "key key",
+                'car': "car key",
+            }
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    def test_append(self):
+        """
+        Example 8: Append
+        """
+        input_data = {
+            'highlander[]': "one",
+        }
+        expected_output = {
+            'highlander': ["one"],
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output
+
+    # TODO: Example 9: Files
+
+    def test_invalid(self):
+        """
+        Example 10: Invalid
+        """
+        input_data = {
+            "error[good]": "BOOM!",
+            "error[bad": "BOOM BOOM!",
+        }
+        expected_output = {
+            'error': {
+                'good': "BOOM!",
+            },
+            'error[bad': "BOOM BOOM!",
+        }
+        result = html.parse_json_form(input_data)
+        assert result == expected_output

tests/test_serializer_lists.py

@@ -125,10 +125,10 @@ class TestListSerializerContainingNestedSerializer:
         style prefixes.
         """
         input_data = MultiValueDict({
-            "[0]integer": ["123"],
-            "[0]boolean": ["true"],
-            "[1]integer": ["456"],
-            "[1]boolean": ["false"]
+            "[0][integer]": ["123"],
+            "[0][boolean]": ["true"],
+            "[1][integer]": ["456"],
+            "[1][boolean]": ["false"]
         })
         expected_output = [
             {"integer": 123, "boolean": True},