From 7fa40cfa57cd478310d40d92d32127b2908c4f6c Mon Sep 17 00:00:00 2001 From: "S. Andrew Sheppard" Date: Thu, 12 Mar 2015 16:13:42 -0500 Subject: [PATCH 1/4] implement HTML JSON form style per spec --- rest_framework/serializers.py | 6 +- rest_framework/utils/html.py | 298 ++++++++++++++++++++++++++------- tests/test_serializer_lists.py | 8 +- 3 files changed, 240 insertions(+), 72 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index c7d4405c5..3fc3d99af 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -375,7 +375,7 @@ class Serializer(BaseSerializer): # We override the default field access in order to support # nested HTML forms. if html.is_html_input(dictionary): - return html.parse_html_dict(dictionary, prefix=self.field_name) or empty + return html.parse_json_form(dictionary, prefix=self.field_name) or empty return dictionary.get(self.field_name, empty) def run_validation(self, data=empty): @@ -525,7 +525,7 @@ class ListSerializer(BaseSerializer): # We override the default field access in order to support # lists in HTML forms. if html.is_html_input(dictionary): - return html.parse_html_list(dictionary, prefix=self.field_name) + return html.parse_json_form(dictionary, prefix=self.field_name) return dictionary.get(self.field_name, empty) def run_validation(self, data=empty): @@ -553,7 +553,7 @@ class ListSerializer(BaseSerializer): List of dicts of native values <- List of dicts of primitive datatypes. """ if html.is_html_input(data): - data = html.parse_html_list(data) + data = html.parse_json_form(data) if not isinstance(data, list): message = self.error_messages['not_a_list'].format( diff --git a/rest_framework/utils/html.py b/rest_framework/utils/html.py index c5fc5d8da..f5a915c25 100644 --- a/rest_framework/utils/html.py +++ b/rest_framework/utils/html.py @@ -3,8 +3,6 @@ Helpers for dealing with HTML input. """ import re -from django.utils.datastructures import MultiValueDict - def is_html_input(dictionary): # MultiDict type datastructures are used to represent HTML form input, @@ -12,78 +10,248 @@ def is_html_input(dictionary): return hasattr(dictionary, 'getlist') -def parse_html_list(dictionary, prefix=''): +def parse_json_form(dictionary, prefix=''): """ - Used to suport list values in HTML forms. - Supports lists of primitives and/or dictionaries. + Parse an HTML JSON form submission as per the W3C Draft spec + An implementation of "The application/json encoding algorithm" + http://www.w3.org/TR/html-json-forms/ + """ + # Step 1: Initialize output object + output = {} + for name, value in dictionary.items(): + # TODO: implement is_file flag - * List of primitives. + # Step 2: Compute steps array + steps = parse_json_path(name) - { - '[0]': 'abc', - '[1]': 'def', - '[2]': 'hij' - } - --> - [ - 'abc', - 'def', - 'hij' + # Step 3: Initialize context + context = output + + # Step 4: Iterate through steps + for step in steps: + # Step 4.1 Retrieve current value from context + current_value = get_value(context, step.key, Undefined()) + + # Steps 4.2, 4.3: Set JSON value on context + context = set_json_value( + context=context, + step=step, + current_value=current_value, + entry_value=value, + is_file=False, + ) + + # Account for DRF prefix (not part of JSON form spec) + result = get_value(output, prefix, Undefined()) + if isinstance(result, Undefined): + return output + else: + return result + + +def parse_json_path(path): + """ + Parse a string as a JSON path + An implementation of "steps to parse a JSON encoding path" + http://www.w3.org/TR/html-json-forms/#dfn-steps-to-parse-a-json-encoding-path + """ + + # Steps 1, 2, 3 + original_path = path + steps = [] + + # Step 11 (Failure) + failed = [ + JsonStep( + type="object", + key=original_path, + last=True, + failed=True, + ) ] - * List of dictionaries. + # Other variables for later use + digit_re = re.compile(r'^\[([0-9]+)\]') + key_re = re.compile(r'^\[([^\]]+)\]') - { - '[0]foo': 'abc', - '[0]bar': 'def', - '[1]foo': 'hij', - '[1]bar': 'klm', - } - --> - [ - {'foo': 'abc', 'bar': 'def'}, - {'foo': 'hij', 'bar': 'klm'} - ] - """ - ret = {} - regex = re.compile(r'^%s\[([0-9]+)\](.*)$' % re.escape(prefix)) - for field, value in dictionary.items(): - match = regex.match(field) - if not match: + # Step 4 - Find characters before first [ (if any) + parts = path.split("[") + first_key = parts[0] + if parts[1:]: + path = "[" + "[".join(parts[1:]) + else: + path = "" + + # Step 5 - According to spec, keys cannot start with [ + # NOTE: This was allowed in older DRF versions, so disabling rule for now + # if not first_key: + # return failed + + # Step 6 - Save initial step + steps.append(JsonStep( + type="object", + key=first_key, + )) + + # Step 7 - Simple single-step case (no [ found) + if not path: + steps[-1].last = True + return steps + + # Step 8 - Loop + while path: + # Step 8.1 - Check for single-item array + if path[:1] == "[]": + steps[-1].append = True + path = path[2:] + if path: + return failed continue - index, key = match.groups() - index = int(index) - if not key: - ret[index] = value - elif isinstance(ret.get(index), dict): - ret[index][key] = value + + # Step 8.2 - Check for array[index] + digit_match = digit_re.match(path) + if digit_match: + path = digit_re.sub("", path) + steps.append(JsonStep( + type="array", + key=int(digit_match.group(1)), + )) + continue + + # Step 8.3 - Check for object[key] + key_match = key_re.match(path) + if key_match: + path = key_re.sub("", path) + steps.append(JsonStep( + type="object", + key=key_match.group(1), + )) + continue + + # Step 8.4 - Invalid key format + return failed + + # Step 9 + next_step = None + for step in reversed(steps): + if next_step: + step.next_type = next_step.type else: - ret[index] = MultiValueDict({key: [value]}) - return [ret[item] for item in sorted(ret.keys())] + step.last = True + next_step = step + + return steps -def parse_html_dict(dictionary, prefix=''): +def set_json_value(context, step, current_value, entry_value, is_file): """ - Used to support dictionary values in HTML forms. - - { - 'profile.username': 'example', - 'profile.email': 'example@example.com', - } - --> - { - 'profile': { - 'username': 'example', - 'email': 'example@example.com' - } - } + Apply a JSON value to a context object + An implementation of "steps to set a JSON encoding value" + http://www.w3.org/TR/html-json-forms/#dfn-steps-to-set-a-json-encoding-value """ - ret = {} - regex = re.compile(r'^%s\.(.+)$' % re.escape(prefix)) - for field, value in dictionary.items(): - match = regex.match(field) - if not match: - continue - key = match.groups()[0] - ret[key] = value - return ret + + # TODO: handle is_file + + # Add empty values to array so indexing works like JavaScript + # TODO: According to spec, these should turn into nulls if they are never + # overriden in a later step. + if isinstance(context, list) and isinstance(step.key, int): + while len(context) <= step.key: + context.append(Undefined()) + + # Step 7: Handle last step + if step.last: + if isinstance(current_value, Undefined): + # Step 7.1: No existing value + if step.append: + context[step.key] = [entry_value] + else: + context[step.key] = entry_value + elif isinstance(current_value, list): + # Step 7.2: Existing value is an Array + context[step.key].append(entry_value) + elif isinstance(current_value, dict) and not is_file: + # Step 7.3: Existing value is an Object + return set_json_value( + context=current_value, + step=JsonStep(type="object", key="", last=True), + current_value=current_value.get("", Undefined()), + entry_value=entry_value, + is_file=is_file, + ) + else: + # Step 7.4: Existing value is a scalar; preserve both values + context[step.key] = [current_value, entry_value] + + # Step 7.5 + return context + + # Step 8: Handle intermediate steps + if isinstance(current_value, Undefined): + # 8.1: No existing value + if step.next_type == "array": + context[step.key] = [] + else: + context[step.key] = {} + return context[step.key] + elif isinstance(current_value, dict): + # Step 7.2: Existing value is an Object + return get_value(context, step.key, Undefined()) + elif isinstance(current_value, list): + # Step 7.3: Existing value is an Array + if step.next_type == "array": + return current_value + obj = {} + for i, item in enumerate(current_value): + if not isinstance(item, Undefined): + obj[i] = item + context[step.key] = obj + return obj + else: + # 7.4: Existing value is a scalar; convert to Object, preserving + # current value via an empty key + obj = {'': current_value} + context[step.key] = obj + return obj + + +def get_value(obj, key, default=None): + """ + Mimic JavaScript Object/Array behavior by allowing access to nonexistent + indexes. + """ + if isinstance(obj, dict): + return obj.get(key, default) + elif isinstance(obj, list): + try: + return obj[key] + except IndexError: + return default + + +class Undefined(object): + """ + Use Undefined() rather than None to distinguish from null. + """ + pass + + +class JsonStep(object): + """ + Struct to represent "step" as described in HTML JSON form algorithm + """ + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __repr__(self): + vals = ",".join( + "%s=%s" % (key, val) for key, val in self.__dict__.items() + ) + return "JsonStep(%s)" % vals + + type = None + next_type = None + key = None + append = None + last = None + failed = None diff --git a/tests/test_serializer_lists.py b/tests/test_serializer_lists.py index e9234d8f7..f2454c150 100644 --- a/tests/test_serializer_lists.py +++ b/tests/test_serializer_lists.py @@ -125,10 +125,10 @@ class TestListSerializerContainingNestedSerializer: style prefixes. """ input_data = MultiValueDict({ - "[0]integer": ["123"], - "[0]boolean": ["true"], - "[1]integer": ["456"], - "[1]boolean": ["false"] + "[0][integer]": ["123"], + "[0][boolean]": ["true"], + "[1][integer]": ["456"], + "[1][boolean]": ["false"] }) expected_output = [ {"integer": 123, "boolean": True}, From 19fa92ebd311bdbe93da4d955da236875f352203 Mon Sep 17 00:00:00 2001 From: "S. Andrew Sheppard" Date: Thu, 12 Mar 2015 18:11:14 -0500 Subject: [PATCH 2/4] tests for examples listed in spec --- rest_framework/utils/html.py | 65 ++++++++++--- tests/test_json_forms.py | 178 +++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 13 deletions(-) create mode 100644 tests/test_json_forms.py diff --git a/rest_framework/utils/html.py b/rest_framework/utils/html.py index f5a915c25..77e1402d7 100644 --- a/rest_framework/utils/html.py +++ b/rest_framework/utils/html.py @@ -18,7 +18,7 @@ def parse_json_form(dictionary, prefix=''): """ # Step 1: Initialize output object output = {} - for name, value in dictionary.items(): + for name, value in get_all_items(dictionary): # TODO: implement is_file flag # Step 2: Compute steps array @@ -40,6 +40,8 @@ def parse_json_form(dictionary, prefix=''): entry_value=value, is_file=False, ) + # Convert any remaining Undefined array entries to None + output = clean_undefined(output) # Account for DRF prefix (not part of JSON form spec) result = get_value(output, prefix, Undefined()) @@ -101,7 +103,7 @@ def parse_json_path(path): # Step 8 - Loop while path: # Step 8.1 - Check for single-item array - if path[:1] == "[]": + if path[:2] == "[]": steps[-1].append = True path = path[2:] if path: @@ -153,8 +155,6 @@ def set_json_value(context, step, current_value, entry_value, is_file): # TODO: handle is_file # Add empty values to array so indexing works like JavaScript - # TODO: According to spec, these should turn into nulls if they are never - # overriden in a later step. if isinstance(context, list) and isinstance(step.key, int): while len(context) <= step.key: context.append(Undefined()) @@ -163,13 +163,22 @@ def set_json_value(context, step, current_value, entry_value, is_file): if step.last: if isinstance(current_value, Undefined): # Step 7.1: No existing value + key = step.key + if isinstance(context, dict) and isinstance(key, int): + key = str(key) if step.append: - context[step.key] = [entry_value] + context[key] = [entry_value] else: - context[step.key] = entry_value + context[key] = entry_value elif isinstance(current_value, list): - # Step 7.2: Existing value is an Array + # Step 7.2: Existing value is an Array, assume multi-valued field + # and add entry to end. + + # FIXME: What if the other items in the array had explicit keys and + # this one is supposed to be the "" value? + # (See step 8.4 and Example 7) context[step.key].append(entry_value) + elif isinstance(current_value, dict) and not is_file: # Step 7.3: Existing value is an Object return set_json_value( @@ -195,20 +204,21 @@ def set_json_value(context, step, current_value, entry_value, is_file): context[step.key] = {} return context[step.key] elif isinstance(current_value, dict): - # Step 7.2: Existing value is an Object + # Step 8.2: Existing value is an Object return get_value(context, step.key, Undefined()) elif isinstance(current_value, list): - # Step 7.3: Existing value is an Array + # Step 8.3: Existing value is an Array if step.next_type == "array": return current_value + # Convert array to object to facilitate mixed keys obj = {} for i, item in enumerate(current_value): if not isinstance(item, Undefined): - obj[i] = item - context[step.key] = obj - return obj + obj[str(i)] = item + context[step.key] = obj + return obj else: - # 7.4: Existing value is a scalar; convert to Object, preserving + # 8.4: Existing value is a scalar; convert to Object, preserving # current value via an empty key obj = {'': current_value} context[step.key] = obj @@ -255,3 +265,32 @@ class JsonStep(object): append = None last = None failed = None + + +def clean_undefined(obj): + """ + Convert Undefined array entries to None (null) + """ + if isinstance(obj, list): + return [ + None if isinstance(item, Undefined) else item + for item in obj + ] + if isinstance(obj, dict): + for key in obj: + obj[key] = clean_undefined(obj[key]) + return obj + + +def get_all_items(obj): + """ + dict.items() but with a separate row for each value in a MultiValueDict + """ + if hasattr(obj, 'getlist'): + items = [] + for key in obj: + for value in obj.getlist(key): + items.append((key, value)) + return items + else: + return obj.items() diff --git a/tests/test_json_forms.py b/tests/test_json_forms.py new file mode 100644 index 000000000..e98247b83 --- /dev/null +++ b/tests/test_json_forms.py @@ -0,0 +1,178 @@ +from rest_framework.utils import html +from django.utils.datastructures import MultiValueDict + + +class TestHtmlJsonExamples: + """ + Tests for the HTML JSON form algorithm + Examples 1-10 from http://www.w3.org/TR/html-json-forms/ + """ + + def test_basic_keys(self): + """ + Example 1: Basic Keys + """ + input_data = { + 'name': "Bender", + 'hind': "Bitable", + 'shiny': True + } + expected_output = input_data + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_multiple_values(self): + """ + Example 2: Multiple Values + """ + input_data = MultiValueDict() + input_data.setlist('bottle-on-wall', [1, 2, 3]) + expected_output = { + 'bottle-on-wall': [1, 2, 3] + } + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_deeper_structure(self): + """ + Example 3: Deeper Structure + """ + input_data = { + 'pet[species]': "Dahut", + 'pet[name]': "Hypatia", + 'kids[1]': "Thelma", + 'kids[0]': "Ashley", + } + expected_output = { + 'pet': { + 'species': "Dahut", + 'name': "Hypatia", + }, + 'kids': ["Ashley", "Thelma"], + } + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_sparse_arrays(self): + """ + Example 4: Sparse Arrays + """ + input_data = { + "hearbeat[0]": "thunk", + "hearbeat[2]": "thunk", + } + expected_output = { + "hearbeat": ["thunk", None, "thunk"] + } + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_even_deeper(self): + """ + Example 5: Even Deeper + """ + input_data = { + 'pet[0][species]': "Dahut", + 'pet[0][name]': "Hypatia", + 'pet[1][species]': "Felis Stultus", + 'pet[1][name]': "Billie", + } + expected_output = { + 'pet': [ + { + 'species': "Dahut", + 'name': "Hypatia", + }, + { + 'species': "Felis Stultus", + 'name': "Billie", + } + ] + } + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_such_deep(self): + """ + Example 6: Such Deep + """ + input_data = { + 'wow[such][deep][3][much][power][!]': "Amaze", + } + expected_output = { + 'wow': { + 'such': { + 'deep': [ + None, + None, + None, + { + 'much': { + 'power': { + '!': "Amaze", + } + } + } + ] + } + } + } + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_merge_behaviour(self): + """ + Example 7: Merge Behaviour + """ + # FIXME: Shouldn't this work regardless of key order? + from rest_framework.compat import OrderedDict + input_data = OrderedDict([ + ('mix', "scalar"), + ('mix[0]', "array 1"), + ('mix[2]', "array 2"), + ('mix[key]', "key key"), + ('mix[car]', "car key"), + ]) + expected_output = { + 'mix': { + '': "scalar", + '0': "array 1", + '2': "array 2", + 'key': "key key", + 'car': "car key", + } + } + result = html.parse_json_form(input_data) + assert result == expected_output + + def test_append(self): + """ + Example 8: Append + """ + input_data = { + 'highlander[]': "one", + } + expected_output = { + 'highlander': ["one"], + } + result = html.parse_json_form(input_data) + assert result == expected_output + + # TODO: Example 9: Files + + def test_invalid(self): + """ + Example 10: Invalid + """ + input_data = { + "error[good]": "BOOM!", + "error[bad": "BOOM BOOM!", + } + expected_output = { + 'error': { + 'good': "BOOM!", + }, + 'error[bad': "BOOM BOOM!", + } + result = html.parse_json_form(input_data) + assert result == expected_output From b209e35fd75b41580cc05de0904404ec5abc6a22 Mon Sep 17 00:00:00 2001 From: "S. Andrew Sheppard" Date: Wed, 18 Mar 2015 11:47:53 -0500 Subject: [PATCH 3/4] replace remaining references to old api --- rest_framework/fields.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest_framework/fields.py b/rest_framework/fields.py index 1d12b1d92..5c4cba588 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -1387,7 +1387,7 @@ class ListField(Field): if len(val) > 1: # Support QueryDict lists in HTML input. return val - return html.parse_html_list(dictionary, prefix=self.field_name) + return html.parse_json_form(dictionary, prefix=self.field_name) return dictionary.get(self.field_name, empty) def to_internal_value(self, data): @@ -1395,7 +1395,7 @@ class ListField(Field): List of dicts of native values <- List of dicts of primitive datatypes. """ if html.is_html_input(data): - data = html.parse_html_list(data) + data = html.parse_json_form(data) if isinstance(data, type('')) or not hasattr(data, '__iter__'): self.fail('not_a_list', input_type=type(data).__name__) if not self.allow_empty and len(data) == 0: @@ -1426,7 +1426,7 @@ class DictField(Field): # We override the default field access in order to support # dictionaries in HTML forms. if html.is_html_input(dictionary): - return html.parse_html_dict(dictionary, prefix=self.field_name) + return html.parse_json_form(dictionary, prefix=self.field_name) return dictionary.get(self.field_name, empty) def to_internal_value(self, data): @@ -1434,7 +1434,7 @@ class DictField(Field): Dicts of native values <- Dicts of primitive datatypes. """ if html.is_html_input(data): - data = html.parse_html_dict(data) + data = html.parse_json_form(data) if not isinstance(data, dict): self.fail('not_a_dict', input_type=type(data).__name__) return dict([ From 303a304a28cff3d7b48e8a6bf142e19ca73ab359 Mon Sep 17 00:00:00 2001 From: "S. Andrew Sheppard" Date: Thu, 6 Aug 2015 09:39:38 -0500 Subject: [PATCH 4/4] fix import order --- tests/test_json_forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_json_forms.py b/tests/test_json_forms.py index e98247b83..a3c31612b 100644 --- a/tests/test_json_forms.py +++ b/tests/test_json_forms.py @@ -1,6 +1,7 @@ -from rest_framework.utils import html from django.utils.datastructures import MultiValueDict +from rest_framework.utils import html + class TestHtmlJsonExamples: """