From 1ba4dad44704d2b09050a992851b3e6b75736ee2 Mon Sep 17 00:00:00 2001 From: Alan Braithwaite Date: Wed, 2 Apr 2014 15:16:07 -0700 Subject: [PATCH 1/5] Functionality to convert field names on objects Added the necessary functionality for translating fields to have different names between the frontend and backend. --- rest_framework/serializers.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index cb7539e0b..0529874c0 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -264,6 +264,15 @@ class BaseSerializer(WritableField): """ return field_name + def get_field_name_map(self): + """ + Return a map of serialized->python field names + """ + ret = SortedDict() + for name, value in list(self.fields.items()): + ret[self.get_field_key(name)] = name + return ret + def restore_fields(self, data, files): """ Core of deserialization, together with `restore_object`. @@ -275,10 +284,20 @@ class BaseSerializer(WritableField): self._errors['non_field_errors'] = ['Invalid data'] return None + translated_data = SortedDict() + field_name_map = self.get_field_name_map() + for k,v in data.items(): + try: + python_field = field_name_map[k] + except KeyError: + pass + else: + translated_data[python_field] = v + for field_name, field in self.fields.items(): field.initialize(parent=self, field_name=field_name) try: - field.field_from_native(data, files, field_name, reverted_data) + field.field_from_native(translated_data, files, field_name, reverted_data) except ValidationError as err: self._errors[field_name] = list(err.messages) @@ -757,9 +776,9 @@ class ModelSerializer(Serializer): field.read_only = True ret[accessor_name] = field - + # Ensure that 'read_only_fields' is an iterable - assert isinstance(self.opts.read_only_fields, (list, tuple)), '`read_only_fields` must be a list or tuple' + assert isinstance(self.opts.read_only_fields, (list, tuple)), '`read_only_fields` must be a list or tuple' # Add the `read_only` flag to any fields that have been specified # in the `read_only_fields` option @@ -774,10 +793,10 @@ class ModelSerializer(Serializer): "on serializer '%s'." % (field_name, self.__class__.__name__)) ret[field_name].read_only = True - + # Ensure that 'write_only_fields' is an iterable - assert isinstance(self.opts.write_only_fields, (list, tuple)), '`write_only_fields` must be a list or tuple' - + assert isinstance(self.opts.write_only_fields, (list, tuple)), '`write_only_fields` must be a list or tuple' + for field_name in self.opts.write_only_fields: assert field_name not in self.base_fields.keys(), ( "field '%s' on serializer '%s' specified in " @@ -788,7 +807,7 @@ class ModelSerializer(Serializer): "Non-existant field '%s' specified in `write_only_fields` " "on serializer '%s'." % (field_name, self.__class__.__name__)) - ret[field_name].write_only = True + ret[field_name].write_only = True return ret From e8da3fd9ab0673a8637692ee4f2e08daa896a249 Mon Sep 17 00:00:00 2001 From: Alan Braithwaite Date: Fri, 4 Apr 2014 11:22:00 -0700 Subject: [PATCH 2/5] Handle MultiValueDicts properly --- rest_framework/serializers.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 0529874c0..34ed758f7 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -270,7 +270,10 @@ class BaseSerializer(WritableField): """ ret = SortedDict() for name, value in list(self.fields.items()): - ret[self.get_field_key(name)] = name + key = self.get_field_key(name) + if key in ret: + raise Warning() + ret[key] = name return ret def restore_fields(self, data, files): @@ -284,15 +287,22 @@ class BaseSerializer(WritableField): self._errors['non_field_errors'] = ['Invalid data'] return None - translated_data = SortedDict() - field_name_map = self.get_field_name_map() - for k,v in data.items(): - try: - python_field = field_name_map[k] - except KeyError: - pass - else: - translated_data[python_field] = v + # Handle translation of serialized fields into non serailzed fields + if data is not None: + translated_data = copy.deepcopy(data) + field_name_map = self.get_field_name_map() + for key in translated_data.keys(): + if key in field_name_map: + newkey = field_name_map.get(key) + try: # MultiValueDict + value = translated_data.getlist(key) + del translated_data[key] + translated_data.setlist(newkey, value) + except AttributeError: + value = translated_data.pop(key) + translated_data[newkey] = value + else: # Data can be None so translated_data is too + translated_data = None for field_name, field in self.fields.items(): field.initialize(parent=self, field_name=field_name) From 6f250baa062a66eaaad0dc63587432bd3cff6d23 Mon Sep 17 00:00:00 2001 From: Alan Braithwaite Date: Fri, 4 Apr 2014 14:24:02 -0700 Subject: [PATCH 3/5] Added tests --- rest_framework/serializers.py | 50 ++++++++++++++++-------- rest_framework/tests/models.py | 5 +++ rest_framework/tests/test_serializer.py | 52 +++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 34ed758f7..dba9fd9f0 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -130,6 +130,34 @@ def _is_protected_type(obj): ) +def _convert_fields(data, mapping): + """ + Takes data as dict or none and str->str map of keys and + returns None or dictionary with converted keys + """ + # Handle translation of serialized fields into non serailzed fields + if data is not None: + translated_data = copy.deepcopy(data) + for key in mapping.keys(): + if key not in translated_data: + continue + newkey = mapping.get(key) + try: # MultiValueDict + value = translated_data.getlist(key) + del translated_data[key] + translated_data.setlist(newkey, value) + except AttributeError: + value = translated_data.pop(key) + translated_data[newkey] = value + +# for key in translated_data.keys(): +# if key in mapping: + else: # Data can be None so translated_data is too + translated_data = None + + return translated_data + + def _get_declared_fields(bases, attrs): """ Create a list of serializer field instances from the passed in 'attrs', @@ -166,6 +194,7 @@ class SerializerOptions(object): self.depth = getattr(meta, 'depth', 0) self.fields = getattr(meta, 'fields', ()) self.exclude = getattr(meta, 'exclude', ()) + self.convert_fields = getattr(meta, 'convert_fields', False) class BaseSerializer(WritableField): @@ -287,27 +316,14 @@ class BaseSerializer(WritableField): self._errors['non_field_errors'] = ['Invalid data'] return None - # Handle translation of serialized fields into non serailzed fields - if data is not None: - translated_data = copy.deepcopy(data) - field_name_map = self.get_field_name_map() - for key in translated_data.keys(): - if key in field_name_map: - newkey = field_name_map.get(key) - try: # MultiValueDict - value = translated_data.getlist(key) - del translated_data[key] - translated_data.setlist(newkey, value) - except AttributeError: - value = translated_data.pop(key) - translated_data[newkey] = value - else: # Data can be None so translated_data is too - translated_data = None + if self.opts.convert_fields: + key_map = self.get_field_name_map() + data = _convert_fields(data, key_map) for field_name, field in self.fields.items(): field.initialize(parent=self, field_name=field_name) try: - field.field_from_native(translated_data, files, field_name, reverted_data) + field.field_from_native(data, files, field_name, reverted_data) except ValidationError as err: self._errors[field_name] = list(err.messages) diff --git a/rest_framework/tests/models.py b/rest_framework/tests/models.py index 6c8f2342b..7646facb7 100644 --- a/rest_framework/tests/models.py +++ b/rest_framework/tests/models.py @@ -175,3 +175,8 @@ class FilterableItem(models.Model): text = models.CharField(max_length=100) decimal = models.DecimalField(max_digits=4, decimal_places=2) date = models.DateField() + + +class ModelWithUnderscoreFields(RESTFrameworkModel): + char_field = models.CharField(max_length=100) + number_field = models.IntegerField() diff --git a/rest_framework/tests/test_serializer.py b/rest_framework/tests/test_serializer.py index 3ee2b38a7..c4851da09 100644 --- a/rest_framework/tests/test_serializer.py +++ b/rest_framework/tests/test_serializer.py @@ -7,9 +7,12 @@ from django.utils import unittest from django.utils.datastructures import MultiValueDict from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers, fields, relations -from rest_framework.tests.models import (HasPositiveIntegerAsChoice, Album, ActionItem, Anchor, BasicModel, - BlankFieldModel, BlogPost, BlogPostComment, Book, CallableDefaultValueModel, DefaultValueModel, - ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel) +from rest_framework.tests.models import ( + HasPositiveIntegerAsChoice, Album, ActionItem, Anchor, BasicModel, + BlankFieldModel, BlogPost, BlogPostComment, Book, CallableDefaultValueModel, + DefaultValueModel, ManyToManyModel, Person, ReadOnlyManyToManyModel, + Photo, RESTFrameworkModel, ModelWithUnderscoreFields + ) from rest_framework.tests.models import BasicModelSerializer import datetime import pickle @@ -176,6 +179,16 @@ class PositiveIntegerAsChoiceSerializer(serializers.ModelSerializer): fields = ['some_integer'] +class ModelFieldConversionSerializer(serializers.ModelSerializer): + + def get_field_key(self, field_name): + return field_name.replace('_','').capitalize() + + class Meta: + convert_fields = True + model = ModelWithUnderscoreFields + + class BasicTests(TestCase): def setUp(self): self.comment = Comment( @@ -331,6 +344,23 @@ class BasicTests(TestCase): exclusions = serializer.get_validation_exclusions() self.assertTrue('title' in exclusions, '`title` field was marked `required=False` and should be excluded') + def test_serialize_with_conversion(self): + """ + Verify keys get converted from serialized value to deserialized value + """ + + underscore = ModelWithUnderscoreFields(char_field='slartibartfast', number_field=42) + underscore.save() + serializer = ModelFieldConversionSerializer(underscore) + serialized = {'Id': 1, 'Numberfield': 42, 'Charfield':'slartibartfast'} + self.assertEqual(serialized, serializer.data, "Validate that serialing data with conversion works") + + serializer = ModelFieldConversionSerializer(data=serialized) + self.assertTrue(serializer.is_valid(), + 'Data should get converted from serialized value into deserialized value') + self.assertEqual('slartibartfast', serializer.object.char_field) + self.assertEqual(42, serializer.object.number_field) + class DictStyleSerializer(serializers.Serializer): """ @@ -836,6 +866,22 @@ class ManyToManyTests(TestCase): self.assertEqual(instance.pk, 2) self.assertEqual(list(instance.rel.all()), []) + def test_create_empty_relationship_flat_data_field_convert(self): + """ + Create an instance of a model with a ManyToMany relationship, + containing no items, using a representation that does not support + lists (eg form data). + """ + data = MultiValueDict() + data.setlist('rel', ['']) + self.serializer_class.Meta.convert_fields = True + serializer = self.serializer_class(data=data) + self.assertEqual(serializer.is_valid(), True) + instance = serializer.save() + self.assertEqual(len(ManyToManyModel.objects.all()), 2) + self.assertEqual(instance.pk, 2) + self.assertEqual(list(instance.rel.all()), []) + class ReadOnlyManyToManyTests(TestCase): def setUp(self): From 7d1c1b464932638499ed366ea65ef572a25a8c1d Mon Sep 17 00:00:00 2001 From: Alan Braithwaite Date: Fri, 4 Apr 2014 14:41:55 -0700 Subject: [PATCH 4/5] Cleanup kludge --- rest_framework/serializers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index dba9fd9f0..907c19ae0 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -133,7 +133,7 @@ def _is_protected_type(obj): def _convert_fields(data, mapping): """ Takes data as dict or none and str->str map of keys and - returns None or dictionary with converted keys + returns None or new dictionary with converted keys """ # Handle translation of serialized fields into non serailzed fields if data is not None: @@ -149,9 +149,6 @@ def _convert_fields(data, mapping): except AttributeError: value = translated_data.pop(key) translated_data[newkey] = value - -# for key in translated_data.keys(): -# if key in mapping: else: # Data can be None so translated_data is too translated_data = None From f9dd11df726f0b2be925ffc0827f3532b3a7e941 Mon Sep 17 00:00:00 2001 From: Alan Braithwaite Date: Mon, 7 Apr 2014 15:06:48 -0700 Subject: [PATCH 5/5] Added detailed warning about duplicate keys --- rest_framework/serializers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 907c19ae0..3d8a0797c 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -15,6 +15,7 @@ import copy import datetime import inspect import types +import warnings from decimal import Decimal from django.core.paginator import Page from django.db import models @@ -298,7 +299,9 @@ class BaseSerializer(WritableField): for name, value in list(self.fields.items()): key = self.get_field_key(name) if key in ret: - raise Warning() + warnings.warn("Duplicate key found in fields. This can happen if `get_field_key`" + " can return the same string for two different inputs! Ensure your keys are unique" + " after running them all through `get_field_key`", Warning, stacklevel=3) ret[key] = name return ret