From 6b4bb48dd410d0a878b0142d351c7c41cd51f819 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 12 Mar 2013 13:33:02 +0000 Subject: [PATCH 01/50] Initial support for writable nested serialization (Not ModelSerializer) --- rest_framework/serializers.py | 70 +++++++++++++++++------ rest_framework/tests/serializer_nested.py | 62 ++++++++++++++++++++ 2 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 rest_framework/tests/serializer_nested.py diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 2ae7c215f..81619b3af 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -20,6 +20,11 @@ from rest_framework.relations import * from rest_framework.fields import * +class NestedValidationError(ValidationError): + def __init__(self, message): + self.messages = message + + class DictWithMetadata(dict): """ A dict-like object, that can have additional properties attached. @@ -98,7 +103,7 @@ class SerializerOptions(object): self.exclude = getattr(meta, 'exclude', ()) -class BaseSerializer(Field): +class BaseSerializer(WritableField): """ This is the Serializer implementation. We need to implement it as `BaseSerializer` due to metaclass magicks. @@ -303,33 +308,64 @@ class BaseSerializer(Field): return self.to_native(obj) try: - if self.source: - for component in self.source.split('.'): - obj = getattr(obj, component) - if is_simple_callable(obj): - obj = obj() - else: - obj = getattr(obj, field_name) - if is_simple_callable(obj): - obj = obj() + source = self.source or field_name + value = obj + + for component in source.split('.'): + value = get_component(value, component) + if value is None: + break except ObjectDoesNotExist: return None - # If the object has an "all" method, assume it's a relationship - if is_simple_callable(getattr(obj, 'all', None)): - return [self.to_native(item) for item in obj.all()] + if is_simple_callable(getattr(value, 'all', None)): + return [self.to_native(item) for item in value.all()] - if obj is None: + if value is None: return None if self.many is not None: many = self.many else: - many = hasattr(obj, '__iter__') and not isinstance(obj, (Page, dict, six.text_type)) + many = hasattr(value, '__iter__') and not isinstance(value, (Page, dict, six.text_type)) if many: - return [self.to_native(item) for item in obj] - return self.to_native(obj) + return [self.to_native(item) for item in value] + return self.to_native(value) + + def field_from_native(self, data, files, field_name, into): + if self.read_only: + return + + try: + value = data[field_name] + except KeyError: + if self.required: + raise ValidationError(self.error_messages['required']) + return + + if self.parent.object: + # Set the serializer object if it exists + obj = getattr(self.parent.object, field_name) + self.object = obj + + if value in (None, ''): + into[(self.source or field_name)] = None + else: + kwargs = { + 'data': value, + 'context': self.context, + 'partial': self.partial, + 'many': self.many + } + serializer = self.__class__(**kwargs) + + if serializer.is_valid(): + self.object = serializer.object + into[self.source or field_name] = serializer.object + else: + # Propagate errors up to our parent + raise NestedValidationError(serializer.errors) @property def errors(self): diff --git a/rest_framework/tests/serializer_nested.py b/rest_framework/tests/serializer_nested.py new file mode 100644 index 000000000..c8987bc56 --- /dev/null +++ b/rest_framework/tests/serializer_nested.py @@ -0,0 +1,62 @@ +from __future__ import unicode_literals +from django.test import TestCase +from rest_framework import serializers + + +class TrackSerializer(serializers.Serializer): + order = serializers.IntegerField() + title = serializers.CharField(max_length=100) + duration = serializers.IntegerField() + + +class AlbumSerializer(serializers.Serializer): + album_name = serializers.CharField(max_length=100) + artist = serializers.CharField(max_length=100) + tracks = TrackSerializer(many=True) + + +class NestedSerializerTestCase(TestCase): + def test_nested_validation_success(self): + """ + Correct nested serialization should return the input data. + """ + + data = { + 'album_name': 'Discovery', + 'artist': 'Daft Punk', + 'tracks': [ + {'order': 1, 'title': 'One More Time', 'duration': 235}, + {'order': 2, 'title': 'Aerodynamic', 'duration': 184}, + {'order': 3, 'title': 'Digital Love', 'duration': 239} + ] + } + + serializer = AlbumSerializer(data=data) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.data, data) + + def test_nested_validation_error(self): + """ + Incorrect nested serialization should return appropriate error data. + """ + + data = { + 'album_name': 'Discovery', + 'artist': 'Daft Punk', + 'tracks': [ + {'order': 1, 'title': 'One More Time', 'duration': 235}, + {'order': 2, 'title': 'Aerodynamic', 'duration': 184}, + {'order': 3, 'title': 'Digital Love', 'duration': 'foobar'} + ] + } + expected_errors = { + 'tracks': [ + {}, + {}, + {'duration': ['Enter a whole number.']} + ] + } + + serializer = AlbumSerializer(data=data) + self.assertEqual(serializer.is_valid(), False) + self.assertEqual(serializer.errors, expected_errors) From d8c5dca9aea5ad073936c1c5b3975ed53a6aeca8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 12 Mar 2013 18:34:52 +0000 Subject: [PATCH 02/50] Extra tests for nested serializers --- rest_framework/tests/serializer_nested.py | 206 ++++++++++++++++++++-- 1 file changed, 194 insertions(+), 12 deletions(-) diff --git a/rest_framework/tests/serializer_nested.py b/rest_framework/tests/serializer_nested.py index c8987bc56..fcf644c75 100644 --- a/rest_framework/tests/serializer_nested.py +++ b/rest_framework/tests/serializer_nested.py @@ -1,21 +1,30 @@ +""" +Tests to cover nested serializers. +""" from __future__ import unicode_literals from django.test import TestCase from rest_framework import serializers -class TrackSerializer(serializers.Serializer): - order = serializers.IntegerField() - title = serializers.CharField(max_length=100) - duration = serializers.IntegerField() +class WritableNestedSerializerBasicTests(TestCase): + """ + Tests for deserializing nested entities. + Basic tests that use serializers that simply restore to dicts. + """ + def setUp(self): + class TrackSerializer(serializers.Serializer): + order = serializers.IntegerField() + title = serializers.CharField(max_length=100) + duration = serializers.IntegerField() -class AlbumSerializer(serializers.Serializer): - album_name = serializers.CharField(max_length=100) - artist = serializers.CharField(max_length=100) - tracks = TrackSerializer(many=True) + class AlbumSerializer(serializers.Serializer): + album_name = serializers.CharField(max_length=100) + artist = serializers.CharField(max_length=100) + tracks = TrackSerializer(many=True) + self.AlbumSerializer = AlbumSerializer -class NestedSerializerTestCase(TestCase): def test_nested_validation_success(self): """ Correct nested serialization should return the input data. @@ -31,9 +40,9 @@ class NestedSerializerTestCase(TestCase): ] } - serializer = AlbumSerializer(data=data) + serializer = self.AlbumSerializer(data=data) self.assertEqual(serializer.is_valid(), True) - self.assertEqual(serializer.data, data) + self.assertEqual(serializer.object, data) def test_nested_validation_error(self): """ @@ -57,6 +66,179 @@ class NestedSerializerTestCase(TestCase): ] } - serializer = AlbumSerializer(data=data) + serializer = self.AlbumSerializer(data=data) self.assertEqual(serializer.is_valid(), False) self.assertEqual(serializer.errors, expected_errors) + + def test_many_nested_validation_error(self): + """ + Incorrect nested serialization should return appropriate error data + when multiple entities are being deserialized. + """ + + data = [ + { + 'album_name': 'Russian Red', + 'artist': 'I Love Your Glasses', + 'tracks': [ + {'order': 1, 'title': 'Cigarettes', 'duration': 121}, + {'order': 2, 'title': 'No Past Land', 'duration': 198}, + {'order': 3, 'title': 'They Don\'t Believe', 'duration': 191} + ] + }, + { + 'album_name': 'Discovery', + 'artist': 'Daft Punk', + 'tracks': [ + {'order': 1, 'title': 'One More Time', 'duration': 235}, + {'order': 2, 'title': 'Aerodynamic', 'duration': 184}, + {'order': 3, 'title': 'Digital Love', 'duration': 'foobar'} + ] + } + ] + expected_errors = [ + {}, + { + 'tracks': [ + {}, + {}, + {'duration': ['Enter a whole number.']} + ] + } + ] + + serializer = self.AlbumSerializer(data=data) + self.assertEqual(serializer.is_valid(), False) + self.assertEqual(serializer.errors, expected_errors) + + +class WritableNestedSerializerObjectTests(TestCase): + """ + Tests for deserializing nested entities. + These tests use serializers that restore to concrete objects. + """ + + def setUp(self): + # Couple of concrete objects that we're going to deserialize into + class Track(object): + def __init__(self, order, title, duration): + self.order, self.title, self.duration = order, title, duration + + def __cmp__(self, other): + return ( + self.order == other.order and + self.title == other.title and + self.duration == other.duration + ) + + class Album(object): + def __init__(self, album_name, artist, tracks): + self.album_name, self.artist, self.tracks = album_name, artist, tracks + + def __cmp__(self, other): + return ( + self.album_name == other.album_name and + self.artist == other.artist and + self.tracks == other.tracks + ) + + # And their corresponding serializers + class TrackSerializer(serializers.Serializer): + order = serializers.IntegerField() + title = serializers.CharField(max_length=100) + duration = serializers.IntegerField() + + def restore_object(self, attrs, instance=None): + return Track(attrs['order'], attrs['title'], attrs['duration']) + + class AlbumSerializer(serializers.Serializer): + album_name = serializers.CharField(max_length=100) + artist = serializers.CharField(max_length=100) + tracks = TrackSerializer(many=True) + + def restore_object(self, attrs, instance=None): + return Album(attrs['album_name'], attrs['artist'], attrs['tracks']) + + self.Album, self.Track = Album, Track + self.AlbumSerializer = AlbumSerializer + + def test_nested_validation_success(self): + """ + Correct nested serialization should return a restored object + that corresponds to the input data. + """ + + data = { + 'album_name': 'Discovery', + 'artist': 'Daft Punk', + 'tracks': [ + {'order': 1, 'title': 'One More Time', 'duration': 235}, + {'order': 2, 'title': 'Aerodynamic', 'duration': 184}, + {'order': 3, 'title': 'Digital Love', 'duration': 239} + ] + } + expected_object = self.Album( + album_name='Discovery', + artist='Daft Punk', + tracks=[ + self.Track(order=1, title='One More Time', duration=235), + self.Track(order=2, title='Aerodynamic', duration=184), + self.Track(order=3, title='Digital Love', duration=239), + ] + ) + + serializer = self.AlbumSerializer(data=data) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.object, expected_object) + + def test_many_nested_validation_success(self): + """ + Correct nested serialization should return multiple restored objects + that corresponds to the input data when multiple objects are + being deserialized. + """ + + data = [ + { + 'album_name': 'Russian Red', + 'artist': 'I Love Your Glasses', + 'tracks': [ + {'order': 1, 'title': 'Cigarettes', 'duration': 121}, + {'order': 2, 'title': 'No Past Land', 'duration': 198}, + {'order': 3, 'title': 'They Don\'t Believe', 'duration': 191} + ] + }, + { + 'album_name': 'Discovery', + 'artist': 'Daft Punk', + 'tracks': [ + {'order': 1, 'title': 'One More Time', 'duration': 235}, + {'order': 2, 'title': 'Aerodynamic', 'duration': 184}, + {'order': 3, 'title': 'Digital Love', 'duration': 239} + ] + } + ] + expected_object = [ + self.Album( + album_name='Russian Red', + artist='I Love Your Glasses', + tracks=[ + self.Track(order=1, title='Cigarettes', duration=121), + self.Track(order=2, title='No Past Land', duration=198), + self.Track(order=3, title='They Don\'t Believe', duration=191), + ] + ), + self.Album( + album_name='Discovery', + artist='Daft Punk', + tracks=[ + self.Track(order=1, title='One More Time', duration=235), + self.Track(order=2, title='Aerodynamic', duration=184), + self.Track(order=3, title='Digital Love', duration=239), + ] + ) + ] + + serializer = self.AlbumSerializer(data=data) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.object, expected_object) From 2f1951910f264852b530c94c3a9946afe10eedd2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 12 Mar 2013 18:35:20 +0000 Subject: [PATCH 03/50] Descriptive text for NestedValidationError --- rest_framework/serializers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 81619b3af..f83451d37 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -21,6 +21,16 @@ from rest_framework.fields import * class NestedValidationError(ValidationError): + """ + The default ValidationError behavior is to stringify each item in the list + if the messages are a list of error messages. + + In the case of nested serializers, where the parent has many children, + then the child's `serializer.errors` will be a list of dicts. + + We need to override the default behavior to get properly nested error dicts. + """ + def __init__(self, message): self.messages = message From 3006e3825f29e920f881b816fd71566bf0e8d341 Mon Sep 17 00:00:00 2001 From: Mark Aaron Shirley Date: Tue, 12 Mar 2013 20:59:25 -0700 Subject: [PATCH 04/50] One-to-one writable, nested serializer support --- rest_framework/serializers.py | 44 ++++++-- rest_framework/tests/nesting.py | 125 ++++++++++++++++++++++ rest_framework/tests/serializer_nested.py | 4 +- 3 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 rest_framework/tests/nesting.py diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index f83451d37..893db2ece 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -26,13 +26,17 @@ class NestedValidationError(ValidationError): if the messages are a list of error messages. In the case of nested serializers, where the parent has many children, - then the child's `serializer.errors` will be a list of dicts. + then the child's `serializer.errors` will be a list of dicts. In the case + of a single child, the `serializer.errors` will be a dict. We need to override the default behavior to get properly nested error dicts. """ def __init__(self, message): - self.messages = message + if isinstance(message, dict): + self.messages = [message] + else: + self.messages = message class DictWithMetadata(dict): @@ -143,6 +147,7 @@ class BaseSerializer(WritableField): self._data = None self._files = None self._errors = None + self._delete = False ##### # Methods to determine which fields to use when (de)serializing objects. @@ -354,15 +359,19 @@ class BaseSerializer(WritableField): raise ValidationError(self.error_messages['required']) return - if self.parent.object: - # Set the serializer object if it exists - obj = getattr(self.parent.object, field_name) - self.object = obj + # Set the serializer object if it exists + obj = getattr(self.parent.object, field_name) if self.parent.object else None if value in (None, ''): - into[(self.source or field_name)] = None + if isinstance(self, ModelSerializer): + self._delete = True + self.object = obj + into[(self.source or field_name)] = self + else: + into[(self.source or field_name)] = None else: kwargs = { + 'instance': obj, 'data': value, 'context': self.context, 'partial': self.partial, @@ -371,8 +380,10 @@ class BaseSerializer(WritableField): serializer = self.__class__(**kwargs) if serializer.is_valid(): - self.object = serializer.object - into[self.source or field_name] = serializer.object + if isinstance(serializer, ModelSerializer): + into[self.source or field_name] = serializer + else: + into[self.source or field_name] = serializer.object else: # Propagate errors up to our parent raise NestedValidationError(serializer.errors) @@ -664,10 +675,17 @@ class ModelSerializer(Serializer): if instance: return self.full_clean(instance) - def save_object(self, obj): + def save_object(self, obj, parent=None, fk_field=None): """ Save the deserialized object and return it. """ + if self._delete: + obj.delete() + return + + if parent and fk_field: + setattr(self.object, fk_field, parent) + obj.save() if getattr(self, 'm2m_data', None): @@ -677,7 +695,11 @@ class ModelSerializer(Serializer): if getattr(self, 'related_data', None): for accessor_name, object_list in self.related_data.items(): - setattr(self.object, accessor_name, object_list) + if isinstance(object_list, ModelSerializer): + fk_field = self.object._meta.get_field_by_name(accessor_name)[0].field.name + object_list.save_object(object_list.object, parent=self.object, fk_field=fk_field) + else: + setattr(self.object, accessor_name, object_list) self.related_data = {} diff --git a/rest_framework/tests/nesting.py b/rest_framework/tests/nesting.py new file mode 100644 index 000000000..35b7a365d --- /dev/null +++ b/rest_framework/tests/nesting.py @@ -0,0 +1,125 @@ +from __future__ import unicode_literals +from django.db import models +from django.test import TestCase +from rest_framework import serializers + + +class OneToOneTarget(models.Model): + name = models.CharField(max_length=100) + + +class OneToOneTargetSource(models.Model): + name = models.CharField(max_length=100) + target = models.OneToOneField(OneToOneTarget, null=True, blank=True, + related_name='target_source') + + +class OneToOneSource(models.Model): + name = models.CharField(max_length=100) + target_source = models.OneToOneField(OneToOneTargetSource, related_name='source') + + +class OneToOneSourceSerializer(serializers.ModelSerializer): + class Meta: + model = OneToOneSource + exclude = ('target_source', ) + + +class OneToOneTargetSourceSerializer(serializers.ModelSerializer): + source = OneToOneSourceSerializer() + + class Meta: + model = OneToOneTargetSource + exclude = ('target', ) + +class OneToOneTargetSerializer(serializers.ModelSerializer): + target_source = OneToOneTargetSourceSerializer() + + class Meta: + model = OneToOneTarget + + +class NestedOneToOneTests(TestCase): + def setUp(self): + for idx in range(1, 4): + target = OneToOneTarget(name='target-%d' % idx) + target.save() + target_source = OneToOneTargetSource(name='target-source-%d' % idx, target=target) + target_source.save() + source = OneToOneSource(name='source-%d' % idx, target_source=target_source) + source.save() + + def test_one_to_one_retrieve(self): + queryset = OneToOneTarget.objects.all() + serializer = OneToOneTargetSerializer(queryset) + expected = [ + {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, + {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, + {'id': 3, 'name': 'target-3', 'target_source': {'id': 3, 'name': 'target-source-3', 'source': {'id': 3, 'name': 'source-3'}}} + ] + self.assertEqual(serializer.data, expected) + + + def test_one_to_one_create(self): + data = {'id': 4, 'name': 'target-4', 'target_source': {'id': 4, 'name': 'target-source-4', 'source': {'id': 4, 'name': 'source-4'}}} + serializer = OneToOneTargetSerializer(data=data) + self.assertTrue(serializer.is_valid()) + obj = serializer.save() + self.assertEqual(serializer.data, data) + self.assertEqual(obj.name, 'target-4') + + # Ensure (target 4, target_source 4, source 4) are added, and + # everything else is as expected. + queryset = OneToOneTarget.objects.all() + serializer = OneToOneTargetSerializer(queryset) + expected = [ + {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, + {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, + {'id': 3, 'name': 'target-3', 'target_source': {'id': 3, 'name': 'target-source-3', 'source': {'id': 3, 'name': 'source-3'}}}, + {'id': 4, 'name': 'target-4', 'target_source': {'id': 4, 'name': 'target-source-4', 'source': {'id': 4, 'name': 'source-4'}}} + ] + self.assertEqual(serializer.data, expected) + + def test_one_to_one_create_with_invalid_data(self): + data = {'id': 4, 'name': 'target-4', 'target_source': {'id': 4, 'name': 'target-source-4', 'source': {'id': 4}}} + serializer = OneToOneTargetSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertEqual(serializer.errors, {'target_source': [{'source': [{'name': ['This field is required.']}]}]}) + + def test_one_to_one_update(self): + data = {'id': 3, 'name': 'target-3-updated', 'target_source': {'id': 3, 'name': 'target-source-3-updated', 'source': {'id': 3, 'name': 'source-3-updated'}}} + instance = OneToOneTarget.objects.get(pk=3) + serializer = OneToOneTargetSerializer(instance, data=data) + self.assertTrue(serializer.is_valid()) + obj = serializer.save() + self.assertEqual(serializer.data, data) + self.assertEqual(obj.name, 'target-3-updated') + + # Ensure (target 3, target_source 3, source 3) are updated, + # and everything else is as expected. + queryset = OneToOneTarget.objects.all() + serializer = OneToOneTargetSerializer(queryset) + expected = [ + {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, + {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, + {'id': 3, 'name': 'target-3-updated', 'target_source': {'id': 3, 'name': 'target-source-3-updated', 'source': {'id': 3, 'name': 'source-3-updated'}}} + ] + self.assertEqual(serializer.data, expected) + + def test_one_to_one_delete(self): + data = {'id': 3, 'name': 'target-3', 'target_source': None} + instance = OneToOneTarget.objects.get(pk=3) + serializer = OneToOneTargetSerializer(instance, data=data) + self.assertTrue(serializer.is_valid()) + obj = serializer.save() + + # Ensure (target_source 3, source 3) are deleted, + # and everything else is as expected. + queryset = OneToOneTarget.objects.all() + serializer = OneToOneTargetSerializer(queryset) + expected = [ + {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, + {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, + {'id': 3, 'name': 'target-3', 'target_source': None} + ] + self.assertEqual(serializer.data, expected) diff --git a/rest_framework/tests/serializer_nested.py b/rest_framework/tests/serializer_nested.py index fcf644c75..299c3bc5a 100644 --- a/rest_framework/tests/serializer_nested.py +++ b/rest_framework/tests/serializer_nested.py @@ -124,7 +124,7 @@ class WritableNestedSerializerObjectTests(TestCase): def __init__(self, order, title, duration): self.order, self.title, self.duration = order, title, duration - def __cmp__(self, other): + def __eq__(self, other): return ( self.order == other.order and self.title == other.title and @@ -135,7 +135,7 @@ class WritableNestedSerializerObjectTests(TestCase): def __init__(self, album_name, artist, tracks): self.album_name, self.artist, self.tracks = album_name, artist, tracks - def __cmp__(self, other): + def __eq__(self, other): return ( self.album_name == other.album_name and self.artist == other.artist and From 47492e3ef4e24ecd155091247e479851789ee8e9 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 15 Mar 2013 19:22:31 +0000 Subject: [PATCH 05/50] Clean out ModelSerializer special casing from Serializer.field_from_native --- rest_framework/serializers.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index f073e00aa..5dadebb28 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -379,11 +379,7 @@ class BaseSerializer(WritableField): serializer = self.__class__(**kwargs) if serializer.is_valid(): - if isinstance(serializer, ModelSerializer): - into[self.source or field_name] = serializer - else: - into[self.source or field_name] = serializer.object - # into[self.source or field_name] = serializer.object + into[self.source or field_name] = serializer.object else: # Propagate errors up to our parent raise NestedValidationError(serializer.errors) @@ -681,12 +677,6 @@ class ModelSerializer(Serializer): if instance: return self.full_clean(instance) -# def save_object(self, obj, **kwargs): -# """ -# Save the deserialized object and return it. -# """ -# obj.save(**kwargs) -# ======= def save_object(self, obj, parent=None, fk_field=None, **kwargs): """ Save the deserialized object and return it. @@ -706,13 +696,10 @@ class ModelSerializer(Serializer): if related is None: previous = getattr(self.object, accessor_name, related) previous.delete() - elif isinstance(related, ModelSerializer): - # print related.object - # print related.related_data, related.m2m_data + elif isinstance(related, models.Model): fk_field = obj._meta.get_field_by_name(accessor_name)[0].field.name - related.save_object(related.object, parent=self.object, fk_field=fk_field) - # setattr(related, fk_field, obj) - # related.save(**kwargs) + setattr(related, fk_field, obj) + self.save_object(related) else: setattr(self.object, accessor_name, related) obj._related_data = {} From 32e0e5e18c84e7b720c74df8aeba26e0f335bbf6 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 15 Mar 2013 19:55:32 +0000 Subject: [PATCH 06/50] Remove erronous _delete attribute --- rest_framework/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 5dadebb28..691d2aab9 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -147,7 +147,6 @@ class BaseSerializer(WritableField): self._data = None self._files = None self._errors = None - self._delete = False ##### # Methods to determine which fields to use when (de)serializing objects. From 56653111a6848f6ef5d4bb645b87cbcaf5bffba1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 15 Mar 2013 19:57:57 +0000 Subject: [PATCH 07/50] Remove unneeded arguments to save_object --- rest_framework/serializers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 691d2aab9..ebc2eec95 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -676,13 +676,10 @@ class ModelSerializer(Serializer): if instance: return self.full_clean(instance) - def save_object(self, obj, parent=None, fk_field=None, **kwargs): + def save_object(self, obj, **kwargs): """ Save the deserialized object and return it. """ - if parent and fk_field: - setattr(self.object, fk_field, parent) - obj.save(**kwargs) if getattr(obj, '_m2m_data', None): From ccf551201feb96451ffdc5d824bb0681596bcdae Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 16 Mar 2013 07:32:50 +0000 Subject: [PATCH 08/50] Clean up and comment `restore_object` --- rest_framework/serializers.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index ebc2eec95..fb7722626 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -638,31 +638,38 @@ class ModelSerializer(Serializer): """ m2m_data = {} related_data = {} + meta = self.opts.model._meta - # Reverse fk relations - for (obj, model) in self.opts.model._meta.get_all_related_objects_with_model(): + # Reverse fk or one-to-one relations + for (obj, model) in meta.get_all_related_objects_with_model(): field_name = obj.field.related_query_name() if field_name in attrs: related_data[field_name] = attrs.pop(field_name) # Reverse m2m relations - for (obj, model) in self.opts.model._meta.get_all_related_m2m_objects_with_model(): + for (obj, model) in meta.get_all_related_m2m_objects_with_model(): field_name = obj.field.related_query_name() if field_name in attrs: m2m_data[field_name] = attrs.pop(field_name) # Forward m2m relations - for field in self.opts.model._meta.many_to_many: + for field in meta.many_to_many: if field.name in attrs: m2m_data[field.name] = attrs.pop(field.name) + # Update an existing instance... if instance is not None: for key, val in attrs.items(): setattr(instance, key, val) + # ...or create a new instance else: instance = self.opts.model(**attrs) + # Any relations that cannot be set until we've + # saved the model get hidden away on these + # private attributes, so we can deal with them + # at the point of save. instance._related_data = related_data instance._m2m_data = m2m_data From 3ff103ad043420b430cc2052241994d597b1fe8a Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 16 Mar 2013 07:35:27 +0000 Subject: [PATCH 09/50] Fixes to save_object --- rest_framework/serializers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index fb7722626..5826a1730 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -691,21 +691,23 @@ class ModelSerializer(Serializer): if getattr(obj, '_m2m_data', None): for accessor_name, object_list in obj._m2m_data.items(): - setattr(self.object, accessor_name, object_list) - obj._m2m_data = {} + setattr(obj, accessor_name, object_list) + del(obj._m2m_data) if getattr(obj, '_related_data', None): for accessor_name, related in obj._related_data.items(): if related is None: - previous = getattr(self.object, accessor_name, related) - previous.delete() + previous = getattr(obj, accessor_name, related) + if previous: + previous.delete() elif isinstance(related, models.Model): fk_field = obj._meta.get_field_by_name(accessor_name)[0].field.name setattr(related, fk_field, obj) self.save_object(related) else: setattr(self.object, accessor_name, related) - obj._related_data = {} + setattr(obj, accessor_name, related) + del(obj._related_data) class HyperlinkedModelSerializerOptions(ModelSerializerOptions): From 66bdd608e1e4bbb02a815104572b80034d73aa6b Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 16 Mar 2013 07:35:44 +0000 Subject: [PATCH 10/50] Fixes to save_object --- rest_framework/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 5826a1730..21336dc29 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -705,7 +705,6 @@ class ModelSerializer(Serializer): setattr(related, fk_field, obj) self.save_object(related) else: - setattr(self.object, accessor_name, related) setattr(obj, accessor_name, related) del(obj._related_data) From c8416df0c4b7179eeaf86b61c32907f32494b85c Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Mon, 18 Mar 2013 14:27:15 +1300 Subject: [PATCH 11/50] accept all WritableField kwargs for writable serializers (eg required=True) --- rest_framework/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 21336dc29..c6599886c 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -129,8 +129,8 @@ class BaseSerializer(WritableField): _dict_class = SortedDictWithMetadata def __init__(self, instance=None, data=None, files=None, - context=None, partial=False, many=None, source=None): - super(BaseSerializer, self).__init__(source=source) + context=None, partial=False, many=None, **kwargs): + super(BaseSerializer, self).__init__(**kwargs) self.opts = self._options_class(self.Meta) self.parent = None self.root = None From d6d5b1d82a4ccc1a2fe29ff18e9ecf7c196a07a5 Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Mon, 18 Mar 2013 14:50:08 +1300 Subject: [PATCH 12/50] allow default values in writable serializer fields --- rest_framework/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index c6599886c..cc6d60da1 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -360,7 +360,9 @@ class BaseSerializer(WritableField): except KeyError: if self.required: raise ValidationError(self.error_messages['required']) - return + if self.default is None: + return + value = copy.deepcopy(self.default) # Set the serializer object if it exists obj = getattr(self.parent.object, field_name) if self.parent.object else None From 101fa26ebc092a43acbf3f28617eb58be7629b5f Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Mon, 18 Mar 2013 16:05:34 +1300 Subject: [PATCH 13/50] use writablefield style for serializer handling of self.default --- rest_framework/serializers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index cc6d60da1..a81cbc291 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -358,11 +358,13 @@ class BaseSerializer(WritableField): try: value = data[field_name] except KeyError: - if self.required: - raise ValidationError(self.error_messages['required']) - if self.default is None: + if self.default is not None and not self.partial: + # Note: partial updates shouldn't set defaults + value = copy.deepcopy(self.default) + else: + if self.required: + raise ValidationError(self.error_messages['required']) return - value = copy.deepcopy(self.default) # Set the serializer object if it exists obj = getattr(self.parent.object, field_name) if self.parent.object else None From 09e4ee7ae332326e77b23bac1539d31e582419e9 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 18 Mar 2013 21:11:40 +0000 Subject: [PATCH 14/50] Remove dumbass unneeded test --- rest_framework/tests/status.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 rest_framework/tests/status.py diff --git a/rest_framework/tests/status.py b/rest_framework/tests/status.py deleted file mode 100644 index e1644a6b4..000000000 --- a/rest_framework/tests/status.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Tests for the status module""" -from __future__ import unicode_literals -from django.test import TestCase -from rest_framework import status - - -class TestStatus(TestCase): - """Simple sanity test to check the status module""" - - def test_status(self): - """Ensure the status module is present and correct.""" - self.assertEqual(200, status.HTTP_200_OK) - self.assertEqual(404, status.HTTP_404_NOT_FOUND) From deb5e653e441bf31f3b183b575f72e6b4cf537ea Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 18 Mar 2013 21:35:06 +0000 Subject: [PATCH 15/50] Added bulk create tests --- docs/topics/contributing.md | 137 +++++++++++++++++- .../tests/serializer_bulk_update.py | 73 ++++++++++ 2 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 rest_framework/tests/serializer_bulk_update.py diff --git a/docs/topics/contributing.md b/docs/topics/contributing.md index 7fd61c10e..bc9c2e931 100644 --- a/docs/topics/contributing.md +++ b/docs/topics/contributing.md @@ -4,12 +4,139 @@ > > — [Tim Berners-Lee][cite] -## Running the tests +There are many ways you can contribute to Django REST framework. We'd like it to be a community-led project, so please get involved and help shape the future of the project. -## Building the docs +# Community -## Managing compatibility issues +If you use and enjoy REST framework please consider [staring the project on GitHub][github], and [upvoting it on Django packages][django-packages]. Doing so helps potential new users see that the project is well used, and help us continue to attract new users. -**Describe compat module** +You might also consider writing a blog post on your experience with using REST framework, writing a tutorial about using the project with a particular javascript framework, or simply sharing the love on Twitter. + +Other really great ways you can help move the community forward include helping answer questions on the [discussion group][google-group], or setting up an [email alert on StackOverflow][so-filter] so that you get notified of any new questions with the `django-rest-framework` tag. + +When answering questions make sure to help future contributors find their way around by hyperlinking wherever possible to related threads and tickets, and include backlinks from those items if relevant. + +# Issues + +Usage questions should be directed to the [discussion group][google-group]. Feature requests, bug reports and other issues should be raised on the GitHub [issue tracker][issues]. + +Some tips on good issue reporting: + +* When decribing issues try to phrase your ticket in terms of the *behavior* you think needs changing rather than the *code* you think need changing. +* Search the issue list first for related items, and make sure you're running the latest version of REST framework before reporting an issue. +* If reporting a bug, then try to include a pull request with a failing test case. This'll help us quickly identify if there is a valid issue, and make sure that it gets fixed more quickly if there is one. + + + +* TODO: Triage + +# Development + +* git clone & PYTHONPATH +* Pep8 +* Recommend editor that runs pep8 + +### Pull requests + +* Make pull requests early +* Describe branching + +### Managing compatibility issues + +* Describe compat module + +# Testing + +* Running the tests +* tox + +# Documentation + +The documentation for REST framework is built from the [Markdown][markdown] source files in [the docs directory][docs]. + +There are many great markdown editors that make working with the documentation really easy. The [Mou editor for Mac][mou] is one such editor that comes highly recommended. + +## Building the documentation + +To build the documentation, simply run the `mkdocs.py` script. + + ./mkdocs.py + +This will build the html output into the `html` directory. + +You can build the documentation and open a preview in a browser window by using the `-p` flag. + + ./mkdocs.py -p + +## Language style + +Documentation should be in American English. The tone of the documentation is very important - try to stick to a simple, plain, objective and well-balanced style where possible. + +Some other tips: + +* Keep paragraphs reasonably short. +* Use double spacing after the end of sentences. +* Don't use the abbreviations such as 'e.g..' but instead use long form, such as 'For example'. + +## Markdown style + +There are a couple of conventions you should follow when working on the documentation. + +##### 1. Headers + +Headers should use the hash style. For example: + + ### Some important topic + +The underline style should not be used. **Don't do this:** + + Some important topic + ==================== + +##### 2. Links + +Links should always use the reference style, with the referenced hyperlinks kept at the end of the document. + + Here is a link to [some other thing][other-thing]. + + More text... + + [other-thing]: http://example.com/other/thing + +This style helps keep the documentation source consistent and readable. + +If you are hyperlinking to another REST framework document, you should use a relative link, and link to the `.md` suffix. For example: + + [authentication]: ../api-guide/authentication.md + +Linking in this style means you'll be able to click the hyperlink in your markdown editor to open the referenced document. When the documentation is built, these links will be converted into regular links to HTML pages. + +##### 3. Notes + +If you want to draw attention to a note or warning, use a pair of enclosing lines, like so: + + --- + + **Note:** Make sure you do this thing. + + --- + +# Third party packages + +* Django reusable app + +# Core committers + +* Still use pull reqs +* Credits + +[cite]: http://www.w3.org/People/Berners-Lee/FAQ.html +[github]: https://github.com/tomchristie/django-rest-framework +[django-packages]: https://www.djangopackages.com/grids/g/api/ +[google-group]: https://groups.google.com/forum/?fromgroups#!forum/django-rest-framework +[so-filter]: http://stackexchange.com/filters/66475/rest-framework +[issues]: https://github.com/tomchristie/django-rest-framework/issues?state=open +[markdown]: http://daringfireball.net/projects/markdown/basics +[docs]: https://github.com/tomchristie/django-rest-framework/tree/master/docs +[mou]: http://mouapp.com/ -[cite]: http://www.w3.org/People/Berners-Lee/FAQ.html \ No newline at end of file diff --git a/rest_framework/tests/serializer_bulk_update.py b/rest_framework/tests/serializer_bulk_update.py new file mode 100644 index 000000000..3ecb23edd --- /dev/null +++ b/rest_framework/tests/serializer_bulk_update.py @@ -0,0 +1,73 @@ +""" +Tests to cover bulk create and update using serializers. +""" +from __future__ import unicode_literals +from django.test import TestCase +from rest_framework import serializers + + +class BulkCreateSerializerTests(TestCase): + + def setUp(self): + class BookSerializer(serializers.Serializer): + id = serializers.IntegerField() + title = serializers.CharField(max_length=100) + author = serializers.CharField(max_length=100) + + self.BookSerializer = BookSerializer + + def test_bulk_create_success(self): + """ + Correct bulk update serialization should return the input data. + """ + + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 1, + 'title': 'If this is a man', + 'author': 'Primo Levi' + }, { + 'id': 2, + 'title': 'The wind-up bird chronicle', + 'author': 'Haruki Murakami' + } + ] + + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.object, data) + + def test_bulk_create_errors(self): + """ + Correct bulk update serialization should return the input data. + """ + + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 1, + 'title': 'If this is a man', + 'author': 'Primo Levi' + }, { + 'id': 'foo', + 'title': 'The wind-up bird chronicle', + 'author': 'Haruki Murakami' + } + ] + expected_errors = [ + {}, + {}, + {'id': ['Enter a whole number.']} + ] + + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + self.assertEqual(serializer.errors, expected_errors) + From b2dc66448503c2120d943a2f282eab235afc67ba Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 19 Mar 2013 14:26:48 +0000 Subject: [PATCH 16/50] Basic bulk create and bulk update --- docs/api-guide/serializers.md | 17 +- rest_framework/serializers.py | 38 ++++ rest_framework/tests/serializer.py | 2 +- .../tests/serializer_bulk_update.py | 174 ++++++++++++++++++ 4 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 rest_framework/tests/serializer_bulk_update.py diff --git a/docs/api-guide/serializers.md b/docs/api-guide/serializers.md index 42edf9af1..ee7208a36 100644 --- a/docs/api-guide/serializers.md +++ b/docs/api-guide/serializers.md @@ -37,9 +37,6 @@ Declaring a serializer looks very similar to declaring a form: """ Given a dictionary of deserialized field values, either update an existing model instance, or create a new model instance. - - Note that if we don't define this method, then deserializing - data will simply return a dictionary of items. """ if instance is not None: instance.title = attrs.get('title', instance.title) @@ -48,7 +45,9 @@ Declaring a serializer looks very similar to declaring a form: return instance return Comment(**attrs) -The first part of serializer class defines the fields that get serialized/deserialized. The `restore_object` method defines how fully fledged instances get created when deserializing data. The `restore_object` method is optional, and is only required if we want our serializer to support deserialization. +The first part of serializer class defines the fields that get serialized/deserialized. The `restore_object` method defines how fully fledged instances get created when deserializing data. + +The `restore_object` method is optional, and is only required if we want our serializer to support deserialization into fully fledged object instances. If we don't define this method, then deserializing data will simply return a dictionary of items. ## Serializing objects @@ -88,18 +87,22 @@ By default, serializers must be passed values for all required fields or they wi serializer = CommentSerializer(comment, data={'content': u'foo bar'}, partial=True) # Update `instance` with partial data -## Serializing querysets +## Serializing multiple objects -To serialize a queryset instead of an object instance, you should pass the `many=True` flag when instantiating the serializer. +To serialize a queryset or list of objects instead of a single object instance, you should pass the `many=True` flag when instantiating the serializer. queryset = Comment.objects.all() serializer = CommentSerializer(queryset, many=True) serializer.data - # [{'email': u'leila@example.com', 'content': u'foo bar', 'created': datetime.datetime(2012, 8, 22, 16, 20, 9, 822774)}, {'email': u'jamie@example.com', 'content': u'baz', 'created': datetime.datetime(2013, 1, 12, 16, 12, 45, 104445)}] + # [ + # {'email': u'leila@example.com', 'content': u'foo bar', 'created': datetime.datetime(2012, 8, 22, 16, 20, 9, 822774)}, + # {'email': u'jamie@example.com', 'content': u'baz', 'created': datetime.datetime(2013, 1, 12, 16, 12, 45, 104445)} + # ] ## Validation When deserializing data, you always need to call `is_valid()` before attempting to access the deserialized object. If any validation errors occur, the `.errors` property will contain a dictionary representing the resulting error messages. + Each key in the dictionary will be the field name, and the values will be lists of strings of any error messages corresponding to that field. The `non_field_errors` key may also be present, and will list any general validation errors. When deserializing a list of items, errors will be returned as a list of dictionaries representing each of the deserialized items. diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 4fe857a61..34120dc68 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -128,6 +128,7 @@ class BaseSerializer(Field): self._data = None self._files = None self._errors = None + self._deleted = None ##### # Methods to determine which fields to use when (de)serializing objects. @@ -331,6 +332,13 @@ class BaseSerializer(Field): return [self.to_native(item) for item in obj] return self.to_native(obj) + def get_identity(self, data): + """ + This hook is required for bulk update. + It is used to determine the canonical identity of a given object. + """ + return data.get('id') + @property def errors(self): """ @@ -352,9 +360,32 @@ class BaseSerializer(Field): if many: ret = [] errors = [] + update = self.object is not None + + if update: + # If this is a bulk update we need to map all the objects + # to a canonical identity so we can determine which + # individual object is being updated for each item in the + # incoming data + objects = self.object + identities = [self.get_identity(self.to_native(obj)) for obj in objects] + identity_to_objects = dict(zip(identities, objects)) + for item in data: + if update: + # Determine which object we're updating + try: + identity = self.get_identity(item) + except: + self.object = None + else: + self.object = identity_to_objects.pop(identity, None) + ret.append(self.from_native(item, None)) errors.append(self._errors) + + if update: + self._deleted = identity_to_objects.values() self._errors = any(errors) and errors or [] else: ret = self.from_native(data, files) @@ -394,6 +425,9 @@ class BaseSerializer(Field): def save_object(self, obj, **kwargs): obj.save(**kwargs) + def delete_object(self, obj): + obj.delete() + def save(self, **kwargs): """ Save the deserialized object and return it. @@ -402,6 +436,10 @@ class BaseSerializer(Field): [self.save_object(item, **kwargs) for item in self.object] else: self.save_object(self.object, **kwargs) + + if self._deleted: + [self.delete_object(item) for item in self._deleted] + return self.object diff --git a/rest_framework/tests/serializer.py b/rest_framework/tests/serializer.py index beb372c2b..9c0fdd787 100644 --- a/rest_framework/tests/serializer.py +++ b/rest_framework/tests/serializer.py @@ -266,7 +266,7 @@ class ValidationTests(TestCase): Data of the wrong type is not valid. """ data = ['i am', 'a', 'list'] - serializer = CommentSerializer(self.comment, data=data, many=True) + serializer = CommentSerializer([self.comment], data=data, many=True) self.assertEqual(serializer.is_valid(), False) self.assertTrue(isinstance(serializer.errors, list)) diff --git a/rest_framework/tests/serializer_bulk_update.py b/rest_framework/tests/serializer_bulk_update.py new file mode 100644 index 000000000..66fca8835 --- /dev/null +++ b/rest_framework/tests/serializer_bulk_update.py @@ -0,0 +1,174 @@ +""" +Tests to cover bulk create and update using serializers. +""" +from __future__ import unicode_literals +from django.test import TestCase +from rest_framework import serializers + + +class BulkCreateSerializerTests(TestCase): + + def setUp(self): + class BookSerializer(serializers.Serializer): + id = serializers.IntegerField() + title = serializers.CharField(max_length=100) + author = serializers.CharField(max_length=100) + + self.BookSerializer = BookSerializer + + def test_bulk_create_success(self): + """ + Correct bulk update serialization should return the input data. + """ + + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 1, + 'title': 'If this is a man', + 'author': 'Primo Levi' + }, { + 'id': 2, + 'title': 'The wind-up bird chronicle', + 'author': 'Haruki Murakami' + } + ] + + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.object, data) + + def test_bulk_create_errors(self): + """ + Correct bulk update serialization should return the input data. + """ + + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 1, + 'title': 'If this is a man', + 'author': 'Primo Levi' + }, { + 'id': 'foo', + 'title': 'The wind-up bird chronicle', + 'author': 'Haruki Murakami' + } + ] + expected_errors = [ + {}, + {}, + {'id': ['Enter a whole number.']} + ] + + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + self.assertEqual(serializer.errors, expected_errors) + + +class BulkUpdateSerializerTests(TestCase): + + def setUp(self): + class Book(object): + object_map = {} + + def __init__(self, id, title, author): + self.id = id + self.title = title + self.author = author + + def save(self): + Book.object_map[self.id] = self + + def delete(self): + del Book.object_map[self.id] + + class BookSerializer(serializers.Serializer): + id = serializers.IntegerField() + title = serializers.CharField(max_length=100) + author = serializers.CharField(max_length=100) + + def restore_object(self, attrs, instance=None): + if instance: + instance.id = attrs['id'] + instance.title = attrs['title'] + instance.author = attrs['author'] + return instance + return Book(**attrs) + + self.Book = Book + self.BookSerializer = BookSerializer + + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 1, + 'title': 'If this is a man', + 'author': 'Primo Levi' + }, { + 'id': 2, + 'title': 'The wind-up bird chronicle', + 'author': 'Haruki Murakami' + } + ] + + for item in data: + book = Book(item['id'], item['title'], item['author']) + book.save() + + def books(self): + return self.Book.object_map.values() + + def test_bulk_update_success(self): + """ + Correct bulk update serialization should return the input data. + """ + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 2, + 'title': 'Kafka on the shore', + 'author': 'Haruki Murakami' + } + ] + serializer = self.BookSerializer(self.books(), data=data, many=True) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.data, data) + serializer.save() + new_data = self.BookSerializer(self.books(), many=True).data + self.assertEqual(data, new_data) + + def test_bulk_update_error(self): + """ + Correct bulk update serialization should return the input data. + """ + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 'foo', + 'title': 'Kafka on the shore', + 'author': 'Haruki Murakami' + } + ] + expected_errors = [ + {}, + {'id': ['Enter a whole number.']} + ] + serializer = self.BookSerializer(self.books(), data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + self.assertEqual(serializer.errors, expected_errors) From 20fd738c856dfa21a0d2d251b92459d6641c782c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 20 Mar 2013 13:05:59 +0000 Subject: [PATCH 17/50] iso formated datetime aware fields with +0000 offset should use 'Z' suffix instead --- rest_framework/fields.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rest_framework/fields.py b/rest_framework/fields.py index 4b6931ad4..a0f52f506 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -609,7 +609,10 @@ class DateTimeField(WritableField): return None if self.format.lower() == ISO_8601: - return value.isoformat() + ret = value.isoformat() + if ret.endswith('+00:00'): + ret = ret[:-6] + 'Z' + return ret return value.strftime(self.format) From 8adde506e865005a96cdeff996ec4b5b9bb73a8f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 Mar 2013 08:41:54 +0000 Subject: [PATCH 18/50] Default date/time fields now return python date/time objects again by default --- docs/api-guide/fields.md | 6 ++--- docs/api-guide/settings.md | 27 ++++++++++++++----- rest_framework/fields.py | 18 ++++++------- rest_framework/tests/fields.py | 43 ++++++++++++++++++++++++++++-- rest_framework/tests/filterset.py | 9 +++---- rest_framework/tests/pagination.py | 2 +- rest_framework/tests/serializer.py | 2 +- 7 files changed, 79 insertions(+), 28 deletions(-) diff --git a/docs/api-guide/fields.md b/docs/api-guide/fields.md index 9a745cf19..63bd5b1aa 100644 --- a/docs/api-guide/fields.md +++ b/docs/api-guide/fields.md @@ -199,7 +199,7 @@ If you want to override this behavior, you'll need to declare the `DateTimeField **Signature:** `DateTimeField(format=None, input_formats=None)` -* `format` - A string representing the output format. If not specified, the `DATETIME_FORMAT` setting will be used, which defaults to `'iso-8601'`. +* `format` - A string representing the output format. If not specified, this defaults to `None`, which indicates that python `datetime` objects should be returned by `to_native`. In this case the datetime encoding will be determined by the renderer. * `input_formats` - A list of strings representing the input formats which may be used to parse the date. If not specified, the `DATETIME_INPUT_FORMATS` setting will be used, which defaults to `['iso-8601']`. DateTime format strings may either be [python strftime formats][strftime] which explicitly specifiy the format, or the special string `'iso-8601'`, which indicates that [ISO 8601][iso8601] style datetimes should be used. (eg `'2013-01-29T12:34:56.000000'`) @@ -212,7 +212,7 @@ Corresponds to `django.db.models.fields.DateField` **Signature:** `DateField(format=None, input_formats=None)` -* `format` - A string representing the output format. If not specified, the `DATE_FORMAT` setting will be used, which defaults to `'iso-8601'`. +* `format` - A string representing the output format. If not specified, this defaults to `None`, which indicates that python `date` objects should be returned by `to_native`. In this case the date encoding will be determined by the renderer. * `input_formats` - A list of strings representing the input formats which may be used to parse the date. If not specified, the `DATE_INPUT_FORMATS` setting will be used, which defaults to `['iso-8601']`. Date format strings may either be [python strftime formats][strftime] which explicitly specifiy the format, or the special string `'iso-8601'`, which indicates that [ISO 8601][iso8601] style dates should be used. (eg `'2013-01-29'`) @@ -227,7 +227,7 @@ Corresponds to `django.db.models.fields.TimeField` **Signature:** `TimeField(format=None, input_formats=None)` -* `format` - A string representing the output format. If not specified, the `TIME_FORMAT` setting will be used, which defaults to `'iso-8601'`. +* `format` - A string representing the output format. If not specified, this defaults to `None`, which indicates that python `time` objects should be returned by `to_native`. In this case the time encoding will be determined by the renderer. * `input_formats` - A list of strings representing the input formats which may be used to parse the date. If not specified, the `TIME_INPUT_FORMATS` setting will be used, which defaults to `['iso-8601']`. Time format strings may either be [python strftime formats][strftime] which explicitly specifiy the format, or the special string `'iso-8601'`, which indicates that [ISO 8601][iso8601] style times should be used. (eg `'12:34:56.000000'`) diff --git a/docs/api-guide/settings.md b/docs/api-guide/settings.md index 116386969..c0d8d9eea 100644 --- a/docs/api-guide/settings.md +++ b/docs/api-guide/settings.md @@ -192,44 +192,56 @@ Default: `'format'` --- -## Date/Time formatting +## Date and time formatting *The following settings are used to control how date and time representations may be parsed and rendered.* #### DATETIME_FORMAT -A format string that should be used by default for rendering the output of `DateTimeField` serializer fields. +A format string that should be used by default for rendering the output of `DateTimeField` serializer fields. If `None`, then `DateTimeField` serializer fields will return python `datetime` objects, and the datetime encoding will be determined by the renderer. -Default: `'iso-8601'` +May be any of `None`, `'iso-8601'` or a python [strftime format][strftime] string. + +Default: `None'` #### DATETIME_INPUT_FORMATS A list of format strings that should be used by default for parsing inputs to `DateTimeField` serializer fields. +May be a list including the string `'iso-8601'` or python [strftime format][strftime] strings. + Default: `['iso-8601']` #### DATE_FORMAT -A format string that should be used by default for rendering the output of `DateField` serializer fields. +A format string that should be used by default for rendering the output of `DateField` serializer fields. If `None`, then `DateField` serializer fields will return python `date` objects, and the date encoding will be determined by the renderer. -Default: `'iso-8601'` +May be any of `None`, `'iso-8601'` or a python [strftime format][strftime] string. + +Default: `None` #### DATE_INPUT_FORMATS A list of format strings that should be used by default for parsing inputs to `DateField` serializer fields. +May be a list including the string `'iso-8601'` or python [strftime format][strftime] strings. + Default: `['iso-8601']` #### TIME_FORMAT -A format string that should be used by default for rendering the output of `TimeField` serializer fields. +A format string that should be used by default for rendering the output of `TimeField` serializer fields. If `None`, then `TimeField` serializer fields will return python `time` objects, and the time encoding will be determined by the renderer. -Default: `'iso-8601'` +May be any of `None`, `'iso-8601'` or a python [strftime format][strftime] string. + +Default: `None` #### TIME_INPUT_FORMATS A list of format strings that should be used by default for parsing inputs to `TimeField` serializer fields. +May be a list including the string `'iso-8601'` or python [strftime format][strftime] strings. + Default: `['iso-8601']` --- @@ -243,3 +255,4 @@ The name of a parameter in the URL conf that may be used to provide a format suf Default: `'format'` [cite]: http://www.python.org/dev/peps/pep-0020/ +[strftime]: http://docs.python.org/2/library/time.html#time.strftime \ No newline at end of file diff --git a/rest_framework/fields.py b/rest_framework/fields.py index a0f52f506..f3496b53e 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -494,7 +494,7 @@ class DateField(WritableField): } empty = None input_formats = api_settings.DATE_INPUT_FORMATS - format = api_settings.DATE_FORMAT + format = None def __init__(self, input_formats=None, format=None, *args, **kwargs): self.input_formats = input_formats if input_formats is not None else self.input_formats @@ -536,8 +536,8 @@ class DateField(WritableField): raise ValidationError(msg) def to_native(self, value): - if value is None: - return None + if value is None or self.format is None: + return value if isinstance(value, datetime.datetime): value = value.date() @@ -557,7 +557,7 @@ class DateTimeField(WritableField): } empty = None input_formats = api_settings.DATETIME_INPUT_FORMATS - format = api_settings.DATETIME_FORMAT + format = None def __init__(self, input_formats=None, format=None, *args, **kwargs): self.input_formats = input_formats if input_formats is not None else self.input_formats @@ -605,8 +605,8 @@ class DateTimeField(WritableField): raise ValidationError(msg) def to_native(self, value): - if value is None: - return None + if value is None or self.format is None: + return value if self.format.lower() == ISO_8601: ret = value.isoformat() @@ -626,7 +626,7 @@ class TimeField(WritableField): } empty = None input_formats = api_settings.TIME_INPUT_FORMATS - format = api_settings.TIME_FORMAT + format = None def __init__(self, input_formats=None, format=None, *args, **kwargs): self.input_formats = input_formats if input_formats is not None else self.input_formats @@ -661,8 +661,8 @@ class TimeField(WritableField): raise ValidationError(msg) def to_native(self, value): - if value is None: - return None + if value is None or self.format is None: + return value if isinstance(value, datetime.datetime): value = value.time() diff --git a/rest_framework/tests/fields.py b/rest_framework/tests/fields.py index fd6de7797..19c663d82 100644 --- a/rest_framework/tests/fields.py +++ b/rest_framework/tests/fields.py @@ -153,12 +153,22 @@ class DateFieldTest(TestCase): def test_to_native(self): """ - Make sure to_native() returns isoformat as default. + Make sure to_native() returns datetime as default. """ f = serializers.DateField() result_1 = f.to_native(datetime.date(1984, 7, 31)) + self.assertEqual(datetime.date(1984, 7, 31), result_1) + + def test_to_native_iso(self): + """ + Make sure to_native() with 'iso-8601' returns iso formated date. + """ + f = serializers.DateField(format='iso-8601') + + result_1 = f.to_native(datetime.date(1984, 7, 31)) + self.assertEqual('1984-07-31', result_1) def test_to_native_custom_format(self): @@ -289,6 +299,22 @@ class DateTimeFieldTest(TestCase): result_3 = f.to_native(datetime.datetime(1984, 7, 31, 4, 31, 59)) result_4 = f.to_native(datetime.datetime(1984, 7, 31, 4, 31, 59, 200)) + self.assertEqual(datetime.datetime(1984, 7, 31), result_1) + self.assertEqual(datetime.datetime(1984, 7, 31, 4, 31), result_2) + self.assertEqual(datetime.datetime(1984, 7, 31, 4, 31, 59), result_3) + self.assertEqual(datetime.datetime(1984, 7, 31, 4, 31, 59, 200), result_4) + + def test_to_native_iso(self): + """ + Make sure to_native() with format=iso-8601 returns iso formatted datetime. + """ + f = serializers.DateTimeField(format='iso-8601') + + result_1 = f.to_native(datetime.datetime(1984, 7, 31)) + result_2 = f.to_native(datetime.datetime(1984, 7, 31, 4, 31)) + result_3 = f.to_native(datetime.datetime(1984, 7, 31, 4, 31, 59)) + result_4 = f.to_native(datetime.datetime(1984, 7, 31, 4, 31, 59, 200)) + self.assertEqual('1984-07-31T00:00:00', result_1) self.assertEqual('1984-07-31T04:31:00', result_2) self.assertEqual('1984-07-31T04:31:59', result_3) @@ -419,13 +445,26 @@ class TimeFieldTest(TestCase): def test_to_native(self): """ - Make sure to_native() returns isoformat as default. + Make sure to_native() returns time object as default. """ f = serializers.TimeField() result_1 = f.to_native(datetime.time(4, 31)) result_2 = f.to_native(datetime.time(4, 31, 59)) result_3 = f.to_native(datetime.time(4, 31, 59, 200)) + self.assertEqual(datetime.time(4, 31), result_1) + self.assertEqual(datetime.time(4, 31, 59), result_2) + self.assertEqual(datetime.time(4, 31, 59, 200), result_3) + + def test_to_native_iso(self): + """ + Make sure to_native() with format='iso-8601' returns iso formatted time. + """ + f = serializers.TimeField(format='iso-8601') + result_1 = f.to_native(datetime.time(4, 31)) + result_2 = f.to_native(datetime.time(4, 31, 59)) + result_3 = f.to_native(datetime.time(4, 31, 59, 200)) + self.assertEqual('04:31:00', result_1) self.assertEqual('04:31:59', result_2) self.assertEqual('04:31:59.000200', result_3) diff --git a/rest_framework/tests/filterset.py b/rest_framework/tests/filterset.py index fe92e0bcf..238da56e5 100644 --- a/rest_framework/tests/filterset.py +++ b/rest_framework/tests/filterset.py @@ -65,7 +65,7 @@ class IntegrationTestFiltering(TestCase): self.objects = FilterableItem.objects self.data = [ - {'id': obj.id, 'text': obj.text, 'decimal': obj.decimal, 'date': obj.date.isoformat()} + {'id': obj.id, 'text': obj.text, 'decimal': obj.decimal, 'date': obj.date} for obj in self.objects.all() ] @@ -95,7 +95,7 @@ class IntegrationTestFiltering(TestCase): request = factory.get('/?date=%s' % search_date) # search_date str: '2012-09-22' response = view(request).render() self.assertEqual(response.status_code, status.HTTP_200_OK) - expected_data = [f for f in self.data if datetime.datetime.strptime(f['date'], '%Y-%m-%d').date() == search_date] + expected_data = [f for f in self.data if f['date'] == search_date] self.assertEqual(response.data, expected_data) @unittest.skipUnless(django_filters, 'django-filters not installed') @@ -125,7 +125,7 @@ class IntegrationTestFiltering(TestCase): request = factory.get('/?date=%s' % search_date) # search_date str: '2012-10-02' response = view(request).render() self.assertEqual(response.status_code, status.HTTP_200_OK) - expected_data = [f for f in self.data if datetime.datetime.strptime(f['date'], '%Y-%m-%d').date() > search_date] + expected_data = [f for f in self.data if f['date'] > search_date] self.assertEqual(response.data, expected_data) # Tests that the text filter set with 'icontains' in the filter class works. @@ -142,8 +142,7 @@ class IntegrationTestFiltering(TestCase): request = factory.get('/?decimal=%s&date=%s' % (search_decimal, search_date)) response = view(request).render() self.assertEqual(response.status_code, status.HTTP_200_OK) - expected_data = [f for f in self.data if - datetime.datetime.strptime(f['date'], '%Y-%m-%d').date() > search_date and + expected_data = [f for f in self.data if f['date'] > search_date and f['decimal'] < search_decimal] self.assertEqual(response.data, expected_data) diff --git a/rest_framework/tests/pagination.py b/rest_framework/tests/pagination.py index 1a2d68a68..d2c9b0513 100644 --- a/rest_framework/tests/pagination.py +++ b/rest_framework/tests/pagination.py @@ -102,7 +102,7 @@ class IntegrationTestPaginationAndFiltering(TestCase): self.objects = FilterableItem.objects self.data = [ - {'id': obj.id, 'text': obj.text, 'decimal': obj.decimal, 'date': obj.date.isoformat()} + {'id': obj.id, 'text': obj.text, 'decimal': obj.decimal, 'date': obj.date} for obj in self.objects.all() ] diff --git a/rest_framework/tests/serializer.py b/rest_framework/tests/serializer.py index beb372c2b..d0799b85d 100644 --- a/rest_framework/tests/serializer.py +++ b/rest_framework/tests/serializer.py @@ -112,7 +112,7 @@ class BasicTests(TestCase): self.expected = { 'email': 'tom@example.com', 'content': 'Happy new year!', - 'created': '2012-01-01T00:00:00', + 'created': datetime.datetime(2012, 1, 1), 'sub_comment': 'And Merry Christmas!' } self.person_data = {'name': 'dwight', 'age': 35} From c32d9969acaebc83ad1e2dd888d4a2829c35571e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 12:33:09 +0000 Subject: [PATCH 19/50] Add extra tests for errors from incorrect data with multiple create/update --- rest_framework/serializers.py | 36 ++++++----- .../tests/serializer_bulk_update.py | 60 ++++++++++++++++++- 2 files changed, 81 insertions(+), 15 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 34120dc68..3029cf1c1 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -371,22 +371,30 @@ class BaseSerializer(Field): identities = [self.get_identity(self.to_native(obj)) for obj in objects] identity_to_objects = dict(zip(identities, objects)) - for item in data: + try: + iter(data) + if isinstance(data, dict): + raise TypeError + except TypeError: + self._errors = {'non_field_errors': ['Expected a list of items']} + else: + for item in data: + if update: + # Determine which object we're updating + try: + identity = self.get_identity(item) + except: + self.object = None + else: + self.object = identity_to_objects.pop(identity, None) + + ret.append(self.from_native(item, None)) + errors.append(self._errors) + if update: - # Determine which object we're updating - try: - identity = self.get_identity(item) - except: - self.object = None - else: - self.object = identity_to_objects.pop(identity, None) + self._deleted = identity_to_objects.values() - ret.append(self.from_native(item, None)) - errors.append(self._errors) - - if update: - self._deleted = identity_to_objects.values() - self._errors = any(errors) and errors or [] + self._errors = any(errors) and errors or [] else: ret = self.from_native(data, files) diff --git a/rest_framework/tests/serializer_bulk_update.py b/rest_framework/tests/serializer_bulk_update.py index 66fca8835..2f4164880 100644 --- a/rest_framework/tests/serializer_bulk_update.py +++ b/rest_framework/tests/serializer_bulk_update.py @@ -7,6 +7,9 @@ from rest_framework import serializers class BulkCreateSerializerTests(TestCase): + """ + Creating multiple instances using serializers. + """ def setUp(self): class BookSerializer(serializers.Serializer): @@ -71,11 +74,63 @@ class BulkCreateSerializerTests(TestCase): self.assertEqual(serializer.is_valid(), False) self.assertEqual(serializer.errors, expected_errors) + def test_invalid_list_datatype(self): + """ + Data containing list of incorrect data type should return errors. + """ + data = ['foo', 'bar', 'baz'] + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + + expected_errors = [ + {'non_field_errors': ['Invalid data']}, + {'non_field_errors': ['Invalid data']}, + {'non_field_errors': ['Invalid data']} + ] + + self.assertEqual(serializer.errors, expected_errors) + + def test_invalid_single_datatype(self): + """ + Data containing a single incorrect data type should return errors. + """ + data = 123 + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + + expected_errors = {'non_field_errors': ['Expected a list of items']} + + self.assertEqual(serializer.errors, expected_errors) + + def test_invalid_single_object(self): + """ + Data containing only a single object, instead of a list of objects + should return errors. + """ + data = { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + } + serializer = self.BookSerializer(data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + + expected_errors = {'non_field_errors': ['Expected a list of items']} + + self.assertEqual(serializer.errors, expected_errors) + class BulkUpdateSerializerTests(TestCase): + """ + Updating multiple instances using serializers. + """ def setUp(self): class Book(object): + """ + A data type that can be persisted to a mock storage backend + with `.save()` and `.delete()`. + """ object_map = {} def __init__(self, id, title, author): @@ -126,6 +181,9 @@ class BulkUpdateSerializerTests(TestCase): book.save() def books(self): + """ + Return all the objects in the mock storage backend. + """ return self.Book.object_map.values() def test_bulk_update_success(self): @@ -152,7 +210,7 @@ class BulkUpdateSerializerTests(TestCase): def test_bulk_update_error(self): """ - Correct bulk update serialization should return the input data. + Incorrect bulk update serialization should return error data. """ data = [ { From b4210f9a56fb7f8913f674aecb304da01e2e9f64 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 12:46:03 +0000 Subject: [PATCH 20/50] Test moved out of serializer.py into serializer_bulk_update --- rest_framework/tests/serializer.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/rest_framework/tests/serializer.py b/rest_framework/tests/serializer.py index 9c0fdd787..1ee671dcb 100644 --- a/rest_framework/tests/serializer.py +++ b/rest_framework/tests/serializer.py @@ -261,34 +261,6 @@ class ValidationTests(TestCase): self.assertEqual(serializer.is_valid(), True) self.assertEqual(serializer.errors, {}) - def test_bad_type_data_is_false(self): - """ - Data of the wrong type is not valid. - """ - data = ['i am', 'a', 'list'] - serializer = CommentSerializer([self.comment], data=data, many=True) - self.assertEqual(serializer.is_valid(), False) - self.assertTrue(isinstance(serializer.errors, list)) - - self.assertEqual( - serializer.errors, - [ - {'non_field_errors': ['Invalid data']}, - {'non_field_errors': ['Invalid data']}, - {'non_field_errors': ['Invalid data']} - ] - ) - - data = 'and i am a string' - serializer = CommentSerializer(self.comment, data=data) - self.assertEqual(serializer.is_valid(), False) - self.assertEqual(serializer.errors, {'non_field_errors': ['Invalid data']}) - - data = 42 - serializer = CommentSerializer(self.comment, data=data) - self.assertEqual(serializer.is_valid(), False) - self.assertEqual(serializer.errors, {'non_field_errors': ['Invalid data']}) - def test_cross_field_validation(self): class CommentSerializerWithCrossFieldValidator(CommentSerializer): From 13794baf7016f7d44daffb55d29e3dbc56f7612d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 17:01:06 +0000 Subject: [PATCH 21/50] Bit of extra tidying and plenty of docs --- docs/api-guide/fields.md | 43 ++++ docs/api-guide/serializers.md | 189 ++++++++++++------ rest_framework/serializers.py | 49 +++-- .../tests/serializer_bulk_update.py | 26 ++- 4 files changed, 225 insertions(+), 82 deletions(-) diff --git a/docs/api-guide/fields.md b/docs/api-guide/fields.md index 9a745cf19..4d73eec76 100644 --- a/docs/api-guide/fields.md +++ b/docs/api-guide/fields.md @@ -273,6 +273,49 @@ Django's regular [FILE_UPLOAD_HANDLERS] are used for handling uploaded files. --- +# Custom fields + +If you want to create a custom field, you'll probably want to override either one or both of the `.to_native()` and `.from_native()` methods. These two methods are used to convert between the intial datatype, and a primative, serializable datatype. Primative datatypes may be any of a number, string, date/time/datetime or None. They may also be any list or dictionary like object that only contains other primative objects. + +The `.to_native()` method is called to convert the initial datatype into a primative, serializable datatype. The `from_native()` method is called to restore a primative datatype into it's initial representation. + +## Examples + +Let's look at an example of serializing a class that represents an RGB color value: + + class Color(object): + """ + A color represented in the RGB colorspace. + """ + def __init__(self, red, green, blue): + assert(red >= 0 and green >= 0 and blue >= 0) + assert(red < 256 and green < 256 and blue < 256) + self.red, self.green, self.blue = red, green, blue + + class ColourField(serializers.WritableField): + """ + Color objects are serialized into "rgb(#, #, #)" notation. + """ + def to_native(self, obj): + return "rgb(%d, %d, %d)" % (obj.red, obj.green, obj.blue) + + def from_native(self, data): + data = data.strip('rgb(').rstrip(')') + red, green, blue = [int(col) for col in data.split(',')] + return Color(red, green, blue) + + +By default field values are treated as mapping to an attribute on the object. If you need to customize how the field value is accessed and set you need to override `.field_to_native()` and/or `.field_from_native()`. + +As an example, let's create a field that can be used represent the class name of the object being serialized: + + class ClassNameField(serializers.Field): + def field_to_native(self, obj, field_name): + """ + Serialize the object's class name. + """ + return obj.__class__ + [cite]: https://docs.djangoproject.com/en/dev/ref/forms/api/#django.forms.Form.cleaned_data [FILE_UPLOAD_HANDLERS]: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-FILE_UPLOAD_HANDLERS [strftime]: http://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior diff --git a/docs/api-guide/serializers.md b/docs/api-guide/serializers.md index ee7208a36..42e81cad5 100644 --- a/docs/api-guide/serializers.md +++ b/docs/api-guide/serializers.md @@ -87,27 +87,21 @@ By default, serializers must be passed values for all required fields or they wi serializer = CommentSerializer(comment, data={'content': u'foo bar'}, partial=True) # Update `instance` with partial data -## Serializing multiple objects - -To serialize a queryset or list of objects instead of a single object instance, you should pass the `many=True` flag when instantiating the serializer. - - queryset = Comment.objects.all() - serializer = CommentSerializer(queryset, many=True) - serializer.data - # [ - # {'email': u'leila@example.com', 'content': u'foo bar', 'created': datetime.datetime(2012, 8, 22, 16, 20, 9, 822774)}, - # {'email': u'jamie@example.com', 'content': u'baz', 'created': datetime.datetime(2013, 1, 12, 16, 12, 45, 104445)} - # ] - ## Validation -When deserializing data, you always need to call `is_valid()` before attempting to access the deserialized object. If any validation errors occur, the `.errors` property will contain a dictionary representing the resulting error messages. +When deserializing data, you always need to call `is_valid()` before attempting to access the deserialized object. If any validation errors occur, the `.errors` property will contain a dictionary representing the resulting error messages. For example: + + serializer = CommentSerializer(data={'email': 'foobar', 'content': 'baz'}) + serializer.is_valid() + # False + serializer.errors + # {'email': [u'Enter a valid e-mail address.'], 'created': [u'This field is required.']} Each key in the dictionary will be the field name, and the values will be lists of strings of any error messages corresponding to that field. The `non_field_errors` key may also be present, and will list any general validation errors. When deserializing a list of items, errors will be returned as a list of dictionaries representing each of the deserialized items. -### Field-level validation +#### Field-level validation You can specify custom field-level validation by adding `.validate_` methods to your `Serializer` subclass. These are analagous to `.clean_` methods on Django forms, but accept slightly different arguments. @@ -130,7 +124,7 @@ Your `validate_` methods should either just return the `attrs` dictio raise serializers.ValidationError("Blog post is not about Django") return attrs -### Object-level validation +#### Object-level validation To do any other validation that requires access to multiple fields, add a method called `.validate()` to your `Serializer` subclass. This method takes a single argument, which is the `attrs` dictionary. It should raise a `ValidationError` if necessary, or just return `attrs`. For example: @@ -151,26 +145,44 @@ To do any other validation that requires access to multiple fields, add a method ## Saving object state -Serializers also include a `.save()` method that you can override if you want to provide a method of persisting the state of a deserialized object. The default behavior of the method is to simply call `.save()` on the deserialized object instance. +To save the deserialized objects created by a serializer, call the `.save()` method: + + if serializer.is_valid(): + serializer.save() + +The default behavior of the method is to simply call `.save()` on the deserialized object instance. You can override the default save behaviour by overriding the `.save_object(obj)` method on the serializer class. The generic views provided by REST framework call the `.save()` method when updating or creating entities. ## Dealing with nested objects -The previous example is fine for dealing with objects that only have simple datatypes, but sometimes we also need to be able to represent more complex objects, -where some of the attributes of an object might not be simple datatypes such as strings, dates or integers. +The previous examples are fine for dealing with objects that only have simple datatypes, but sometimes we also need to be able to represent more complex objects, where some of the attributes of an object might not be simple datatypes such as strings, dates or integers. The `Serializer` class is itself a type of `Field`, and can be used to represent relationships where one object type is nested inside another. class UserSerializer(serializers.Serializer): - email = serializers.Field() - username = serializers.Field() + email = serializers.EmailField() + username = serializers.CharField(max_length=100) class CommentSerializer(serializers.Serializer): user = UserSerializer() - title = serializers.Field() - content = serializers.Field() - created = serializers.Field() + content = serializers.CharField(max_length=200) + created = serializers.DateTimeField() + +If a nested representation may optionally accept the `None` value you should pass the `required=False` flag to the nested serializer. + + class CommentSerializer(serializers.Serializer): + user = UserSerializer(required=False) # May be an anonymous user. + content = serializers.CharField(max_length=200) + created = serializers.DateTimeField() + +Similarly if a nested representation should be a list of items, you should the `many=True` flag to the nested serialized. + + class CommentSerializer(serializers.Serializer): + user = UserSerializer(required=False) + edits = EditItemSerializer(many=True) # A nested list of 'edit' items. + content = serializers.CharField(max_length=200) + created = serializers.DateTimeField() --- @@ -178,6 +190,96 @@ The `Serializer` class is itself a type of `Field`, and can be used to represent --- +## Dealing with multiple objects + +The `Serializer` class can also handle serializing or deserializing lists of objects. + +#### Serializing multiple objects + +To serialize a queryset or list of objects instead of a single object instance, you should pass the `many=True` flag when instantiating the serializer. You can then pass a queryset or list of objects to be serialized. + + queryset = Book.objects.all() + serializer = BookSerializer(queryset, many=True) + serializer.data + # [ + # {'id': 0, 'title': 'The electric kool-aid acid test', 'author': 'Tom Wolfe'}, + # {'id': 1, 'title': 'If this is a man', 'author': 'Primo Levi'}, + # {'id': 2, 'title': 'The wind-up bird chronicle', 'author': 'Haruki Murakami'} + # ] + +#### Deserializing multiple objects for creation + +To deserialize a list of object data, and create multiple object instances in a single pass, you should also set the `many=True` flag, and pass a list of data to be deserialized. + +This allows you to write views that create multiple items when a `POST` request is made. + +For example: + + data = [ + {'title': 'The bell jar', 'author': 'Sylvia Plath'}, + {'title': 'For whom the bell tolls', 'author': 'Ernest Hemingway'} + ] + serializer = BookSerializer(data=data, many=True) + serializer.is_valid() + # True + serializer.save() # `.save()` will be called on each deserialized instance + +#### Deserializing multiple objects for update + +You can also deserialize a list of objects as part of a bulk update of multiple existing items. +In this case you need to supply both an existing list or queryset of items, as well as a list of data to update those items with. + +This allows you to write views that update or create multiple items when a `PUT` request is made. + + # Capitalizing the titles of the books + queryset = Book.objects.all() + data = [ + {'id': 3, 'title': 'The Bell Jar', 'author': 'Sylvia Plath'}, + {'id': 4, 'title': 'For Whom the Bell Tolls', 'author': 'Ernest Hemingway'} + ] + serializer = BookSerializer(queryset, data=data, many=True) + serializer.is_valid() + # True + serialize.save() # `.save()` will be called on each updated or newly created instance. + +Bulk updates will update any instances that already exist, and create new instances for data items that do not have a corresponding instance. + +When performing a bulk update you may want any items that are not present in the incoming data to be deleted. To do so, pass `allow_delete=True` to the serializer. + + serializer = BookSerializer(queryset, data=data, many=True, allow_delete=True) + serializer.is_valid() + # True + serializer.save() # `.save()` will be called on each updated or newly created instance. + # `.delete()` will be called on any other items in the `queryset`. + +Passing `allow_delete=True` ensures that any update operations will completely overwrite the existing queryset, rather than simply updating any objects found in the incoming data. + +#### How identity is determined when performing bulk updates + +Performing a bulk update is slightly more complicated than performing a bulk creation, because the serializer needs a way of determining how the items in the incoming data should be matched against the existing object instances. + +By default the serializer class will use the `id` key on the incoming data to determine the canonical identity of an object. If you need to change this behavior you should override the `get_identity` method on the `Serializer` class. For example: + + class AccountSerializer(serializers.Serializer): + slug = serializers.CharField(max_length=100) + created = serializers.DateTimeField() + ... # Various other fields + + def get_identity(self, data): + """ + This hook is required for bulk update. + We need to override the default, to use the slug as the identity. + + Note that the data has not yet been validated at this point, + so we need to deal gracefully with incorrect datatypes. + """ + try: + return data.get('slug', None) + except AttributeError: + return None + +To map the incoming data items to their corresponding object instances, the `.get_identity()` method will be called both against the incoming data, and against the serialized representation of the existing objects. + ## Including extra context There are some cases where you need to provide extra context to the serializer in addition to the object being serialized. One common case is if you're using a serializer that includes hyperlinked relations, which requires the serializer to have access to the current request so that it can properly generate fully qualified URLs. @@ -190,47 +292,6 @@ You can provide arbitrary additional context by passing a `context` argument whe The context dictionary can be used within any serializer field logic, such as a custom `.to_native()` method, by accessing the `self.context` attribute. -## Creating custom fields - -If you want to create a custom field, you'll probably want to override either one or both of the `.to_native()` and `.from_native()` methods. These two methods are used to convert between the intial datatype, and a primative, serializable datatype. Primative datatypes may be any of a number, string, date/time/datetime or None. They may also be any list or dictionary like object that only contains other primative objects. - -The `.to_native()` method is called to convert the initial datatype into a primative, serializable datatype. The `from_native()` method is called to restore a primative datatype into it's initial representation. - -Let's look at an example of serializing a class that represents an RGB color value: - - class Color(object): - """ - A color represented in the RGB colorspace. - """ - def __init__(self, red, green, blue): - assert(red >= 0 and green >= 0 and blue >= 0) - assert(red < 256 and green < 256 and blue < 256) - self.red, self.green, self.blue = red, green, blue - - class ColourField(serializers.WritableField): - """ - Color objects are serialized into "rgb(#, #, #)" notation. - """ - def to_native(self, obj): - return "rgb(%d, %d, %d)" % (obj.red, obj.green, obj.blue) - - def from_native(self, data): - data = data.strip('rgb(').rstrip(')') - red, green, blue = [int(col) for col in data.split(',')] - return Color(red, green, blue) - - -By default field values are treated as mapping to an attribute on the object. If you need to customize how the field value is accessed and set you need to override `.field_to_native()` and/or `.field_from_native()`. - -As an example, let's create a field that can be used represent the class name of the object being serialized: - - class ClassNameField(serializers.Field): - def field_to_native(self, obj, field_name): - """ - Serialize the object's class name. - """ - return obj.__class__ - --- # ModelSerializers diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 3029cf1c1..0b0c2c9cc 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -110,13 +110,15 @@ class BaseSerializer(Field): _dict_class = SortedDictWithMetadata def __init__(self, instance=None, data=None, files=None, - context=None, partial=False, many=None, source=None): + context=None, partial=False, many=None, source=None, + allow_delete=False): super(BaseSerializer, self).__init__(source=source) self.opts = self._options_class(self.Meta) self.parent = None self.root = None self.partial = partial self.many = many + self.allow_delete = allow_delete self.context = context or {} @@ -130,6 +132,12 @@ class BaseSerializer(Field): self._errors = None self._deleted = None + if many and instance is not None and not hasattr(instance, '__iter__'): + raise ValueError('instance should be a queryset or other iterable with many=True') + + if allow_delete and not many: + raise ValueError('allow_delete should only be used for bulk updates, but you have not set many=True') + ##### # Methods to determine which fields to use when (de)serializing objects. @@ -336,8 +344,15 @@ class BaseSerializer(Field): """ This hook is required for bulk update. It is used to determine the canonical identity of a given object. + + Note that the data has not been validated at this point, so we need + to make sure that we catch any cases of incorrect datatypes being + passed to this method. """ - return data.get('id') + try: + return data.get('id', None) + except AttributeError: + return None @property def errors(self): @@ -371,22 +386,12 @@ class BaseSerializer(Field): identities = [self.get_identity(self.to_native(obj)) for obj in objects] identity_to_objects = dict(zip(identities, objects)) - try: - iter(data) - if isinstance(data, dict): - raise TypeError - except TypeError: - self._errors = {'non_field_errors': ['Expected a list of items']} - else: + if hasattr(data, '__iter__') and not isinstance(data, (dict, six.text_type)): for item in data: if update: # Determine which object we're updating - try: - identity = self.get_identity(item) - except: - self.object = None - else: - self.object = identity_to_objects.pop(identity, None) + identity = self.get_identity(item) + self.object = identity_to_objects.pop(identity, None) ret.append(self.from_native(item, None)) errors.append(self._errors) @@ -395,6 +400,8 @@ class BaseSerializer(Field): self._deleted = identity_to_objects.values() self._errors = any(errors) and errors or [] + else: + self._errors = {'non_field_errors': ['Expected a list of items']} else: ret = self.from_native(data, files) @@ -445,7 +452,7 @@ class BaseSerializer(Field): else: self.save_object(self.object, **kwargs) - if self._deleted: + if self.allow_delete and self._deleted: [self.delete_object(item) for item in self._deleted] return self.object @@ -736,3 +743,13 @@ class HyperlinkedModelSerializer(ModelSerializer): 'many': to_many } return HyperlinkedRelatedField(**kwargs) + + def get_identity(self, data): + """ + This hook is required for bulk update. + We need to override the default, to use the url as the identity. + """ + try: + return data.get('url', None) + except AttributeError: + return None diff --git a/rest_framework/tests/serializer_bulk_update.py b/rest_framework/tests/serializer_bulk_update.py index 2f4164880..afc1a1a9f 100644 --- a/rest_framework/tests/serializer_bulk_update.py +++ b/rest_framework/tests/serializer_bulk_update.py @@ -201,7 +201,29 @@ class BulkUpdateSerializerTests(TestCase): 'author': 'Haruki Murakami' } ] - serializer = self.BookSerializer(self.books(), data=data, many=True) + serializer = self.BookSerializer(self.books(), data=data, many=True, allow_delete=True) + self.assertEqual(serializer.is_valid(), True) + self.assertEqual(serializer.data, data) + serializer.save() + new_data = self.BookSerializer(self.books(), many=True).data + self.assertEqual(data, new_data) + + def test_bulk_update_and_create(self): + """ + Bulk update serialization may also include created items. + """ + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 3, + 'title': 'Kafka on the shore', + 'author': 'Haruki Murakami' + } + ] + serializer = self.BookSerializer(self.books(), data=data, many=True, allow_delete=True) self.assertEqual(serializer.is_valid(), True) self.assertEqual(serializer.data, data) serializer.save() @@ -227,6 +249,6 @@ class BulkUpdateSerializerTests(TestCase): {}, {'id': ['Enter a whole number.']} ] - serializer = self.BookSerializer(self.books(), data=data, many=True) + serializer = self.BookSerializer(self.books(), data=data, many=True, allow_delete=True) self.assertEqual(serializer.is_valid(), False) self.assertEqual(serializer.errors, expected_errors) From 4055129662023a13c40ccea108a990b818d244cd Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 21:31:32 +0000 Subject: [PATCH 22/50] If oauth is not attempted don't throw an error. Fixes #748. --- rest_framework/authentication.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest_framework/authentication.py b/rest_framework/authentication.py index b4b73699e..8f4ec536e 100644 --- a/rest_framework/authentication.py +++ b/rest_framework/authentication.py @@ -204,6 +204,9 @@ class OAuthAuthentication(BaseAuthentication): except oauth.Error as err: raise exceptions.AuthenticationFailed(err.message) + if not oauth_request: + return None + oauth_params = oauth_provider.consts.OAUTH_PARAMETERS_NAMES found = any(param for param in oauth_params if param in oauth_request) From 9a2ba4bf54d72452c2800bb3e51ea718a72250bc Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 21:37:51 +0000 Subject: [PATCH 23/50] Update release notes --- docs/topics/release-notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/release-notes.md b/docs/topics/release-notes.md index c45fff880..f506c610f 100644 --- a/docs/topics/release-notes.md +++ b/docs/topics/release-notes.md @@ -42,6 +42,8 @@ You can determine your currently installed version using `pip freeze`: ### Master +* Regression fix: Date and time fields return date/time objects by default. Fixes regressions caused by 2.2.2. See [#743][743] for more details. +* Bugfix: Fix 500 error is OAuth not attempted with OAuthAuthentication class installed. * `Serializer.save()` now supports arbitrary keyword args which are passed through to the object `.save()` method. Mixins use `force_insert` and `force_update` where appropriate, resulting in one less database query. ### 2.2.4 @@ -434,6 +436,7 @@ This change will not affect user code, so long as it's following the recommended [django-deprecation-policy]: https://docs.djangoproject.com/en/dev/internals/release-process/#internal-release-deprecation-policy [defusedxml-announce]: http://blog.python.org/2013/02/announcing-defusedxml-fixes-for-xml.html [2.2-announcement]: 2.2-announcement.md +[743]: https://github.com/tomchristie/django-rest-framework/pull/743 [staticfiles14]: https://docs.djangoproject.com/en/1.4/howto/static-files/#with-a-template-tag [staticfiles13]: https://docs.djangoproject.com/en/1.3/howto/static-files/#with-a-template-tag [2.1.0-notes]: https://groups.google.com/d/topic/django-rest-framework/Vv2M0CMY9bg/discussion From 624ce250cd666d647a8498dd6610ab84b459ee6f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 22:06:38 +0000 Subject: [PATCH 24/50] Updating contribution docs (still wip) --- docs/topics/contributing.md | 136 ++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/docs/topics/contributing.md b/docs/topics/contributing.md index 7fd61c10e..a13f4461e 100644 --- a/docs/topics/contributing.md +++ b/docs/topics/contributing.md @@ -4,12 +4,138 @@ > > — [Tim Berners-Lee][cite] -## Running the tests +There are many ways you can contribute to Django REST framework. We'd like it to be a community-led project, so please get involved and help shape the future of the project. -## Building the docs +# Community -## Managing compatibility issues +If you use and enjoy REST framework please consider [staring the project on GitHub][github], and [upvoting it on Django packages][django-packages]. Doing so helps potential new users see that the project is well used, and help us continue to attract new users. -**Describe compat module** +You might also consider writing a blog post on your experience with using REST framework, writing a tutorial about using the project with a particular javascript framework, or simply sharing the love on Twitter. -[cite]: http://www.w3.org/People/Berners-Lee/FAQ.html \ No newline at end of file +Other really great ways you can help move the community forward include helping answer questions on the [discussion group][google-group], or setting up an [email alert on StackOverflow][so-filter] so that you get notified of any new questions with the `django-rest-framework` tag. + +When answering questions make sure to help future contributors find their way around by hyperlinking wherever possible to related threads and tickets, and include backlinks from those items if relevant. + +# Issues + +Usage questions should be directed to the [discussion group][google-group]. Feature requests, bug reports and other issues should be raised on the GitHub [issue tracker][issues]. + +Some tips on good issue reporting: + +* When decribing issues try to phrase your ticket in terms of the *behavior* you think needs changing rather than the *code* you think need changing. +* Search the issue list first for related items, and make sure you're running the latest version of REST framework before reporting an issue. +* If reporting a bug, then try to include a pull request with a failing test case. This'll help us quickly identify if there is a valid issue, and make sure that it gets fixed more quickly if there is one. + + + +* TODO: Triage + +# Development + +* git clone & PYTHONPATH +* Pep8 +* Recommend editor that runs pep8 + +### Pull requests + +* Make pull requests early +* Describe branching + +### Managing compatibility issues + +* Describe compat module + +# Testing + +* Running the tests +* tox + +# Documentation + +The documentation for REST framework is built from the [Markdown][markdown] source files in [the docs directory][docs]. + +There are many great markdown editors that make working with the documentation really easy. The [Mou editor for Mac][mou] is one such editor that comes highly recommended. + +## Building the documentation + +To build the documentation, simply run the `mkdocs.py` script. + + ./mkdocs.py + +This will build the html output into the `html` directory. + +You can build the documentation and open a preview in a browser window by using the `-p` flag. + + ./mkdocs.py -p + +## Language style + +Documentation should be in American English. The tone of the documentation is very important - try to stick to a simple, plain, objective and well-balanced style where possible. + +Some other tips: + +* Keep paragraphs reasonably short. +* Use double spacing after the end of sentences. +* Don't use the abbreviations such as 'e.g..' but instead use long form, such as 'For example'. + +## Markdown style + +There are a couple of conventions you should follow when working on the documentation. + +##### 1. Headers + +Headers should use the hash style. For example: + + ### Some important topic + +The underline style should not be used. **Don't do this:** + + Some important topic + ==================== + +##### 2. Links + +Links should always use the reference style, with the referenced hyperlinks kept at the end of the document. + + Here is a link to [some other thing][other-thing]. + + More text... + + [other-thing]: http://example.com/other/thing + +This style helps keep the documentation source consistent and readable. + +If you are hyperlinking to another REST framework document, you should use a relative link, and link to the `.md` suffix. For example: + + [authentication]: ../api-guide/authentication.md + +Linking in this style means you'll be able to click the hyperlink in your markdown editor to open the referenced document. When the documentation is built, these links will be converted into regular links to HTML pages. + +##### 3. Notes + +If you want to draw attention to a note or warning, use a pair of enclosing lines, like so: + + --- + + **Note:** Make sure you do this thing. + + --- + +# Third party packages + +* Django reusable app + +# Core committers + +* Still use pull reqs +* Credits + +[cite]: http://www.w3.org/People/Berners-Lee/FAQ.html +[github]: https://github.com/tomchristie/django-rest-framework +[django-packages]: https://www.djangopackages.com/grids/g/api/ +[google-group]: https://groups.google.com/forum/?fromgroups#!forum/django-rest-framework +[so-filter]: http://stackexchange.com/filters/66475/rest-framework +[issues]: https://github.com/tomchristie/django-rest-framework/issues?state=open +[markdown]: http://daringfireball.net/projects/markdown/basics +[docs]: https://github.com/tomchristie/django-rest-framework/tree/master/docs +[mou]: http://mouapp.com/ From 9cdf8411698296fdbedf978d7b83c9d46b30e0d7 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 22:11:30 +0000 Subject: [PATCH 25/50] Tweaking --- docs/topics/contributing.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/contributing.md b/docs/topics/contributing.md index f8e2baabf..a13f4461e 100644 --- a/docs/topics/contributing.md +++ b/docs/topics/contributing.md @@ -26,6 +26,8 @@ Some tips on good issue reporting: * Search the issue list first for related items, and make sure you're running the latest version of REST framework before reporting an issue. * If reporting a bug, then try to include a pull request with a failing test case. This'll help us quickly identify if there is a valid issue, and make sure that it gets fixed more quickly if there is one. + + * TODO: Triage # Development @@ -119,7 +121,7 @@ If you want to draw attention to a note or warning, use a pair of enclosing line --- -# Third party packages +# Third party packages * Django reusable app From addf7e9b36a274506cc940744487977ee8a7b574 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 22:27:03 +0000 Subject: [PATCH 26/50] Defer the writable nested modelserializers work --- rest_framework/serializers.py | 11 +- rest_framework/tests/relations_nested.py | 160 +++++++++++------------ 2 files changed, 76 insertions(+), 95 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 26c34044c..6aca2f574 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -753,16 +753,7 @@ class ModelSerializer(Serializer): if getattr(obj, '_related_data', None): for accessor_name, related in obj._related_data.items(): - if related is None: - previous = getattr(obj, accessor_name, related) - if previous: - previous.delete() - elif isinstance(related, models.Model): - fk_field = obj._meta.get_field_by_name(accessor_name)[0].field.name - setattr(related, fk_field, obj) - self.save_object(related) - else: - setattr(obj, accessor_name, related) + setattr(obj, accessor_name, related) del(obj._related_data) diff --git a/rest_framework/tests/relations_nested.py b/rest_framework/tests/relations_nested.py index 4592e5593..a125ba656 100644 --- a/rest_framework/tests/relations_nested.py +++ b/rest_framework/tests/relations_nested.py @@ -1,125 +1,115 @@ from __future__ import unicode_literals -from django.db import models from django.test import TestCase from rest_framework import serializers +from rest_framework.tests.models import ForeignKeyTarget, ForeignKeySource, NullableForeignKeySource, OneToOneTarget, NullableOneToOneSource -class OneToOneTarget(models.Model): - name = models.CharField(max_length=100) - - -class OneToOneTargetSource(models.Model): - name = models.CharField(max_length=100) - target = models.OneToOneField(OneToOneTarget, null=True, blank=True, - related_name='target_source') - - -class OneToOneSource(models.Model): - name = models.CharField(max_length=100) - target_source = models.OneToOneField(OneToOneTargetSource, related_name='source') - - -class OneToOneSourceSerializer(serializers.ModelSerializer): +class ForeignKeySourceSerializer(serializers.ModelSerializer): class Meta: - model = OneToOneSource - exclude = ('target_source', ) + depth = 1 + model = ForeignKeySource -class OneToOneTargetSourceSerializer(serializers.ModelSerializer): - source = OneToOneSourceSerializer() +class FlatForeignKeySourceSerializer(serializers.ModelSerializer): + class Meta: + model = ForeignKeySource + + +class ForeignKeyTargetSerializer(serializers.ModelSerializer): + sources = FlatForeignKeySourceSerializer(many=True) class Meta: - model = OneToOneTargetSource - exclude = ('target', ) + model = ForeignKeyTarget -class OneToOneTargetSerializer(serializers.ModelSerializer): - target_source = OneToOneTargetSourceSerializer() +class NullableForeignKeySourceSerializer(serializers.ModelSerializer): + class Meta: + depth = 1 + model = NullableForeignKeySource + + +class NullableOneToOneSourceSerializer(serializers.ModelSerializer): + class Meta: + model = NullableOneToOneSource + + +class NullableOneToOneTargetSerializer(serializers.ModelSerializer): + nullable_source = NullableOneToOneSourceSerializer() class Meta: model = OneToOneTarget -class NestedOneToOneTests(TestCase): +class ReverseForeignKeyTests(TestCase): def setUp(self): + target = ForeignKeyTarget(name='target-1') + target.save() + new_target = ForeignKeyTarget(name='target-2') + new_target.save() for idx in range(1, 4): - target = OneToOneTarget(name='target-%d' % idx) - target.save() - target_source = OneToOneTargetSource(name='target-source-%d' % idx, target=target) - target_source.save() - source = OneToOneSource(name='source-%d' % idx, target_source=target_source) + source = ForeignKeySource(name='source-%d' % idx, target=target) source.save() - def test_one_to_one_retrieve(self): - queryset = OneToOneTarget.objects.all() - serializer = OneToOneTargetSerializer(queryset) + def test_foreign_key_retrieve(self): + queryset = ForeignKeySource.objects.all() + serializer = ForeignKeySourceSerializer(queryset, many=True) expected = [ - {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, - {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, - {'id': 3, 'name': 'target-3', 'target_source': {'id': 3, 'name': 'target-source-3', 'source': {'id': 3, 'name': 'source-3'}}} + {'id': 1, 'name': 'source-1', 'target': {'id': 1, 'name': 'target-1'}}, + {'id': 2, 'name': 'source-2', 'target': {'id': 1, 'name': 'target-1'}}, + {'id': 3, 'name': 'source-3', 'target': {'id': 1, 'name': 'target-1'}}, ] self.assertEqual(serializer.data, expected) - def test_one_to_one_create(self): - data = {'id': 4, 'name': 'target-4', 'target_source': {'id': 4, 'name': 'target-source-4', 'source': {'id': 4, 'name': 'source-4'}}} - serializer = OneToOneTargetSerializer(data=data) - self.assertTrue(serializer.is_valid()) - obj = serializer.save() - self.assertEqual(serializer.data, data) - self.assertEqual(obj.name, 'target-4') - - # Ensure (target 4, target_source 4, source 4) are added, and - # everything else is as expected. - queryset = OneToOneTarget.objects.all() - serializer = OneToOneTargetSerializer(queryset) + def test_reverse_foreign_key_retrieve(self): + queryset = ForeignKeyTarget.objects.all() + serializer = ForeignKeyTargetSerializer(queryset, many=True) expected = [ - {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, - {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, - {'id': 3, 'name': 'target-3', 'target_source': {'id': 3, 'name': 'target-source-3', 'source': {'id': 3, 'name': 'source-3'}}}, - {'id': 4, 'name': 'target-4', 'target_source': {'id': 4, 'name': 'target-source-4', 'source': {'id': 4, 'name': 'source-4'}}} + {'id': 1, 'name': 'target-1', 'sources': [ + {'id': 1, 'name': 'source-1', 'target': 1}, + {'id': 2, 'name': 'source-2', 'target': 1}, + {'id': 3, 'name': 'source-3', 'target': 1}, + ]}, + {'id': 2, 'name': 'target-2', 'sources': [ + ]} ] self.assertEqual(serializer.data, expected) - def test_one_to_one_create_with_invalid_data(self): - data = {'id': 4, 'name': 'target-4', 'target_source': {'id': 4, 'name': 'target-source-4', 'source': {'id': 4}}} - serializer = OneToOneTargetSerializer(data=data) - self.assertFalse(serializer.is_valid()) - self.assertEqual(serializer.errors, {'target_source': [{'source': [{'name': ['This field is required.']}]}]}) - def test_one_to_one_update(self): - data = {'id': 3, 'name': 'target-3-updated', 'target_source': {'id': 3, 'name': 'target-source-3-updated', 'source': {'id': 3, 'name': 'source-3-updated'}}} - instance = OneToOneTarget.objects.get(pk=3) - serializer = OneToOneTargetSerializer(instance, data=data) - self.assertTrue(serializer.is_valid()) - obj = serializer.save() - self.assertEqual(serializer.data, data) - self.assertEqual(obj.name, 'target-3-updated') +class NestedNullableForeignKeyTests(TestCase): + def setUp(self): + target = ForeignKeyTarget(name='target-1') + target.save() + for idx in range(1, 4): + if idx == 3: + target = None + source = NullableForeignKeySource(name='source-%d' % idx, target=target) + source.save() - # Ensure (target 3, target_source 3, source 3) are updated, - # and everything else is as expected. - queryset = OneToOneTarget.objects.all() - serializer = OneToOneTargetSerializer(queryset) + def test_foreign_key_retrieve_with_null(self): + queryset = NullableForeignKeySource.objects.all() + serializer = NullableForeignKeySourceSerializer(queryset, many=True) expected = [ - {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, - {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, - {'id': 3, 'name': 'target-3-updated', 'target_source': {'id': 3, 'name': 'target-source-3-updated', 'source': {'id': 3, 'name': 'source-3-updated'}}} + {'id': 1, 'name': 'source-1', 'target': {'id': 1, 'name': 'target-1'}}, + {'id': 2, 'name': 'source-2', 'target': {'id': 1, 'name': 'target-1'}}, + {'id': 3, 'name': 'source-3', 'target': None}, ] self.assertEqual(serializer.data, expected) - def test_one_to_one_delete(self): - data = {'id': 3, 'name': 'target-3', 'target_source': None} - instance = OneToOneTarget.objects.get(pk=3) - serializer = OneToOneTargetSerializer(instance, data=data) - self.assertTrue(serializer.is_valid()) - serializer.save() - # Ensure (target_source 3, source 3) are deleted, - # and everything else is as expected. +class NestedNullableOneToOneTests(TestCase): + def setUp(self): + target = OneToOneTarget(name='target-1') + target.save() + new_target = OneToOneTarget(name='target-2') + new_target.save() + source = NullableOneToOneSource(name='source-1', target=target) + source.save() + + def test_reverse_foreign_key_retrieve_with_null(self): queryset = OneToOneTarget.objects.all() - serializer = OneToOneTargetSerializer(queryset) + serializer = NullableOneToOneTargetSerializer(queryset, many=True) expected = [ - {'id': 1, 'name': 'target-1', 'target_source': {'id': 1, 'name': 'target-source-1', 'source': {'id': 1, 'name': 'source-1'}}}, - {'id': 2, 'name': 'target-2', 'target_source': {'id': 2, 'name': 'target-source-2', 'source': {'id': 2, 'name': 'source-2'}}}, - {'id': 3, 'name': 'target-3', 'target_source': None} + {'id': 1, 'name': 'target-1', 'nullable_source': {'id': 1, 'name': 'source-1', 'target': 1}}, + {'id': 2, 'name': 'target-2', 'nullable_source': None}, ] self.assertEqual(serializer.data, expected) From 6111dcef5216ff12fb9353bb8c7d71b021634513 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 22 Mar 2013 22:58:04 +0000 Subject: [PATCH 27/50] Update release notes --- docs/topics/release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/release-notes.md b/docs/topics/release-notes.md index f506c610f..935bd9053 100644 --- a/docs/topics/release-notes.md +++ b/docs/topics/release-notes.md @@ -42,6 +42,7 @@ You can determine your currently installed version using `pip freeze`: ### Master +* Serializers now support bulk create and bulk update operations. * Regression fix: Date and time fields return date/time objects by default. Fixes regressions caused by 2.2.2. See [#743][743] for more details. * Bugfix: Fix 500 error is OAuth not attempted with OAuthAuthentication class installed. * `Serializer.save()` now supports arbitrary keyword args which are passed through to the object `.save()` method. Mixins use `force_insert` and `force_update` where appropriate, resulting in one less database query. From dab158e1fd827285032e331c10acc60e9719ace3 Mon Sep 17 00:00:00 2001 From: Sitong Peng Date: Mon, 25 Mar 2013 09:27:12 -0700 Subject: [PATCH 28/50] Tiny typo --- docs/tutorial/4-authentication-and-permissions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/4-authentication-and-permissions.md b/docs/tutorial/4-authentication-and-permissions.md index 3ee755a2f..878672bb0 100644 --- a/docs/tutorial/4-authentication-and-permissions.md +++ b/docs/tutorial/4-authentication-and-permissions.md @@ -179,7 +179,7 @@ Now, if you open a browser again, you find that the 'DELETE' and 'PUT' actions o ## Authenticating with the API -Because we now have a set of permissions on the API, we need to authenticate our requests to it if we want to edit any snippets. We havn't set up any [authentication classes][authentication], so the defaults are currently applied, which are `SessionAuthentication` and `BasicAuthentication`. +Because we now have a set of permissions on the API, we need to authenticate our requests to it if we want to edit any snippets. We haven't set up any [authentication classes][authentication], so the defaults are currently applied, which are `SessionAuthentication` and `BasicAuthentication`. When we interact with the API through the web browser, we can login, and the browser session will then provide the required authentication for the requests. From dca24cd91488af62152b7e711cd6869b2d60c0ec Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 25 Mar 2013 16:44:36 +0000 Subject: [PATCH 29/50] Added @stoneg for typo fix #754. Thank you! --- docs/topics/credits.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/credits.md b/docs/topics/credits.md index b533daa99..7dd9cd2c0 100644 --- a/docs/topics/credits.md +++ b/docs/topics/credits.md @@ -112,6 +112,7 @@ The following people have helped make REST framework great. * Bouke Haarsma - [bouke] * Pierre Dulac - [dulaccc] * Dave Kuhn - [kuhnza] +* Sitong Peng - [stoneg] Many thanks to everyone who's contributed to the project. @@ -258,3 +259,4 @@ You can also contact [@_tomchristie][twitter] directly on twitter. [bouke]: https://github.com/bouke [dulaccc]: https://github.com/dulaccc [kuhnza]: https://github.com/kuhnza +[stoneg]: https://github.com/stoneg From 7eefcf7e53f2bc37733a601041f23d354c7729f5 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 25 Mar 2013 20:26:34 +0000 Subject: [PATCH 30/50] Bulk update, allow_add_remove flag --- docs/api-guide/serializers.md | 10 +++--- rest_framework/serializers.py | 16 +++++---- .../tests/serializer_bulk_update.py | 34 ++++++++++++++++--- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/docs/api-guide/serializers.md b/docs/api-guide/serializers.md index 42e81cad5..aaff760ef 100644 --- a/docs/api-guide/serializers.md +++ b/docs/api-guide/serializers.md @@ -242,17 +242,17 @@ This allows you to write views that update or create multiple items when a `PUT` # True serialize.save() # `.save()` will be called on each updated or newly created instance. -Bulk updates will update any instances that already exist, and create new instances for data items that do not have a corresponding instance. +By default bulk updates will be limited to updating instances that already exist in the provided queryset. -When performing a bulk update you may want any items that are not present in the incoming data to be deleted. To do so, pass `allow_delete=True` to the serializer. +When performing a bulk update you may want to allow new items to be created, and missing items to be deleted. To do so, pass `allow_add_remove=True` to the serializer. - serializer = BookSerializer(queryset, data=data, many=True, allow_delete=True) + serializer = BookSerializer(queryset, data=data, many=True, allow_add_remove=True) serializer.is_valid() # True - serializer.save() # `.save()` will be called on each updated or newly created instance. + serializer.save() # `.save()` will be called on updated or newly created instances. # `.delete()` will be called on any other items in the `queryset`. -Passing `allow_delete=True` ensures that any update operations will completely overwrite the existing queryset, rather than simply updating any objects found in the incoming data. +Passing `allow_delete=True` ensures that any update operations will completely overwrite the existing queryset, rather than simply updating existing objects. #### How identity is determined when performing bulk updates diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 6aca2f574..1b2b08217 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -130,14 +130,14 @@ class BaseSerializer(WritableField): def __init__(self, instance=None, data=None, files=None, context=None, partial=False, many=None, - allow_delete=False, **kwargs): + allow_add_remove=False, **kwargs): super(BaseSerializer, self).__init__(**kwargs) self.opts = self._options_class(self.Meta) self.parent = None self.root = None self.partial = partial self.many = many - self.allow_delete = allow_delete + self.allow_add_remove = allow_add_remove self.context = context or {} @@ -154,8 +154,8 @@ class BaseSerializer(WritableField): if many and instance is not None and not hasattr(instance, '__iter__'): raise ValueError('instance should be a queryset or other iterable with many=True') - if allow_delete and not many: - raise ValueError('allow_delete should only be used for bulk updates, but you have not set many=True') + if allow_add_remove and not many: + raise ValueError('allow_add_remove should only be used for bulk updates, but you have not set many=True') ##### # Methods to determine which fields to use when (de)serializing objects. @@ -448,6 +448,10 @@ class BaseSerializer(WritableField): # Determine which object we're updating identity = self.get_identity(item) self.object = identity_to_objects.pop(identity, None) + if self.object is None and not self.allow_add_remove: + ret.append(None) + errors.append({'non_field_errors': ['Cannot create a new item, only existing items may be updated.']}) + continue ret.append(self.from_native(item, None)) errors.append(self._errors) @@ -457,7 +461,7 @@ class BaseSerializer(WritableField): self._errors = any(errors) and errors or [] else: - self._errors = {'non_field_errors': ['Expected a list of items']} + self._errors = {'non_field_errors': ['Expected a list of items.']} else: ret = self.from_native(data, files) @@ -508,7 +512,7 @@ class BaseSerializer(WritableField): else: self.save_object(self.object, **kwargs) - if self.allow_delete and self._deleted: + if self.allow_add_remove and self._deleted: [self.delete_object(item) for item in self._deleted] return self.object diff --git a/rest_framework/tests/serializer_bulk_update.py b/rest_framework/tests/serializer_bulk_update.py index afc1a1a9f..8b0ded1a8 100644 --- a/rest_framework/tests/serializer_bulk_update.py +++ b/rest_framework/tests/serializer_bulk_update.py @@ -98,7 +98,7 @@ class BulkCreateSerializerTests(TestCase): serializer = self.BookSerializer(data=data, many=True) self.assertEqual(serializer.is_valid(), False) - expected_errors = {'non_field_errors': ['Expected a list of items']} + expected_errors = {'non_field_errors': ['Expected a list of items.']} self.assertEqual(serializer.errors, expected_errors) @@ -115,7 +115,7 @@ class BulkCreateSerializerTests(TestCase): serializer = self.BookSerializer(data=data, many=True) self.assertEqual(serializer.is_valid(), False) - expected_errors = {'non_field_errors': ['Expected a list of items']} + expected_errors = {'non_field_errors': ['Expected a list of items.']} self.assertEqual(serializer.errors, expected_errors) @@ -201,11 +201,12 @@ class BulkUpdateSerializerTests(TestCase): 'author': 'Haruki Murakami' } ] - serializer = self.BookSerializer(self.books(), data=data, many=True, allow_delete=True) + serializer = self.BookSerializer(self.books(), data=data, many=True, allow_add_remove=True) self.assertEqual(serializer.is_valid(), True) self.assertEqual(serializer.data, data) serializer.save() new_data = self.BookSerializer(self.books(), many=True).data + self.assertEqual(data, new_data) def test_bulk_update_and_create(self): @@ -223,13 +224,36 @@ class BulkUpdateSerializerTests(TestCase): 'author': 'Haruki Murakami' } ] - serializer = self.BookSerializer(self.books(), data=data, many=True, allow_delete=True) + serializer = self.BookSerializer(self.books(), data=data, many=True, allow_add_remove=True) self.assertEqual(serializer.is_valid(), True) self.assertEqual(serializer.data, data) serializer.save() new_data = self.BookSerializer(self.books(), many=True).data self.assertEqual(data, new_data) + def test_bulk_update_invalid_create(self): + """ + Bulk update serialization without allow_add_remove may not create items. + """ + data = [ + { + 'id': 0, + 'title': 'The electric kool-aid acid test', + 'author': 'Tom Wolfe' + }, { + 'id': 3, + 'title': 'Kafka on the shore', + 'author': 'Haruki Murakami' + } + ] + expected_errors = [ + {}, + {'non_field_errors': ['Cannot create a new item, only existing items may be updated.']} + ] + serializer = self.BookSerializer(self.books(), data=data, many=True) + self.assertEqual(serializer.is_valid(), False) + self.assertEqual(serializer.errors, expected_errors) + def test_bulk_update_error(self): """ Incorrect bulk update serialization should return error data. @@ -249,6 +273,6 @@ class BulkUpdateSerializerTests(TestCase): {}, {'id': ['Enter a whole number.']} ] - serializer = self.BookSerializer(self.books(), data=data, many=True, allow_delete=True) + serializer = self.BookSerializer(self.books(), data=data, many=True, allow_add_remove=True) self.assertEqual(serializer.is_valid(), False) self.assertEqual(serializer.errors, expected_errors) From 8387cb5d1655e4d29cf8bca1919038091427e584 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 25 Mar 2013 20:28:17 +0000 Subject: [PATCH 31/50] Docs fix --- docs/api-guide/serializers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api-guide/serializers.md b/docs/api-guide/serializers.md index aaff760ef..dfa0cacea 100644 --- a/docs/api-guide/serializers.md +++ b/docs/api-guide/serializers.md @@ -252,7 +252,7 @@ When performing a bulk update you may want to allow new items to be created, and serializer.save() # `.save()` will be called on updated or newly created instances. # `.delete()` will be called on any other items in the `queryset`. -Passing `allow_delete=True` ensures that any update operations will completely overwrite the existing queryset, rather than simply updating existing objects. +Passing `allow_add_remove=True` ensures that any update operations will completely overwrite the existing queryset, rather than simply updating existing objects. #### How identity is determined when performing bulk updates From 92c929094c88125ea4a2fd359ec99d2b4114f081 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 26 Mar 2013 07:48:53 +0000 Subject: [PATCH 32/50] Version 2.2.5 --- docs/api-guide/fields.md | 7 ++++++- docs/api-guide/serializers.md | 2 +- docs/topics/release-notes.md | 5 ++++- rest_framework/__init__.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/api-guide/fields.md b/docs/api-guide/fields.md index 028769368..42f89f46a 100644 --- a/docs/api-guide/fields.md +++ b/docs/api-guide/fields.md @@ -197,12 +197,16 @@ If you want to override this behavior, you'll need to declare the `DateTimeField class Meta: model = Comment +Note that by default, datetime representations are deteremined by the renderer in use, although this can be explicitly overridden as detailed below. + +In the case of JSON this means the default datetime representation uses the [ECMA 262 date time string specification][ecma262]. This is a subset of ISO 8601 which uses millisecond precision, and includes the 'Z' suffix for the UTC timezone, for example: `2013-01-29T12:34:56.123Z`. + **Signature:** `DateTimeField(format=None, input_formats=None)` * `format` - A string representing the output format. If not specified, this defaults to `None`, which indicates that python `datetime` objects should be returned by `to_native`. In this case the datetime encoding will be determined by the renderer. * `input_formats` - A list of strings representing the input formats which may be used to parse the date. If not specified, the `DATETIME_INPUT_FORMATS` setting will be used, which defaults to `['iso-8601']`. -DateTime format strings may either be [python strftime formats][strftime] which explicitly specifiy the format, or the special string `'iso-8601'`, which indicates that [ISO 8601][iso8601] style datetimes should be used. (eg `'2013-01-29T12:34:56.000000'`) +DateTime format strings may either be [python strftime formats][strftime] which explicitly specifiy the format, or the special string `'iso-8601'`, which indicates that [ISO 8601][iso8601] style datetimes should be used. (eg `'2013-01-29T12:34:56.000000Z'`) ## DateField @@ -318,5 +322,6 @@ As an example, let's create a field that can be used represent the class name of [cite]: https://docs.djangoproject.com/en/dev/ref/forms/api/#django.forms.Form.cleaned_data [FILE_UPLOAD_HANDLERS]: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-FILE_UPLOAD_HANDLERS +[ecma262]: http://ecma-international.org/ecma-262/5.1/#sec-15.9.1.15 [strftime]: http://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior [iso8601]: http://www.w3.org/TR/NOTE-datetime diff --git a/docs/api-guide/serializers.md b/docs/api-guide/serializers.md index dfa0cacea..1a3c3431b 100644 --- a/docs/api-guide/serializers.md +++ b/docs/api-guide/serializers.md @@ -256,7 +256,7 @@ Passing `allow_add_remove=True` ensures that any update operations will complete #### How identity is determined when performing bulk updates -Performing a bulk update is slightly more complicated than performing a bulk creation, because the serializer needs a way of determining how the items in the incoming data should be matched against the existing object instances. +Performing a bulk update is slightly more complicated than performing a bulk creation, because the serializer needs a way to determine how the items in the incoming data should be matched against the existing object instances. By default the serializer class will use the `id` key on the incoming data to determine the canonical identity of an object. If you need to change this behavior you should override the `get_identity` method on the `Serializer` class. For example: diff --git a/docs/topics/release-notes.md b/docs/topics/release-notes.md index f506c610f..e63aee495 100644 --- a/docs/topics/release-notes.md +++ b/docs/topics/release-notes.md @@ -40,8 +40,11 @@ You can determine your currently installed version using `pip freeze`: ## 2.2.x series -### Master +### 2.2.5 +**Date**: 26th March 2013 + +* Serializer support for bulk create and bulk update operations. * Regression fix: Date and time fields return date/time objects by default. Fixes regressions caused by 2.2.2. See [#743][743] for more details. * Bugfix: Fix 500 error is OAuth not attempted with OAuthAuthentication class installed. * `Serializer.save()` now supports arbitrary keyword args which are passed through to the object `.save()` method. Mixins use `force_insert` and `force_update` where appropriate, resulting in one less database query. diff --git a/rest_framework/__init__.py b/rest_framework/__init__.py index cf0056360..c86403d83 100644 --- a/rest_framework/__init__.py +++ b/rest_framework/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.2.4' +__version__ = '2.2.5' VERSION = __version__ # synonym From f1b8fee4f1e0ea2503d4e0453bdc3049edaa2598 Mon Sep 17 00:00:00 2001 From: Fernando Rocha Date: Wed, 27 Mar 2013 14:05:46 -0300 Subject: [PATCH 33/50] client credentials should be optional (fix #759) client credentials should only be required on token request Signed-off-by: Fernando Rocha --- rest_framework/authentication.py | 30 +++++++++++++++----------- rest_framework/tests/authentication.py | 12 +++++++++++ 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/rest_framework/authentication.py b/rest_framework/authentication.py index 8f4ec536e..f4626a2e3 100644 --- a/rest_framework/authentication.py +++ b/rest_framework/authentication.py @@ -2,14 +2,16 @@ Provides a set of pluggable authentication policies. """ from __future__ import unicode_literals +import base64 +from datetime import datetime + from django.contrib.auth import authenticate from django.core.exceptions import ImproperlyConfigured from rest_framework import exceptions, HTTP_HEADER_ENCODING from rest_framework.compat import CsrfViewMiddleware from rest_framework.compat import oauth, oauth_provider, oauth_provider_store -from rest_framework.compat import oauth2_provider, oauth2_provider_forms, oauth2_provider_backends +from rest_framework.compat import oauth2_provider, oauth2_provider_forms from rest_framework.authtoken.models import Token -import base64 def get_authorization_header(request): @@ -314,22 +316,24 @@ class OAuth2Authentication(BaseAuthentication): """ Authenticate the request, given the access token. """ + client = None # Authenticate the client - oauth2_client_form = oauth2_provider_forms.ClientAuthForm(request.REQUEST) - if not oauth2_client_form.is_valid(): - raise exceptions.AuthenticationFailed('Client could not be validated') - client = oauth2_client_form.cleaned_data.get('client') + if 'client_id' in request.REQUEST: + oauth2_client_form = oauth2_provider_forms.ClientAuthForm(request.REQUEST) + if not oauth2_client_form.is_valid(): + raise exceptions.AuthenticationFailed('Client could not be validated') + client = oauth2_client_form.cleaned_data.get('client') - # Retrieve the `OAuth2AccessToken` instance from the access_token - auth_backend = oauth2_provider_backends.AccessTokenBackend() - token = auth_backend.authenticate(access_token, client) - if token is None: + try: + token = oauth2_provider.models.AccessToken.objects.select_related('user') + if client is not None: + token = token.filter(client=client) + token = token.get(token=access_token, expires__gt=datetime.now()) + except oauth2_provider.models.AccessToken.DoesNotExist: raise exceptions.AuthenticationFailed('Invalid token') - user = token.user - - if not user.is_active: + if not token.user.is_active: msg = 'User inactive or deleted: %s' % user.username raise exceptions.AuthenticationFailed(msg) diff --git a/rest_framework/tests/authentication.py b/rest_framework/tests/authentication.py index b663ca48f..375b19bd2 100644 --- a/rest_framework/tests/authentication.py +++ b/rest_framework/tests/authentication.py @@ -516,6 +516,18 @@ class OAuth2Tests(TestCase): response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 200) + @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') + def test_get_form_passing_auth_without_client_params(self): + """ + Ensure GETing form over OAuth without client credentials + + Regression test for issue #759: + https://github.com/tomchristie/django-rest-framework/issues/759 + """ + auth = self._create_authorization_header() + response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) + self.assertEqual(response.status_code, 200) + @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') def test_post_form_passing_auth(self): """Ensure POSTing form over OAuth with correct credentials passes and does not require CSRF""" From 5f48b4a77e0a767694a32310a6368cd32b9a924c Mon Sep 17 00:00:00 2001 From: glic3rinu Date: Wed, 27 Mar 2013 22:43:41 +0100 Subject: [PATCH 34/50] Refactored urlize_quoted_links code, now based on Django 1.5 urlize --- rest_framework/templatetags/rest_framework.py | 79 +++++++++++-------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/rest_framework/templatetags/rest_framework.py b/rest_framework/templatetags/rest_framework.py index c21ddcd7b..50e485db6 100644 --- a/rest_framework/templatetags/rest_framework.py +++ b/rest_framework/templatetags/rest_framework.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals, absolute_import from django import template from django.core.urlresolvers import reverse, NoReverseMatch from django.http import QueryDict -from django.utils.html import escape +from django.utils.html import escape, smart_urlquote from django.utils.safestring import SafeData, mark_safe from rest_framework.compat import urlparse from rest_framework.compat import force_text @@ -112,22 +112,6 @@ def replace_query_param(url, key, val): class_re = re.compile(r'(?<=class=["\'])(.*)(?=["\'])') -# Bunch of stuff cloned from urlize -LEADING_PUNCTUATION = ['(', '<', '<', '"', "'"] -TRAILING_PUNCTUATION = ['.', ',', ')', '>', '\n', '>', '"', "'"] -DOTS = ['·', '*', '\xe2\x80\xa2', '•', '•', '•'] -unencoded_ampersands_re = re.compile(r'&(?!(\w+|#\d+);)') -word_split_re = re.compile(r'(\s+)') -punctuation_re = re.compile('^(?P(?:%s)*)(?P.*?)(?P(?:%s)*)$' % \ - ('|'.join([re.escape(x) for x in LEADING_PUNCTUATION]), - '|'.join([re.escape(x) for x in TRAILING_PUNCTUATION]))) -simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$') -link_target_attribute_re = re.compile(r'(]*?)target=[^\s>]+') -html_gunk_re = re.compile(r'(?:
|<\/i>|<\/b>|<\/em>|<\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE) -hard_coded_bullets_re = re.compile(r'((?:

(?:%s).*?[a-zA-Z].*?

\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL) -trailing_empty_content_re = re.compile(r'(?:

(?: |\s|
)*?

\s*)+\Z') - - # And the template tags themselves... @register.simple_tag @@ -195,15 +179,25 @@ def add_class(value, css_class): return value +# Bunch of stuff cloned from urlize +TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)'] +WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('<', '>'), + ('"', '"'), ("'", "'")] +word_split_re = re.compile(r'(\s+)') +simple_url_re = re.compile(r'^https?://\w', re.IGNORECASE) +simple_url_2_re = re.compile(r'^www\.|^(?!http)\w[^@]+\.(com|edu|gov|int|mil|net|org)$', re.IGNORECASE) +simple_email_re = re.compile(r'^\S+@\S+\.\S+$') + + @register.filter def urlize_quoted_links(text, trim_url_limit=None, nofollow=True, autoescape=True): """ Converts any URLs in text into clickable links. - Works on http://, https://, www. links and links ending in .org, .net or - .com. Links can have trailing punctuation (periods, commas, close-parens) - and leading punctuation (opening parens) and it'll still do the right - thing. + Works on http://, https://, www. links, and also on links ending in one of + the original seven gTLDs (.com, .edu, .gov, .int, .mil, .net, and .org). + Links can have trailing punctuation (periods, commas, close-parens) and + leading punctuation (opening parens) and it'll still do the right thing. If trim_url_limit is not None, the URLs in link text longer than this limit will truncated to trim_url_limit-3 characters and appended with an elipsis. @@ -216,24 +210,41 @@ def urlize_quoted_links(text, trim_url_limit=None, nofollow=True, autoescape=Tru trim_url = lambda x, limit=trim_url_limit: limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x safe_input = isinstance(text, SafeData) words = word_split_re.split(force_text(text)) - nofollow_attr = nofollow and ' rel="nofollow"' or '' for i, word in enumerate(words): match = None if '.' in word or '@' in word or ':' in word: - match = punctuation_re.match(word) - if match: - lead, middle, trail = match.groups() + # Deal with punctuation. + lead, middle, trail = '', word, '' + for punctuation in TRAILING_PUNCTUATION: + if middle.endswith(punctuation): + middle = middle[:-len(punctuation)] + trail = punctuation + trail + for opening, closing in WRAPPING_PUNCTUATION: + if middle.startswith(opening): + middle = middle[len(opening):] + lead = lead + opening + # Keep parentheses at the end only if they're balanced. + if (middle.endswith(closing) + and middle.count(closing) == middle.count(opening) + 1): + middle = middle[:-len(closing)] + trail = closing + trail + # Make URL we want to point to. url = None - if middle.startswith('http://') or middle.startswith('https://'): - url = middle - elif middle.startswith('www.') or ('@' not in middle and \ - middle and middle[0] in string.ascii_letters + string.digits and \ - (middle.endswith('.org') or middle.endswith('.net') or middle.endswith('.com'))): - url = 'http://%s' % middle - elif '@' in middle and not ':' in middle and simple_email_re.match(middle): - url = 'mailto:%s' % middle + nofollow_attr = ' rel="nofollow"' if nofollow else '' + if simple_url_re.match(middle): + url = smart_urlquote(middle) + elif simple_url_2_re.match(middle): + url = smart_urlquote('http://%s' % middle) + elif not ':' in middle and simple_email_re.match(middle): + local, domain = middle.rsplit('@', 1) + try: + domain = domain.encode('idna').decode('ascii') + except UnicodeError: + continue + url = 'mailto:%s@%s' % (local, domain) nofollow_attr = '' + # Make link. if url: trimmed = trim_url(middle) @@ -251,4 +262,4 @@ def urlize_quoted_links(text, trim_url_limit=None, nofollow=True, autoescape=Tru words[i] = mark_safe(word) elif autoescape: words[i] = escape(word) - return mark_safe(''.join(words)) + return ''.join(words) From 2c0363ddaec22ac54385f7e0c2e1401ed3ff0879 Mon Sep 17 00:00:00 2001 From: glic3rinu Date: Wed, 27 Mar 2013 22:58:11 +0100 Subject: [PATCH 35/50] Added quotes to TRAILING_PUNCTUATION used by urlize_quoted_links --- rest_framework/templatetags/rest_framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_framework/templatetags/rest_framework.py b/rest_framework/templatetags/rest_framework.py index 50e485db6..78a3a9a17 100644 --- a/rest_framework/templatetags/rest_framework.py +++ b/rest_framework/templatetags/rest_framework.py @@ -180,7 +180,7 @@ def add_class(value, css_class): # Bunch of stuff cloned from urlize -TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)'] +TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "'"] WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('<', '>'), ('"', '"'), ("'", "'")] word_split_re = re.compile(r'(\s+)') From b2cea84fae4f721e8eb6432b3d1bab1309e21a00 Mon Sep 17 00:00:00 2001 From: Fernando Rocha Date: Wed, 27 Mar 2013 19:00:36 -0300 Subject: [PATCH 36/50] Complete remove of client checks from oauth2 Signed-off-by: Fernando Rocha --- docs/api-guide/authentication.md | 2 +- rest_framework/authentication.py | 12 ++---------- rest_framework/tests/authentication.py | 9 --------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/docs/api-guide/authentication.md b/docs/api-guide/authentication.md index 541c65756..f1dd6f5fe 100644 --- a/docs/api-guide/authentication.md +++ b/docs/api-guide/authentication.md @@ -294,7 +294,7 @@ The only thing needed to make the `OAuth2Authentication` class work is to insert The command line to test the authentication looks like: - curl -H "Authorization: Bearer " http://localhost:8000/api/?client_id=YOUR_CLIENT_ID\&client_secret=YOUR_CLIENT_SECRET + curl -H "Authorization: Bearer " http://localhost:8000/api/ --- diff --git a/rest_framework/authentication.py b/rest_framework/authentication.py index f4626a2e3..145d42954 100644 --- a/rest_framework/authentication.py +++ b/rest_framework/authentication.py @@ -316,19 +316,11 @@ class OAuth2Authentication(BaseAuthentication): """ Authenticate the request, given the access token. """ - client = None - - # Authenticate the client - if 'client_id' in request.REQUEST: - oauth2_client_form = oauth2_provider_forms.ClientAuthForm(request.REQUEST) - if not oauth2_client_form.is_valid(): - raise exceptions.AuthenticationFailed('Client could not be validated') - client = oauth2_client_form.cleaned_data.get('client') try: token = oauth2_provider.models.AccessToken.objects.select_related('user') - if client is not None: - token = token.filter(client=client) + # TODO: Change to timezone aware datetime when oauth2_provider add + # support to it. token = token.get(token=access_token, expires__gt=datetime.now()) except oauth2_provider.models.AccessToken.DoesNotExist: raise exceptions.AuthenticationFailed('Invalid token') diff --git a/rest_framework/tests/authentication.py b/rest_framework/tests/authentication.py index 375b19bd2..629db4226 100644 --- a/rest_framework/tests/authentication.py +++ b/rest_framework/tests/authentication.py @@ -499,15 +499,6 @@ class OAuth2Tests(TestCase): response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_with_wrong_client_data_failing_auth(self): - """Ensure GETing form over OAuth with incorrect client credentials fails""" - auth = self._create_authorization_header() - params = self._client_credentials_params() - params['client_id'] += 'a' - response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') def test_get_form_passing_auth(self): """Ensure GETing form over OAuth with correct client credentials succeed""" From 8ec60a22e1c14792b7021ff9b4e940e16528788a Mon Sep 17 00:00:00 2001 From: Pierre Dulac Date: Thu, 28 Mar 2013 00:57:23 +0100 Subject: [PATCH 37/50] Remove client credentials from all OAuth 2 tests --- rest_framework/tests/authentication.py | 45 ++++++-------------------- 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/rest_framework/tests/authentication.py b/rest_framework/tests/authentication.py index 629db4226..8e6d3e51a 100644 --- a/rest_framework/tests/authentication.py +++ b/rest_framework/tests/authentication.py @@ -466,17 +466,13 @@ class OAuth2Tests(TestCase): def _create_authorization_header(self, token=None): return "Bearer {0}".format(token or self.access_token.token) - def _client_credentials_params(self): - return {'client_id': self.CLIENT_ID, 'client_secret': self.CLIENT_SECRET} - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') def test_get_form_with_wrong_authorization_header_token_type_failing(self): """Ensure that a wrong token type lead to the correct HTTP error status code""" auth = "Wrong token-type-obsviously" response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) - params = self._client_credentials_params() - response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') @@ -485,8 +481,7 @@ class OAuth2Tests(TestCase): auth = "Bearer wrong token format" response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) - params = self._client_credentials_params() - response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') @@ -495,27 +490,13 @@ class OAuth2Tests(TestCase): auth = "Bearer wrong-token" response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) - params = self._client_credentials_params() - response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 401) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') def test_get_form_passing_auth(self): """Ensure GETing form over OAuth with correct client credentials succeed""" auth = self._create_authorization_header() - params = self._client_credentials_params() - response = self.csrf_client.get('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_passing_auth_without_client_params(self): - """ - Ensure GETing form over OAuth without client credentials - - Regression test for issue #759: - https://github.com/tomchristie/django-rest-framework/issues/759 - """ - auth = self._create_authorization_header() response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 200) @@ -523,8 +504,7 @@ class OAuth2Tests(TestCase): def test_post_form_passing_auth(self): """Ensure POSTing form over OAuth with correct credentials passes and does not require CSRF""" auth = self._create_authorization_header() - params = self._client_credentials_params() - response = self.csrf_client.post('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 200) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') @@ -532,16 +512,14 @@ class OAuth2Tests(TestCase): """Ensure POSTing when there is no OAuth access token in db fails""" self.access_token.delete() auth = self._create_authorization_header() - params = self._client_credentials_params() - response = self.csrf_client.post('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') def test_post_form_with_refresh_token_failing_auth(self): """Ensure POSTing with refresh token instead of access token fails""" auth = self._create_authorization_header(token=self.refresh_token.token) - params = self._client_credentials_params() - response = self.csrf_client.post('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') @@ -550,8 +528,7 @@ class OAuth2Tests(TestCase): self.access_token.expires = datetime.datetime.now() - datetime.timedelta(seconds=10) # 10 seconds late self.access_token.save() auth = self._create_authorization_header() - params = self._client_credentials_params() - response = self.csrf_client.post('/oauth2-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) self.assertIn('Invalid token', response.content) @@ -562,10 +539,9 @@ class OAuth2Tests(TestCase): read_only_access_token.scope = oauth2_provider_scope.SCOPE_NAME_DICT['read'] read_only_access_token.save() auth = self._create_authorization_header(token=read_only_access_token.token) - params = self._client_credentials_params() - response = self.csrf_client.get('/oauth2-with-scope-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.get('/oauth2-with-scope-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 200) - response = self.csrf_client.post('/oauth2-with-scope-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.post('/oauth2-with-scope-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') @@ -575,6 +551,5 @@ class OAuth2Tests(TestCase): read_write_access_token.scope = oauth2_provider_scope.SCOPE_NAME_DICT['write'] read_write_access_token.save() auth = self._create_authorization_header(token=read_write_access_token.token) - params = self._client_credentials_params() - response = self.csrf_client.post('/oauth2-with-scope-test/', params, HTTP_AUTHORIZATION=auth) + response = self.csrf_client.post('/oauth2-with-scope-test/', HTTP_AUTHORIZATION=auth) self.assertEqual(response.status_code, 200) From fa61b2b2f10bf07e3cb87ca947ce7f0ca51a2ede Mon Sep 17 00:00:00 2001 From: Pierre Dulac Date: Thu, 28 Mar 2013 01:05:51 +0100 Subject: [PATCH 38/50] Remove oauth2-provider backends reference from compat.py --- rest_framework/compat.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rest_framework/compat.py b/rest_framework/compat.py index 7b2ef7384..c3e423e89 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -445,14 +445,12 @@ except ImportError: # OAuth 2 support is optional try: import provider.oauth2 as oauth2_provider - from provider.oauth2 import backends as oauth2_provider_backends from provider.oauth2 import models as oauth2_provider_models from provider.oauth2 import forms as oauth2_provider_forms from provider import scope as oauth2_provider_scope from provider import constants as oauth2_constants except ImportError: oauth2_provider = None - oauth2_provider_backends = None oauth2_provider_models = None oauth2_provider_forms = None oauth2_provider_scope = None From b10663e02408404844aca4b362aa24df816aca98 Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Wed, 27 Mar 2013 17:55:36 -0700 Subject: [PATCH 39/50] Fixed DjangoFilterBackend not returning a query set. Fixed bug unveiled in #682. Signed-off-by: Kevin Stone --- rest_framework/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_framework/filters.py b/rest_framework/filters.py index 6fea46faf..413fa0d29 100644 --- a/rest_framework/filters.py +++ b/rest_framework/filters.py @@ -55,6 +55,6 @@ class DjangoFilterBackend(BaseFilterBackend): filter_class = self.get_filter_class(view) if filter_class: - return filter_class(request.QUERY_PARAMS, queryset=queryset) + return filter_class(request.QUERY_PARAMS, queryset=queryset).qs return queryset From d4df617f8c1980c1d5f1b91a6b9928185c4c4dce Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Wed, 27 Mar 2013 18:29:50 -0700 Subject: [PATCH 40/50] Added unit test for failing DjangoFilterBackend on SingleObjectMixin that was resolved in b10663e02408404844aca4b362aa24df816aca98 Signed-off-by: Kevin Stone --- rest_framework/tests/filterset.py | 75 ++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/rest_framework/tests/filterset.py b/rest_framework/tests/filterset.py index 238da56e5..1a71558c0 100644 --- a/rest_framework/tests/filterset.py +++ b/rest_framework/tests/filterset.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals import datetime from decimal import Decimal +from django.core.urlresolvers import reverse from django.test import TestCase from django.test.client import RequestFactory from django.utils import unittest from rest_framework import generics, status, filters -from rest_framework.compat import django_filters +from rest_framework.compat import django_filters, patterns, url from rest_framework.tests.models import FilterableItem, BasicModel factory = RequestFactory() @@ -46,12 +47,21 @@ if django_filters: filter_class = MisconfiguredFilter filter_backend = filters.DjangoFilterBackend + class FilterClassDetailView(generics.RetrieveAPIView): + model = FilterableItem + filter_class = SeveralFieldsFilter + filter_backend = filters.DjangoFilterBackend -class IntegrationTestFiltering(TestCase): - """ - Integration tests for filtered list views. - """ + urlpatterns = patterns('', + url(r'^(?P\d+)/$', FilterClassDetailView.as_view(), name='detail-view'), + url(r'^$', FilterClassRootView.as_view(), name='root-view'), + ) + +class CommonFilteringTestCase(TestCase): + def _serialize_object(self, obj): + return {'id': obj.id, 'text': obj.text, 'decimal': obj.decimal, 'date': obj.date} + def setUp(self): """ Create 10 FilterableItem instances. @@ -65,10 +75,16 @@ class IntegrationTestFiltering(TestCase): self.objects = FilterableItem.objects self.data = [ - {'id': obj.id, 'text': obj.text, 'decimal': obj.decimal, 'date': obj.date} + self._serialize_object(obj) for obj in self.objects.all() ] + +class IntegrationTestFiltering(CommonFilteringTestCase): + """ + Integration tests for filtered list views. + """ + @unittest.skipUnless(django_filters, 'django-filters not installed') def test_get_filtered_fields_root_view(self): """ @@ -167,3 +183,50 @@ class IntegrationTestFiltering(TestCase): request = factory.get('/?integer=%s' % search_integer) response = view(request).render() self.assertEqual(response.status_code, status.HTTP_200_OK) + + +class IntegrationTestDetailFiltering(CommonFilteringTestCase): + """ + Integration tests for filtered detail views. + """ + urls = 'rest_framework.tests.filterset' + + def _get_url(self, item): + return reverse('detail-view', kwargs=dict(pk=item.pk)) + + @unittest.skipUnless(django_filters, 'django-filters not installed') + def test_get_filtered_detail_view(self): + """ + GET requests to filtered RetrieveAPIView that have a filter_class set + should return filtered results. + """ + item = self.objects.all()[0] + data = self._serialize_object(item) + + # Basic test with no filter. + response = self.client.get(self._get_url(item)) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, data) + + # Tests that the decimal filter set that should fail. + search_decimal = Decimal('4.25') + high_item = self.objects.filter(decimal__gt=search_decimal)[0] + response = self.client.get('{url}?decimal={param}'.format(url=self._get_url(high_item), param=search_decimal)) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + # Tests that the decimal filter set that should succeed. + search_decimal = Decimal('4.25') + low_item = self.objects.filter(decimal__lt=search_decimal)[0] + low_item_data = self._serialize_object(low_item) + response = self.client.get('{url}?decimal={param}'.format(url=self._get_url(low_item), param=search_decimal)) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, low_item_data) + + # Tests that multiple filters works. + search_decimal = Decimal('5.25') + search_date = datetime.date(2012, 10, 2) + valid_item = self.objects.filter(decimal__lt=search_decimal, date__gt=search_date)[0] + valid_item_data = self._serialize_object(valid_item) + response = self.client.get('{url}?decimal={decimal}&date={date}'.format(url=self._get_url(valid_item), decimal=search_decimal, date=search_date)) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, valid_item_data) From 3774ba3ed2af918563eb6ed945cc13aa7fa2345a Mon Sep 17 00:00:00 2001 From: glic3rinu Date: Thu, 28 Mar 2013 12:01:08 +0100 Subject: [PATCH 41/50] Added force_text to compat --- rest_framework/compat.py | 31 +++++++++++++++++++ rest_framework/templatetags/rest_framework.py | 3 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/rest_framework/compat.py b/rest_framework/compat.py index 7b2ef7384..f0bb9c089 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -395,6 +395,37 @@ except ImportError: kw = dict((k, int(v)) for k, v in kw.iteritems() if v is not None) return datetime.datetime(**kw) + +# smart_urlquote is new on Django 1.4 +try: + from django.utils.html import smart_urlquote +except ImportError: + try: + from urllib.parse import quote, urlsplit, urlunsplit + except ImportError: # Python 2 + from urllib import quote + from urlparse import urlsplit, urlunsplit + + def smart_urlquote(url): + "Quotes a URL if it isn't already quoted." + # Handle IDN before quoting. + scheme, netloc, path, query, fragment = urlsplit(url) + try: + netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE + except UnicodeError: # invalid domain part + pass + else: + url = urlunsplit((scheme, netloc, path, query, fragment)) + + # An URL is considered unquoted if it contains no % characters or + # contains a % not followed by two hexadecimal digits. See #9655. + if '%' not in url or unquoted_percents_re.search(url): + # See http://bugs.python.org/issue2637 + url = quote(force_bytes(url), safe=b'!*\'();:@&=+$,/?#[]~') + + return force_text(url) + + # Markdown is optional try: import markdown diff --git a/rest_framework/templatetags/rest_framework.py b/rest_framework/templatetags/rest_framework.py index 78a3a9a17..33bae241e 100644 --- a/rest_framework/templatetags/rest_framework.py +++ b/rest_framework/templatetags/rest_framework.py @@ -2,11 +2,12 @@ from __future__ import unicode_literals, absolute_import from django import template from django.core.urlresolvers import reverse, NoReverseMatch from django.http import QueryDict -from django.utils.html import escape, smart_urlquote +from django.utils.html import escape from django.utils.safestring import SafeData, mark_safe from rest_framework.compat import urlparse from rest_framework.compat import force_text from rest_framework.compat import six +from rest_framework.compat import smart_urlquote import re import string From 9c32f048b51ec6852236363932f0ab0dcc7473ac Mon Sep 17 00:00:00 2001 From: glic3rinu Date: Thu, 28 Mar 2013 12:01:47 +0100 Subject: [PATCH 42/50] Cleaned imports on templatetags/rest_framework module --- rest_framework/templatetags/rest_framework.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rest_framework/templatetags/rest_framework.py b/rest_framework/templatetags/rest_framework.py index 33bae241e..b6ab2de31 100644 --- a/rest_framework/templatetags/rest_framework.py +++ b/rest_framework/templatetags/rest_framework.py @@ -4,12 +4,8 @@ from django.core.urlresolvers import reverse, NoReverseMatch from django.http import QueryDict from django.utils.html import escape from django.utils.safestring import SafeData, mark_safe -from rest_framework.compat import urlparse -from rest_framework.compat import force_text -from rest_framework.compat import six -from rest_framework.compat import smart_urlquote -import re -import string +from rest_framework.compat import urlparse, force_text, six, smart_urlquote +import re, string register = template.Library() From fb105d138cdcb178ed08c6616c0c25f4a03fb2e1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 28 Mar 2013 14:54:25 +0000 Subject: [PATCH 43/50] Minor tweak --- docs/topics/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/contributing.md b/docs/topics/contributing.md index a13f4461e..1d1fe8920 100644 --- a/docs/topics/contributing.md +++ b/docs/topics/contributing.md @@ -18,7 +18,7 @@ When answering questions make sure to help future contributors find their way ar # Issues -Usage questions should be directed to the [discussion group][google-group]. Feature requests, bug reports and other issues should be raised on the GitHub [issue tracker][issues]. +It's really helpful if you make sure you address issues to the correct channel. Usage questions should be directed to the [discussion group][google-group]. Feature requests, bug reports and other issues should be raised on the GitHub [issue tracker][issues]. Some tips on good issue reporting: From d243538547982781635e01d9b6e74afbbd628e16 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 28 Mar 2013 14:54:42 +0000 Subject: [PATCH 44/50] Note on using curl with token auth --- docs/api-guide/authentication.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/api-guide/authentication.md b/docs/api-guide/authentication.md index 541c65756..757b8673b 100644 --- a/docs/api-guide/authentication.md +++ b/docs/api-guide/authentication.md @@ -119,6 +119,8 @@ To use the `TokenAuthentication` scheme, include `rest_framework.authtoken` in y ... 'rest_framework.authtoken' ) + +Make sure to run `manage.py syncdb` after changing your settings. You'll also need to create tokens for your users. @@ -140,6 +142,10 @@ Unauthenticated responses that are denied permission will result in an `HTTP 401 WWW-Authenticate: Token +The `curl` command line tool may be useful for testing token authenticated APIs. For example: + + curl -X GET http://127.0.0.1:8000/api/example/ -H 'Authorization: Token 9944b09199c62bcf9418ad846dd0e4bbdfc6ee4b' + --- **Note:** If you use `TokenAuthentication` in production you must ensure that your API is only available over `https` only. From 69cbafc64f65a23b4ed4c652a8965873a18929a0 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 28 Mar 2013 15:58:53 +0000 Subject: [PATCH 45/50] Add search and next/prev --- docs/css/default.css | 11 +++++++ docs/template.html | 31 ++++++++++++++++++ mkdocs.py | 75 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/docs/css/default.css b/docs/css/default.css index c160b63d1..173d70e0d 100644 --- a/docs/css/default.css +++ b/docs/css/default.css @@ -277,3 +277,14 @@ footer a { footer a:hover { color: gray; } + +.btn-inverse { + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#606060), to(#404040)) !important; + background-image: -webkit-linear-gradient(top, #606060, #404040) !important; +} + +.modal-open .modal,.btn:focus{outline:none;} + +@media (max-width: 650px) { + .repo-link.btn-inverse {display: none;} +} diff --git a/docs/template.html b/docs/template.html index 3e0f29aa0..7e9297627 100644 --- a/docs/template.html +++ b/docs/template.html @@ -41,6 +41,9 @@