From 0081d744b9f530b2418d1e82d7ad94a2ebc31c9c Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Sat, 23 Mar 2013 14:18:11 +0100 Subject: [PATCH 01/13] Added tests for issue 747 in serializer.py --- rest_framework/tests/serializer.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rest_framework/tests/serializer.py b/rest_framework/tests/serializer.py index 05217f35a..0386ca76e 100644 --- a/rest_framework/tests/serializer.py +++ b/rest_framework/tests/serializer.py @@ -1082,3 +1082,32 @@ class DeserializeListTestCase(TestCase): self.assertFalse(serializer.is_valid()) expected = [{}, {'email': ['This field is required.']}, {}] self.assertEqual(serializer.errors, expected) + + +# test for issue 747 + +class LazyStringModel(object): + def __init__(self, lazystring): + self.lazystring = lazystring + + +class LazyStringSerializer(serializers.Serializer): + lazystring = serializers.Field() + + def restore_object(self, attrs, instance=None): + if instance is not None: + instance.lazystring = attrs.get('lazystring', instance.lazystring) + return instance + return Comment(**attrs) + + +class LazyStringsTestCase(TestCase): + + def setUp(self): + from django.utils.translation import ugettext_lazy as _ + + self.model = LazyStringModel(lazystring=_("lazystring")) + + def test_lazy_strings_are_translated(self): + serializer = LazyStringSerializer(self.model) + self.assertEqual(type(serializer.data['lazystring']), type("lazystring")) From b5640bb77843c50f42a649982b9b9592113c6f59 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Sat, 23 Mar 2013 14:18:55 +0100 Subject: [PATCH 02/13] Forcing translations of lazy translatable strings in Field to_native method --- rest_framework/fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest_framework/fields.py b/rest_framework/fields.py index f3496b53e..09f076ab7 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -18,7 +18,7 @@ from rest_framework import ISO_8601 from rest_framework.compat import timezone, parse_date, parse_datetime, parse_time from rest_framework.compat import BytesIO from rest_framework.compat import six -from rest_framework.compat import smart_text +from rest_framework.compat import smart_text, force_text from rest_framework.settings import api_settings @@ -165,7 +165,7 @@ class Field(object): return [self.to_native(item) for item in value] elif isinstance(value, dict): return dict(map(self.to_native, (k, v)) for k, v in value.items()) - return smart_text(value) + return force_text(value) def attributes(self): """ From ef383d969c440a20fdf25748de590e07cb1abfda Mon Sep 17 00:00:00 2001 From: Ryan Kaskel Date: Sat, 18 May 2013 14:31:29 +0100 Subject: [PATCH 03/13] Clean up test case. --- rest_framework/tests/serializer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rest_framework/tests/serializer.py b/rest_framework/tests/serializer.py index d978dc872..e999b624b 100644 --- a/rest_framework/tests/serializer.py +++ b/rest_framework/tests/serializer.py @@ -1,8 +1,9 @@ from __future__ import unicode_literals from django.db import models from django.db.models.fields import BLANK_CHOICE_DASH -from django.utils.datastructures import MultiValueDict from django.test import TestCase +from django.utils.datastructures import MultiValueDict +from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from rest_framework.tests.models import (HasPositiveIntegerAsChoice, Album, ActionItem, Anchor, BasicModel, BlankFieldModel, BlogPost, BlogPostComment, Book, CallableDefaultValueModel, DefaultValueModel, @@ -1246,6 +1247,7 @@ class DeserializeListTestCase(TestCase): # test for issue 747 + class LazyStringModel(object): def __init__(self, lazystring): self.lazystring = lazystring @@ -1258,16 +1260,14 @@ class LazyStringSerializer(serializers.Serializer): if instance is not None: instance.lazystring = attrs.get('lazystring', instance.lazystring) return instance - return Comment(**attrs) + return LazyStringModel(**attrs) class LazyStringsTestCase(TestCase): - def setUp(self): - from django.utils.translation import ugettext_lazy as _ - - self.model = LazyStringModel(lazystring=_("lazystring")) + self.model = LazyStringModel(lazystring=_('lazystring')) def test_lazy_strings_are_translated(self): serializer = LazyStringSerializer(self.model) - self.assertEqual(type(serializer.data['lazystring']), type("lazystring")) + self.assertEqual(type(serializer.data['lazystring']), + type('lazystring')) From 0c81d04170da19e5bfb332641d92d9c3346598f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20Gro=C3=9F?= Date: Sat, 18 May 2013 17:04:20 +0200 Subject: [PATCH 04/13] Add min_value=0 to autogenerated Pos..IntFields --- rest_framework/serializers.py | 4 ++ rest_framework/tests/serializer.py | 73 ++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index ff5eb8732..d0313513b 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -739,6 +739,10 @@ class ModelSerializer(Serializer): if issubclass(model_field.__class__, models.TextField): kwargs['widget'] = widgets.Textarea + if issubclass(model_field.__class__, models.PositiveIntegerField) or\ + issubclass(model_field.__class__, models.PositiveSmallIntegerField): + kwargs['min_value'] = 0 + # TODO: TypedChoiceField? if model_field.flatchoices: # This ModelField contains choices kwargs['choices'] = model_field.flatchoices diff --git a/rest_framework/tests/serializer.py b/rest_framework/tests/serializer.py index 4f188c3e4..220a581ab 100644 --- a/rest_framework/tests/serializer.py +++ b/rest_framework/tests/serializer.py @@ -1402,3 +1402,76 @@ class AttributeMappingOnAutogeneratedFieldsTests(TestCase): def test_url_field(self): self.field_test('url_field') + + +class DefaultValuesOnAutogeneratedFieldsTests(TestCase): + + def setUp(self): + class DVOAFModel(RESTFrameworkModel): + positive_integer_field = models.PositiveIntegerField(blank=True) + positive_small_integer_field = models.PositiveSmallIntegerField(blank=True) + email_field = models.EmailField(blank=True) + file_field = models.FileField(blank=True) + image_field = models.ImageField(blank=True) + slug_field = models.SlugField(blank=True) + url_field = models.URLField(blank=True) + + class DVOAFSerializer(serializers.ModelSerializer): + class Meta: + model = DVOAFModel + + self.serializer_class = DVOAFSerializer + self.fields_attributes = { + 'positive_integer_field': [ + ('min_value', 0), + ], + 'positive_small_integer_field': [ + ('min_value', 0), + ], + 'email_field': [ + ('max_length', 75), + ], + 'file_field': [ + ('max_length', 100), + ], + 'image_field': [ + ('max_length', 100), + ], + 'slug_field': [ + ('max_length', 50), + ], + 'url_field': [ + ('max_length', 200), + ], + } + + def field_test(self, field): + serializer = self.serializer_class(data={}) + self.assertEqual(serializer.is_valid(), True) + + for attribute in self.fields_attributes[field]: + self.assertEqual( + getattr(serializer.fields[field], attribute[0]), + attribute[1] + ) + + def test_positive_integer_field(self): + self.field_test('positive_integer_field') + + def test_positive_small_integer_field(self): + self.field_test('positive_small_integer_field') + + def test_email_field(self): + self.field_test('email_field') + + def test_file_field(self): + self.field_test('file_field') + + def test_image_field(self): + self.field_test('image_field') + + def test_slug_field(self): + self.field_test('slug_field') + + def test_url_field(self): + self.field_test('url_field') From c69f960066dfdef2163a53205f27c2b9e457068f Mon Sep 17 00:00:00 2001 From: Pablo Recio Date: Sat, 18 May 2013 16:30:40 +0200 Subject: [PATCH 05/13] Adding a class attribute into JSONRenderer for ensuring ascii, and using it consistently --- rest_framework/renderers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index 8361cd409..d55a3e0ea 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -49,6 +49,7 @@ class JSONRenderer(BaseRenderer): media_type = 'application/json' format = 'json' encoder_class = encoders.JSONEncoder + ensure_ascii = True def render(self, data, accepted_media_type=None, renderer_context=None): """ @@ -72,7 +73,7 @@ class JSONRenderer(BaseRenderer): except (ValueError, TypeError): indent = None - return json.dumps(data, cls=self.encoder_class, indent=indent) + return json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii) class JSONPRenderer(JSONRenderer): From 97f034e3d65068f7bd1e982e6fd251222a4feea1 Mon Sep 17 00:00:00 2001 From: Pablo Recio Date: Sat, 18 May 2013 16:31:12 +0200 Subject: [PATCH 06/13] Adds UnicodeJSONRenderer which doesn't ensure ascii --- rest_framework/renderers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index d55a3e0ea..12e1107fa 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -76,6 +76,10 @@ class JSONRenderer(BaseRenderer): return json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii) +class UnicodeJSONRenderer(JSONRenderer): + ensure_ascii = False + + class JSONPRenderer(JSONRenderer): """ Renderer which serializes to json, From 6dbbbc16da740fb27f9a5a390fb61400e9f6ff64 Mon Sep 17 00:00:00 2001 From: Pablo Recio Date: Sat, 18 May 2013 16:32:53 +0200 Subject: [PATCH 07/13] Better checking if the content can be printable in the BrowsableAPI --- rest_framework/renderers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index 12e1107fa..4345a313b 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -325,7 +325,7 @@ class BrowsableAPIRenderer(BaseRenderer): renderer_context['indent'] = 4 content = renderer.render(data, accepted_media_type, renderer_context) - if not all(char in string.printable for char in content): + if not isinstance(content, six.text_type): return '[%d bytes of binary content]' return content From 71e29644a2950d8a82cb26f6a3e39fb76faf9707 Mon Sep 17 00:00:00 2001 From: Pablo Recio Date: Sat, 18 May 2013 16:50:09 +0200 Subject: [PATCH 08/13] Adds new renderer into the documentation --- docs/api-guide/renderers.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/api-guide/renderers.md b/docs/api-guide/renderers.md index b9a9fd7a3..1661ceecf 100644 --- a/docs/api-guide/renderers.md +++ b/docs/api-guide/renderers.md @@ -67,7 +67,7 @@ If your API includes views that can serve both regular webpages and API response ## JSONRenderer -Renders the request data into `JSON`. +Renders the request data into `JSON` enforcing ASCII encoding The client may additionally include an `'indent'` media type parameter, in which case the returned `JSON` will be indented. For example `Accept: application/json; indent=4`. @@ -75,6 +75,10 @@ The client may additionally include an `'indent'` media type parameter, in which **.format**: `'.json'` +## UnicodeJSONRenderer + +Same as `JSONRenderer` but doesn't enforce ASCII encoding + ## JSONPRenderer Renders the request data into `JSONP`. The `JSONP` media type provides a mechanism of allowing cross-domain AJAX requests, by wrapping a `JSON` response in a javascript callback. From b9b22976125fffa4552da695183bf75fbaf0b927 Mon Sep 17 00:00:00 2001 From: Pablo Recio Date: Sat, 18 May 2013 17:06:25 +0200 Subject: [PATCH 09/13] Tests proper encoding in JSONRenderer and UnicodeJSONRenderer --- rest_framework/tests/renderers.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/rest_framework/tests/renderers.py b/rest_framework/tests/renderers.py index 40bac9cb3..739f91841 100644 --- a/rest_framework/tests/renderers.py +++ b/rest_framework/tests/renderers.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from decimal import Decimal from django.core.cache import cache from django.test import TestCase @@ -8,7 +9,7 @@ from rest_framework.compat import yaml, etree, patterns, url, include from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \ - XMLRenderer, JSONPRenderer, BrowsableAPIRenderer + XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer from rest_framework.parsers import YAMLParser, XMLParser from rest_framework.settings import api_settings from rest_framework.compat import StringIO @@ -254,6 +255,23 @@ class JSONRendererTests(TestCase): content = renderer.render(obj, 'application/json; indent=2') self.assertEqual(strip_trailing_whitespace(content), _indented_repr) + def test_check_ascii(self): + obj = {'countries': ['United Kingdom', 'France', 'España']} + renderer = JSONRenderer() + content = renderer.render(obj, 'application/json') + self.assertEqual(content, '{"countries": ["United Kingdom", "France", "Espa\\u00f1a"]}') + + +class UnicodeJSONRendererTests(TestCase): + """ + Tests specific for the Unicode JSON Renderer + """ + def test_proper_encoding(self): + obj = {'countries': ['United Kingdom', 'France', 'España']} + renderer = UnicodeJSONRenderer() + content = renderer.render(obj, 'application/json') + self.assertEqual(content, '{"countries": ["United Kingdom", "France", "España"]}') + class JSONPRendererTests(TestCase): """ From 025c40e7d5efbba0c3e65ab3737adef988995a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20Gro=C3=9F?= Date: Sat, 18 May 2013 17:16:29 +0200 Subject: [PATCH 10/13] Fix order for applying min_value to fields --- rest_framework/serializers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index d0313513b..943fba6ba 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -739,15 +739,16 @@ class ModelSerializer(Serializer): if issubclass(model_field.__class__, models.TextField): kwargs['widget'] = widgets.Textarea - if issubclass(model_field.__class__, models.PositiveIntegerField) or\ - issubclass(model_field.__class__, models.PositiveSmallIntegerField): - kwargs['min_value'] = 0 - # TODO: TypedChoiceField? if model_field.flatchoices: # This ModelField contains choices kwargs['choices'] = model_field.flatchoices return ChoiceField(**kwargs) + # put this below the ChoiceField because min_value isn't a valid initializer + if issubclass(model_field.__class__, models.PositiveIntegerField) or\ + issubclass(model_field.__class__, models.PositiveSmallIntegerField): + kwargs['min_value'] = 0 + attribute_dict = { models.CharField: ['max_length'], models.CommaSeparatedIntegerField: ['max_length'], From 10e451a85a034d1158148f54e98147e81454c2ed Mon Sep 17 00:00:00 2001 From: Ryan Kaskel Date: Sat, 18 May 2013 16:21:18 +0100 Subject: [PATCH 11/13] Handle Python 3 strings and lazy strings. --- rest_framework/fields.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/rest_framework/fields.py b/rest_framework/fields.py index 2ab603cfb..c3ac62813 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -19,6 +19,7 @@ from django.db.models.fields import BLANK_CHOICE_DASH from django import forms from django.forms import widgets from django.utils.encoding import is_protected_type +from django.utils.functional import Promise from django.utils.translation import ugettext_lazy as _ from django.utils.datastructures import SortedDict @@ -45,6 +46,15 @@ def is_simple_callable(obj): len_defaults = len(defaults) if defaults else 0 return len_args <= len_defaults +if six.PY3: + def is_non_str_iterable(obj): + if (isinstance(obj, str) or + (isinstance(obj, Promise) and obj._delegate_text)): + return False + return hasattr(obj, '__iter__') +else: + def is_non_str_iterable(obj): + return hasattr(obj, '__iter__') def get_component(obj, attr_name): """ @@ -169,7 +179,8 @@ class Field(object): if is_protected_type(value): return value - elif hasattr(value, '__iter__') and not isinstance(value, (dict, six.string_types)): + elif (is_non_str_iterable(value) and + not isinstance(value, (dict, six.string_types))): return [self.to_native(item) for item in value] elif isinstance(value, dict): # Make sure we preserve field ordering, if it exists From 579f77ceaa03a216a7a635c3d3a4d83b0e5868f8 Mon Sep 17 00:00:00 2001 From: Ryan Kaskel Date: Sat, 18 May 2013 17:04:17 +0100 Subject: [PATCH 12/13] Move function to compat. --- rest_framework/compat.py | 13 +++++++++++++ rest_framework/fields.py | 13 +------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/rest_framework/compat.py b/rest_framework/compat.py index cd39f5445..76dc00526 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -495,3 +495,16 @@ except ImportError: oauth2_provider_forms = None oauth2_provider_scope = None oauth2_constants = None + +# Handle lazy strings +from django.utils.functional import Promise + +if six.PY3: + def is_non_str_iterable(obj): + if (isinstance(obj, str) or + (isinstance(obj, Promise) and obj._delegate_text)): + return False + return hasattr(obj, '__iter__') +else: + def is_non_str_iterable(obj): + return hasattr(obj, '__iter__') diff --git a/rest_framework/fields.py b/rest_framework/fields.py index d5cf30e43..b5f99823a 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -19,7 +19,6 @@ from django.db.models.fields import BLANK_CHOICE_DASH from django import forms from django.forms import widgets from django.utils.encoding import is_protected_type -from django.utils.functional import Promise from django.utils.translation import ugettext_lazy as _ from django.utils.datastructures import SortedDict @@ -27,7 +26,7 @@ from rest_framework import ISO_8601 from rest_framework.compat import timezone, parse_date, parse_datetime, parse_time from rest_framework.compat import BytesIO from rest_framework.compat import six -from rest_framework.compat import smart_text, force_text +from rest_framework.compat import smart_text, force_text, is_non_str_iterable from rest_framework.settings import api_settings @@ -46,16 +45,6 @@ def is_simple_callable(obj): len_defaults = len(defaults) if defaults else 0 return len_args <= len_defaults -if six.PY3: - def is_non_str_iterable(obj): - if (isinstance(obj, str) or - (isinstance(obj, Promise) and obj._delegate_text)): - return False - return hasattr(obj, '__iter__') -else: - def is_non_str_iterable(obj): - return hasattr(obj, '__iter__') - def get_component(obj, attr_name): """ Given an object, and an attribute name, From 6af61a19b78b0ebb9d8403e96c6e4a77c675b141 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 18 May 2013 18:07:43 +0200 Subject: [PATCH 13/13] Added @matteosuppo for work on #865. Thanks! --- docs/topics/credits.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/credits.md b/docs/topics/credits.md index d805c0c1d..0e206de82 100644 --- a/docs/topics/credits.md +++ b/docs/topics/credits.md @@ -130,6 +130,7 @@ The following people have helped make REST framework great. * Òscar Vilaplana - [grimborg] * Ryan Kaskel - [ryankask] * Andy McKay - [andymckay] +* Matteo Suppo - [matteosuppo] Many thanks to everyone who's contributed to the project. @@ -296,3 +297,4 @@ You can also contact [@_tomchristie][twitter] directly on twitter. [grimborg]: https://github.com/grimborg [ryankask]: https://github.com/ryankask [andymckay]: https://github.com/andymckay +[matteosuppo]: https://github.com/matteosuppo