Merge remote-tracking branch 'origin/master' into 2.4.0

Conflicts:
	.travis.yml
	rest_framework/serializers.py
	rest_framework/tests/test_authentication.py

.travis.yml

@@ -7,10 +7,10 @@ python:
   - "3.3"
 
 env:
-  - DJANGO="https://www.djangoproject.com/download/1.7b1/tarball/"
+  - DJANGO="https://www.djangoproject.com/download/1.7b2/tarball/"
-  - DJANGO="django==1.6.2"
+  - DJANGO="django==1.6.3"
-  - DJANGO="django==1.5.5"
+  - DJANGO="django==1.5.6"
-  - DJANGO="django==1.4.10"
+  - DJANGO="django==1.4.11"
 
 install:
   - pip install $DJANGO
@@ -22,7 +22,7 @@ install:
   - "if [[ ${DJANGO::11} == 'django==1.3' ]]; then pip install django-filter==0.5.4; fi"
   - "if [[ ${DJANGO::11} != 'django==1.3' ]]; then pip install django-filter==0.7; fi"
   - "if [[ ${TRAVIS_PYTHON_VERSION::1} == '3' ]]; then pip install -e git+https://github.com/linovia/django-guardian.git@feature/django_1_7#egg=django-guardian-1.2.0; fi"
-  - "if [[ ${DJANGO} == 'https://www.djangoproject.com/download/1.7b1/tarball/' ]]; then pip install -e git+https://github.com/linovia/django-guardian.git@feature/django_1_7#egg=django-guardian-1.2.0; fi"
+  - "if [[ ${DJANGO} == 'https://www.djangoproject.com/download/1.7b2/tarball/' ]]; then pip install -e git+https://github.com/linovia/django-guardian.git@feature/django_1_7#egg=django-guardian-1.2.0; fi"
   - export PYTHONPATH=.
 
 script:
@@ -31,8 +31,8 @@ script:
 matrix:
   exclude:
     - python: "2.6"
-      env: DJANGO="https://www.djangoproject.com/download/1.7b1/tarball/"
+      env: DJANGO="https://www.djangoproject.com/download/1.7b2/tarball/"
     - python: "3.2"
-      env: DJANGO="django==1.4.10"
+      env: DJANGO="django==1.4.11"
     - python: "3.3"
-      env: DJANGO="django==1.4.10"
+      env: DJANGO="django==1.4.11"

docs/api-guide/generic-views.md

@@ -70,7 +70,7 @@ The following attributes control the basic view behavior.
 
 **Shortcuts**:
 
-* `model` - This shortcut may be used instead of setting either (or both) of the `queryset`/`serializer_class` attributes, although using the explicit style is generally preferred.  If used instead of `serializer_class`, then then `DEFAULT_MODEL_SERIALIZER_CLASS` setting will determine the base serializer class.  Note that `model` is only ever used for generating a default queryset or serializer class - the `queryset` and `serializer_class` attributes are always preferred if provided.
+* `model` - This shortcut may be used instead of setting either (or both) of the `queryset`/`serializer_class` attributes, although using the explicit style is generally preferred.  If used instead of `serializer_class`, then `DEFAULT_MODEL_SERIALIZER_CLASS` setting will determine the base serializer class.  Note that `model` is only ever used for generating a default queryset or serializer class - the `queryset` and `serializer_class` attributes are always preferred if provided.
 
 **Pagination**:
 

docs/api-guide/pagination.md

@@ -103,6 +103,7 @@ You can also set the pagination style on a per-view basis, using the `ListAPIVie
         max_paginate_by = 100
 
 Note that using a `paginate_by` value of `None` will turn off pagination for the view.
+Note if you use the `PAGINATE_BY_PARAM` settings, you also have to set the `paginate_by_param` attribute in your view to `None` in order to turn off pagination for those requests that contain the `paginate_by_param` parameter.
 
 For more complex requirements such as serialization that differs depending on the requested media type you can override the `.get_paginate_by()` and `.get_pagination_serializer_class()` methods.
 
@@ -157,4 +158,4 @@ The [`DRF-extensions` package][drf-extensions] includes a [`PaginateByMaxMixin`
 
 [cite]: https://docs.djangoproject.com/en/dev/topics/pagination/
 [drf-extensions]: http://chibisov.github.io/drf-extensions/docs/
-[paginate-by-max-mixin]: http://chibisov.github.io/drf-extensions/docs/#paginatebymaxmixin
+[paginate-by-max-mixin]: http://chibisov.github.io/drf-extensions/docs/#paginatebymaxmixin

docs/api-guide/permissions.md

@@ -56,7 +56,7 @@ You can also set the authentication policy on a per-view, or per-viewset basis,
 using the `APIView` class based views.
 
     from rest_framework.permissions import IsAuthenticated
-	from rest_framework.responses import Response
+	from rest_framework.response import Response
 	from rest_framework.views import APIView
 
     class ExampleView(APIView):

docs/api-guide/renderers.md

@@ -138,6 +138,26 @@ Renders the request data into `YAML`.
 
 Requires the `pyyaml` package to be installed.
 
+Note that non-ascii characters will be rendered using `\uXXXX` character escape.  For example:
+
+    unicode black star: "\u2605"
+
+**.media_type**: `application/yaml`
+
+**.format**: `'.yaml'`
+
+**.charset**: `utf-8`
+
+## UnicodeYAMLRenderer
+
+Renders the request data into `YAML`. 
+
+Requires the `pyyaml` package to be installed.
+
+Note that non-ascii characters will not be character escaped.  For example:
+
+    unicode black star: ★
+
 **.media_type**: `application/yaml`
 
 **.format**: `'.yaml'`

docs/topics/release-notes.md

@@ -51,6 +51,25 @@ You can determine your currently installed version using `pip freeze`:
 
 ## 2.3.x series
 
+### 2.3.x
+
+**Date**: April 2014
+
+* Fix nested serializers linked through a backward foreign key relation
+* Fix bad links for the `BrowsableAPIRenderer` with `YAMLRenderer`
+* Add `UnicodeYAMLRenderer` that extends `YAMLRenderer` with unicode
+* Fix `parse_header` argument convertion
+* Fix mediatype detection under Python3
+* Web browseable API now offers blank option on dropdown when the field is not required
+* `APIException` representation improved for logging purposes
+* Allow source="*" within nested serializers
+* Better support for custom oauth2 provider backends
+* Fix field validation if it's optional and has no value
+* Add `SEARCH_PARAM` and `ORDERING_PARAM`
+* Fix `APIRequestFactory` to support arguments within the url string for GET
+* Allow three transport modes for access tokens when accessing a protected resource
+* Fix `Request`'s `QueryDict` encoding
+
 ### 2.3.13
 ## 2.3.x series
 

rest_framework/authtoken/models.py

@@ -34,7 +34,7 @@ class Token(models.Model):
         return super(Token, self).save(*args, **kwargs)
 
     def generate_key(self):
-        return binascii.hexlify(os.urandom(20))
+        return binascii.hexlify(os.urandom(20)).decode()
 
     def __unicode__(self):
         return self.key

rest_framework/fields.py

@@ -284,7 +284,7 @@ class WritableField(Field):
         self.validators = self.default_validators + validators
         self.default = default if default is not None else self.default
 
-        # Widgets are ony used for HTML forms.
+        # Widgets are only used for HTML forms.
         widget = widget or self.widget
         if isinstance(widget, type):
             widget = widget()

rest_framework/parsers.py

@@ -10,7 +10,7 @@ from django.core.files.uploadhandler import StopFutureHandlers
 from django.http import QueryDict
 from django.http.multipartparser import MultiPartParser as DjangoMultiPartParser
 from django.http.multipartparser import MultiPartParserError, parse_header, ChunkIter
-from rest_framework.compat import etree, six, yaml
+from rest_framework.compat import etree, six, yaml, force_text
 from rest_framework.exceptions import ParseError
 from rest_framework import renderers
 import json
@@ -288,7 +288,7 @@ class FileUploadParser(BaseParser):
 
         try:
             meta = parser_context['request'].META
-            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'])
+            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8'))
-            return disposition[1]['filename']
+            return force_text(disposition[1]['filename'])
         except (AttributeError, KeyError):
             pass

rest_framework/renderers.py

@@ -193,6 +193,7 @@ class YAMLRenderer(BaseRenderer):
     format = 'yaml'
     encoder = encoders.SafeDumper
     charset = 'utf-8'
+    ensure_ascii = True
 
     def render(self, data, accepted_media_type=None, renderer_context=None):
         """
@@ -203,7 +204,15 @@ class YAMLRenderer(BaseRenderer):
         if data is None:
             return ''
 
-        return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder)
+        return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder, allow_unicode=not self.ensure_ascii)
+
+
+class UnicodeYAMLRenderer(YAMLRenderer):
+    """
+    Renderer which serializes to YAML.
+    Does *not* apply character escaping for non-ascii characters.
+    """
+    ensure_ascii = False
 
 
 class TemplateHTMLRenderer(BaseRenderer):

rest_framework/serializers.py

@@ -16,10 +16,12 @@ import datetime
 import inspect
 import types
 from decimal import Decimal
+from django.contrib.contenttypes.generic import GenericForeignKey
+from django.core.paginator import Page
 from django.db import models
 from django.forms import widgets
 from django.utils.datastructures import SortedDict
-from rest_framework.compat import get_concrete_model, six
+from rest_framework.compat import six
 from rest_framework.settings import api_settings
 
 
@@ -821,6 +823,10 @@ class ModelSerializer(Serializer):
 
         if model_field:
             kwargs['required'] = not(model_field.null or model_field.blank)
+            if model_field.help_text is not None:
+                kwargs['help_text'] = model_field.help_text
+            if model_field.verbose_name is not None:
+                kwargs['label'] = model_field.verbose_name
 
         return PrimaryKeyRelatedField(**kwargs)
 
@@ -941,6 +947,8 @@ class ModelSerializer(Serializer):
 
         # Forward m2m relations
         for field in meta.many_to_many + meta.virtual_fields:
+            if isinstance(field, GenericForeignKey):
+                continue
             if field.name in attrs:
                 m2m_data[field.name] = attrs.pop(field.name)
 
@@ -950,17 +958,15 @@ class ModelSerializer(Serializer):
             if isinstance(self.fields.get(field_name, None), Serializer):
                 nested_forward_relations[field_name] = attrs[field_name]
 
-        # Update an existing instance...
+        # Create an empty instance of the model
-        if instance is not None:
+        if instance is None:
-            for key, val in attrs.items():
+            instance = self.opts.model()
-                try:
-                    setattr(instance, key, val)
-                except ValueError:
-                    self._errors[key] = self.error_messages['required']
 
-        # ...or create a new instance
+        for key, val in attrs.items():
-        else:
+            try:
-            instance = self.opts.model(**attrs)
+                setattr(instance, key, val)
+            except ValueError:
+                self._errors[key] = self.error_messages['required']
 
         # Any relations that cannot be set until we've
         # saved the model get hidden away on these
@@ -1085,6 +1091,10 @@ class HyperlinkedModelSerializer(ModelSerializer):
 
         if model_field:
             kwargs['required'] = not(model_field.null or model_field.blank)
+            if model_field.help_text is not None:
+                kwargs['help_text'] = model_field.help_text
+            if model_field.verbose_name is not None:
+                kwargs['label'] = model_field.verbose_name
 
         if self.opts.lookup_field:
             kwargs['lookup_field'] = self.opts.lookup_field

rest_framework/templatetags/rest_framework.py

@@ -98,7 +98,7 @@ def add_class(value, css_class):
 
 
 # Bunch of stuff cloned from urlize
-TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "'"]
+TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "']", "'}", "'"]
 WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('&lt;', '&gt;'),
                         ('"', '"'), ("'", "'")]
 word_split_re = re.compile(r'(\s+)')

rest_framework/tests/models.py

@@ -143,7 +143,8 @@ class ForeignKeyTarget(RESTFrameworkModel):
 
 class ForeignKeySource(RESTFrameworkModel):
     name = models.CharField(max_length=100)
-    target = models.ForeignKey(ForeignKeyTarget, related_name='sources')
+    target = models.ForeignKey(ForeignKeyTarget, related_name='sources',
+                               help_text='Target', verbose_name='Target')
 
 
 # Nullable ForeignKey

rest_framework/tests/test_authentication.py

@@ -20,6 +20,7 @@ from rest_framework.authentication import (
     OAuth2Authentication
 )
 from rest_framework.authtoken.models import Token
+from rest_framework.compat import patterns, url, include, six
 from rest_framework.compat import oauth2_provider, oauth2_provider_scope
 from rest_framework.compat import oauth, oauth_provider
 from rest_framework.test import APIRequestFactory, APIClient
@@ -195,6 +196,12 @@ class TokenAuthTests(TestCase):
         token = Token.objects.create(user=self.user)
         self.assertTrue(bool(token.key))
 
+    def test_generate_key_returns_string(self):
+        """Ensure generate_key returns a string"""
+        token = Token()
+        key = token.generate_key()
+        self.assertTrue(isinstance(key, six.string_types))
+
     def test_token_login_json(self):
         """Ensure token login view using JSON POST works."""
         client = APIClient(enforce_csrf_checks=True)

rest_framework/tests/test_genericrelations.py

@@ -131,3 +131,21 @@ class TestGenericRelations(TestCase):
         }
         ]
         self.assertEqual(serializer.data, expected)
+
+    def test_restore_object_generic_fk(self):
+        """
+        Ensure an object with a generic foreign key can be restored.
+        """
+
+        class TagSerializer(serializers.ModelSerializer):
+            class Meta:
+                model = Tag
+                exclude = ('content_type', 'object_id')
+
+        serializer = TagSerializer()
+
+        bookmark = Bookmark(url='http://example.com')
+        attrs = {'tagged_item': bookmark, 'tag': 'example'}
+
+        tag = serializer.restore_object(attrs)
+        self.assertEqual(tag.tagged_item, bookmark)

rest_framework/tests/test_parsers.py

@@ -96,7 +96,7 @@ class TestFileUploadParser(TestCase):
         request = MockRequest()
         request.upload_handlers = (MemoryFileUploadHandler(),)
         request.META = {
-            'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'),
+            'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt',
             'HTTP_CONTENT_LENGTH': 14,
         }
         self.parser_context = {'request': request, 'kwargs': {}}
@@ -112,4 +112,4 @@ class TestFileUploadParser(TestCase):
     def test_get_filename(self):
         parser = FileUploadParser()
         filename = parser.get_filename(self.stream, None, self.parser_context)
-        self.assertEqual(filename, 'file.txt'.encode('utf-8'))
+        self.assertEqual(filename, 'file.txt')

rest_framework/tests/test_renderers.py

@@ -13,7 +13,7 @@ from rest_framework.compat import yaml, etree, six, StringIO
 from rest_framework.response import Response
 from rest_framework.views import APIView
 from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \
-    XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer
+    XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer, UnicodeYAMLRenderer
 from rest_framework.parsers import YAMLParser, XMLParser
 from rest_framework.settings import api_settings
 from rest_framework.test import APIRequestFactory
@@ -468,6 +468,17 @@ if yaml:
             self.assertTrue(string in content, '%r not in %r' % (string, content))
 
 
+    class UnicodeYAMLRendererTests(TestCase):
+        """
+        Tests specific for the Unicode YAML Renderer
+        """
+        def test_proper_encoding(self):
+            obj = {'countries': ['United Kingdom', 'France', 'España']}
+            renderer = UnicodeYAMLRenderer()
+            content = renderer.render(obj, 'application/yaml')
+            self.assertEqual(content.strip(), 'countries: [United Kingdom, France, España]'.encode('utf-8'))
+
+
 class XMLRendererTestCase(TestCase):
     """
     Tests specific to the XML Renderer

rest_framework/tests/test_serializer.py

@@ -9,7 +9,8 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers, fields, relations
 from rest_framework.tests.models import (HasPositiveIntegerAsChoice, Album, ActionItem, Anchor, BasicModel,
     BlankFieldModel, BlogPost, BlogPostComment, Book, CallableDefaultValueModel, DefaultValueModel,
-    ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel)
+    ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel,
+    ForeignKeySource, ManyToManySource)
 from rest_framework.tests.models import BasicModelSerializer
 import datetime
 import pickle
@@ -176,6 +177,16 @@ class PositiveIntegerAsChoiceSerializer(serializers.ModelSerializer):
         fields = ['some_integer']
 
 
+class ForeignKeySourceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ForeignKeySource
+
+
+class HyperlinkedForeignKeySourceSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = ForeignKeySource
+
+
 class BasicTests(TestCase):
     def setUp(self):
         self.comment = Comment(
@@ -1614,6 +1625,19 @@ class ManyFieldHelpTextTest(TestCase):
         self.assertEqual('Some help text.', rel_field.help_text)
 
 
+class AttributeMappingOnAutogeneratedRelatedFields(TestCase):
+
+    def test_primary_key_related_field(self):
+        serializer = ForeignKeySourceSerializer()
+        self.assertEqual(serializer.fields['target'].help_text, 'Target')
+        self.assertEqual(serializer.fields['target'].label, 'Target')
+
+    def test_hyperlinked_related_field(self):
+        serializer = HyperlinkedForeignKeySourceSerializer()
+        self.assertEqual(serializer.fields['target'].help_text, 'Target')
+        self.assertEqual(serializer.fields['target'].label, 'Target')
+
+
 @unittest.skipUnless(PIL is not None, 'PIL is not installed')
 class AttributeMappingOnAutogeneratedFieldsTests(TestCase):
 

rest_framework/tests/test_urlizer.py

@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+from django.test import TestCase
+from rest_framework.templatetags.rest_framework import urlize_quoted_links
+import sys
+
+
+class URLizerTests(TestCase):
+    """
+    Test if both JSON and YAML URLs are transformed into links well
+    """
+    def _urlize_dict_check(self, data):
+        """
+        For all items in dict test assert that the value is urlized key
+        """
+        for original, urlized in data.items():
+            assert urlize_quoted_links(original, nofollow=False) == urlized
+
+    def test_json_with_url(self):
+        """
+        Test if JSON URLs are transformed into links well
+        """
+        data = {}
+        data['"url": "http://api/users/1/", '] = \
+            '&quot;url&quot;: &quot;<a href="http://api/users/1/">http://api/users/1/</a>&quot;, '
+        data['"foo_set": [\n    "http://api/foos/1/"\n], '] = \
+            '&quot;foo_set&quot;: [\n    &quot;<a href="http://api/foos/1/">http://api/foos/1/</a>&quot;\n], '
+        self._urlize_dict_check(data)
+
+    def test_yaml_with_url(self):
+        """
+        Test if YAML URLs are transformed into links well
+        """
+        data = {}
+        data['''{users: 'http://api/users/'}'''] = \
+            '''{users: &#39;<a href="http://api/users/">http://api/users/</a>&#39;}'''
+        data['''foo_set: ['http://api/foos/1/']'''] = \
+            '''foo_set: [&#39;<a href="http://api/foos/1/">http://api/foos/1/</a>&#39;]'''
+        self._urlize_dict_check(data)

tox.ini

@@ -7,21 +7,21 @@ commands = {envpython} rest_framework/runtests/runtests.py
 
 [testenv:py3.3-django1.7]
 basepython = python3.3
-deps = https://www.djangoproject.com/download/1.7b1/tarball/
+deps = https://www.djangoproject.com/download/1.7b2/tarball/
        django-filter==0.7
        defusedxml==0.3
        Pillow==2.3.0
 
 [testenv:py3.2-django1.7]
 basepython = python3.2
-deps = https://www.djangoproject.com/download/1.7b1/tarball/
+deps = https://www.djangoproject.com/download/1.7b2/tarball/
        django-filter==0.7
        defusedxml==0.3
        Pillow==2.3.0
 
 [testenv:py2.7-django1.7]
 basepython = python2.7
-deps = https://www.djangoproject.com/download/1.7b1/tarball/
+deps = https://www.djangoproject.com/download/1.7b2/tarball/
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1
@@ -32,21 +32,21 @@ deps = https://www.djangoproject.com/download/1.7b1/tarball/
 
 [testenv:py3.3-django1.6]
 basepython = python3.3
-deps = Django==1.6
+deps = Django==1.6.3
        django-filter==0.7
        defusedxml==0.3
        Pillow==2.3.0
 
 [testenv:py3.2-django1.6]
 basepython = python3.2
-deps = Django==1.6
+deps = Django==1.6.3
        django-filter==0.7
        defusedxml==0.3
        Pillow==2.3.0
 
 [testenv:py2.7-django1.6]
 basepython = python2.7
-deps = Django==1.6
+deps = Django==1.6.3
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1
@@ -57,7 +57,7 @@ deps = Django==1.6
 
 [testenv:py2.6-django1.6]
 basepython = python2.6
-deps = Django==1.6
+deps = Django==1.6.3
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1
@@ -68,21 +68,21 @@ deps = Django==1.6
 
 [testenv:py3.3-django1.5]
 basepython = python3.3
-deps = django==1.5.5
+deps = django==1.5.6
        django-filter==0.7
        defusedxml==0.3
        Pillow==2.3.0
 
 [testenv:py3.2-django1.5]
 basepython = python3.2
-deps = django==1.5.5
+deps = django==1.5.6
        django-filter==0.7
        defusedxml==0.3
        Pillow==2.3.0
 
 [testenv:py2.7-django1.5]
 basepython = python2.7
-deps = django==1.5.5
+deps = django==1.5.6
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1
@@ -93,7 +93,7 @@ deps = django==1.5.5
 
 [testenv:py2.6-django1.5]
 basepython = python2.6
-deps = django==1.5.5
+deps = django==1.5.6
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1
@@ -104,7 +104,7 @@ deps = django==1.5.5
 
 [testenv:py2.7-django1.4]
 basepython = python2.7
-deps = django==1.4.10
+deps = django==1.4.11
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1
@@ -115,7 +115,7 @@ deps = django==1.4.10
 
 [testenv:py2.6-django1.4]
 basepython = python2.6
-deps = django==1.4.10
+deps = django==1.4.11
        django-filter==0.7
        defusedxml==0.3
        django-oauth-plus==2.2.1