From e59b3d1718de549d0e165d03aeea1488ddfe20ee Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 21 Jan 2015 14:18:13 +0000 Subject: [PATCH 01/15] Make ReturnDict cachable. Closes #2360. --- rest_framework/utils/serializer_helpers.py | 10 ++++++++++ tests/test_serializer.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/rest_framework/utils/serializer_helpers.py b/rest_framework/utils/serializer_helpers.py index ab0578620..87bb3ac08 100644 --- a/rest_framework/utils/serializer_helpers.py +++ b/rest_framework/utils/serializer_helpers.py @@ -19,6 +19,11 @@ class ReturnDict(OrderedDict): def __repr__(self): return dict.__repr__(self) + def __reduce__(self): + # Pickling these objects will drop the .serializer backlink, + # but preserve the raw data. + return (dict, (dict(self),)) + class ReturnList(list): """ @@ -33,6 +38,11 @@ class ReturnList(list): def __repr__(self): return list.__repr__(self) + def __reduce__(self): + # Pickling these objects will drop the .serializer backlink, + # but preserve the raw data. + return (list, (list(self),)) + class BoundField(object): """ diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 68bbbe983..b7a0484bc 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from .utils import MockObject from rest_framework import serializers from rest_framework.compat import unicode_repr +import pickle import pytest @@ -278,3 +279,19 @@ class TestNotRequiredOutput: serializer = ExampleSerializer(instance) with pytest.raises(AttributeError): serializer.data + + +class TestCacheSerializerData: + def test_cache_serializer_data(self): + """ + Caching serializer data with pickle will drop the serializer info, + but does preserve the data itself. + """ + class ExampleSerializer(serializers.Serializer): + field1 = serializers.CharField() + field2 = serializers.CharField() + + serializer = ExampleSerializer({'field1': 'a', 'field2': 'b'}) + pickled = pickle.dumps(serializer.data) + data = pickle.loads(pickled) + assert data == {'field1': 'a', 'field2': 'b'} From 4cf03e30ff765dda2899048725da4d85ebd8af52 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 21 Jan 2015 14:26:25 +0000 Subject: [PATCH 02/15] Do not render HTML output for hidden fields. Closes #2410. --- rest_framework/renderers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index ba6c9cc15..584332e66 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -410,6 +410,9 @@ class HTMLFormRenderer(BaseRenderer): }) def render_field(self, field, parent_style): + if isinstance(field, serializers.HiddenField): + return '' + style = dict(self.default_style[field]) style.update(field.style) if 'template_pack' not in style: From a7567efa8d6fd008ba0a48f0e8fa7028703af386 Mon Sep 17 00:00:00 2001 From: Xavier Ordoquy Date: Wed, 21 Jan 2015 19:26:57 +0100 Subject: [PATCH 03/15] Use compact traceback for errors reporting. --- runtests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtests.py b/runtests.py index abf15a623..0008bfae5 100755 --- a/runtests.py +++ b/runtests.py @@ -8,8 +8,8 @@ import subprocess PYTEST_ARGS = { - 'default': ['tests'], - 'fast': ['tests', '-q'], + 'default': ['tests', '--tb=short'], + 'fast': ['tests', '--tb=short', '-q'], } FLAKE8_ARGS = ['rest_framework', 'tests', '--ignore=E501'] From 857185cf07bb539083a90bc75a6dd951da8e2206 Mon Sep 17 00:00:00 2001 From: Xavier Ordoquy Date: Wed, 21 Jan 2015 19:29:40 +0100 Subject: [PATCH 04/15] Workaround Django issue 24198. --- rest_framework/serializers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 6320a0751..b1474562b 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -12,7 +12,7 @@ response content is handled by parsers and renderers. """ from __future__ import unicode_literals from django.db import models -from django.db.models.fields import FieldDoesNotExist +from django.db.models.fields import FieldDoesNotExist, Field from django.utils.translation import ugettext_lazy as _ from rest_framework.compat import unicode_to_repr from rest_framework.utils import model_meta @@ -939,6 +939,9 @@ class ModelSerializer(Serializer): except FieldDoesNotExist: continue + if not isinstance(model_field, Field): + continue + # Include each of the `unique_for_*` field names. unique_constraint_names |= set([ model_field.unique_for_date, From 15f797fd3ec61947aaecc05e6fd040e1e3e8776a Mon Sep 17 00:00:00 2001 From: Xavier Ordoquy Date: Wed, 21 Jan 2015 19:46:31 +0100 Subject: [PATCH 05/15] Owned by import * --- rest_framework/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index b1474562b..cf797bdcb 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -12,7 +12,7 @@ response content is handled by parsers and renderers. """ from __future__ import unicode_literals from django.db import models -from django.db.models.fields import FieldDoesNotExist, Field +from django.db.models.fields import FieldDoesNotExist, Field as DjangoField from django.utils.translation import ugettext_lazy as _ from rest_framework.compat import unicode_to_repr from rest_framework.utils import model_meta @@ -939,7 +939,7 @@ class ModelSerializer(Serializer): except FieldDoesNotExist: continue - if not isinstance(model_field, Field): + if not isinstance(model_field, DjangoField): continue # Include each of the `unique_for_*` field names. From 6e471ad8f41dda11365080ca583a0ccbf37de55e Mon Sep 17 00:00:00 2001 From: Duncan Maitland Date: Thu, 22 Jan 2015 18:29:20 +1100 Subject: [PATCH 06/15] fix link to Django CSRF docs --- docs/api-guide/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api-guide/authentication.md b/docs/api-guide/authentication.md index 1222dbf04..0d53de70a 100755 --- a/docs/api-guide/authentication.md +++ b/docs/api-guide/authentication.md @@ -427,7 +427,7 @@ HTTP Signature (currently a [IETF draft][http-signature-ietf-draft]) provides a [oauth]: http://oauth.net/2/ [permission]: permissions.md [throttling]: throttling.md -[csrf-ajax]: https://docs.djangoproject.com/en/dev/ref/contrib/csrf/#ajax +[csrf-ajax]: https://docs.djangoproject.com/en/dev/ref/csrf/#ajax [mod_wsgi_official]: http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives#WSGIPassAuthorization [custom-user-model]: https://docs.djangoproject.com/en/dev/topics/auth/customizing/#specifying-a-custom-user-model [south-dependencies]: http://south.readthedocs.org/en/latest/dependencies.html From 25a703b42c030f712734ed56b8f1996f8d13ac0c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 11:15:11 +0000 Subject: [PATCH 07/15] Work around meta API differences --- rest_framework/utils/model_meta.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rest_framework/utils/model_meta.py b/rest_framework/utils/model_meta.py index 375d2e8c6..6a5835f54 100644 --- a/rest_framework/utils/model_meta.py +++ b/rest_framework/utils/model_meta.py @@ -121,12 +121,17 @@ def _get_reverse_relationships(opts): """ Returns an `OrderedDict` of field names to `RelationInfo`. """ + # Note that we have a hack here to handle internal API differences for + # this internal API across Django 1.7 -> Django 1.8. + # See: https://code.djangoproject.com/ticket/24208 + reverse_relations = OrderedDict() for relation in opts.get_all_related_objects(): accessor_name = relation.get_accessor_name() + related = getattr(relation, 'related_model', relation.model) reverse_relations[accessor_name] = RelationInfo( model_field=None, - related=relation.model, + related=related, to_many=relation.field.rel.multiple, has_through_model=False ) @@ -134,9 +139,10 @@ def _get_reverse_relationships(opts): # Deal with reverse many-to-many relationships. for relation in opts.get_all_related_many_to_many_objects(): accessor_name = relation.get_accessor_name() + related = getattr(relation, 'related_model', relation.model) reverse_relations[accessor_name] = RelationInfo( model_field=None, - related=relation.model, + related=related, to_many=True, has_through_model=( (getattr(relation.field.rel, 'through', None) is not None) From 5eb6949e9f24dd5e94aa5eb50fd6ccaf34b21878 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 11:32:04 +0000 Subject: [PATCH 08/15] Drop django-filter from 1.8 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e43a9234f..75ebe1345 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ deps = {py26,py27}-django{14,15}: django-oauth2-provider==0.2.3 {py26,py27}-django16: django-oauth2-provider==0.2.4 pytest-django==2.8.0 - django-filter==0.7 + {py27,py32,py32,py33,py34}-django{14,15,16,17}: django-filter==0.7 defusedxml==0.3 markdown>=2.1.0 PyYAML>=3.10 From e307fd289c52a9bb97a567cff314a479bbdd21df Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 11:38:58 +0000 Subject: [PATCH 09/15] Fix test matrix --- env/pip-selfcheck.json | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 env/pip-selfcheck.json diff --git a/env/pip-selfcheck.json b/env/pip-selfcheck.json new file mode 100644 index 000000000..50cde9566 --- /dev/null +++ b/env/pip-selfcheck.json @@ -0,0 +1 @@ +{"last_check":"2015-01-23T11:37:11Z","pypi_version":"6.0.6"} \ No newline at end of file diff --git a/tox.ini b/tox.ini index 75ebe1345..193b5813f 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ deps = {py26,py27}-django{14,15}: django-oauth2-provider==0.2.3 {py26,py27}-django16: django-oauth2-provider==0.2.4 pytest-django==2.8.0 - {py27,py32,py32,py33,py34}-django{14,15,16,17}: django-filter==0.7 + {py26,py27,py32,py33,py34}-django{14,15,16,17}: django-filter==0.7 defusedxml==0.3 markdown>=2.1.0 PyYAML>=3.10 From e988d578535fcc820d30dc7c59f1e24f5c911d3c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 11:47:01 +0000 Subject: [PATCH 10/15] Fix template loader monkey patching to also support 1.8 --- tests/test_htmlrenderer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_htmlrenderer.py b/tests/test_htmlrenderer.py index 2edc6b4bd..a33b832f5 100644 --- a/tests/test_htmlrenderer.py +++ b/tests/test_htmlrenderer.py @@ -56,7 +56,13 @@ class TemplateHTMLRendererTests(TestCase): return Template("example: {{ object }}") raise TemplateDoesNotExist(template_name) + def select_template(template_name_list, dirs=None, using=None): + if template_name_list == ['example.html']: + return Template("example: {{ object }}") + raise TemplateDoesNotExist(template_name_list[0]) + django.template.loader.get_template = get_template + django.template.loader.select_template = select_template def tearDown(self): """ From 4cb164b66c0784ce79054925d4744deb5b18d8b2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 11:49:57 +0000 Subject: [PATCH 11/15] Add missing skipUnless(django_filters) --- tests/test_filters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_filters.py b/tests/test_filters.py index dc84dcbd0..5b1b6ca52 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -467,6 +467,7 @@ class DjangoFilterOrderingTests(TestCase): for d in data: DjangoFilterOrderingModel.objects.create(**d) + @unittest.skipUnless(django_filters, 'django-filter not installed') def test_default_ordering(self): class DjangoFilterOrderingView(generics.ListAPIView): serializer_class = DjangoFilterOrderingSerializer From f1ac9d3f9b6c306b7fa48381006d8259c1642a99 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 12:26:44 +0000 Subject: [PATCH 12/15] More graceful handling of malformed Content-Disposition --- rest_framework/parsers.py | 2 +- tests/test_parsers.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 401856ec4..ef72677ce 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -298,7 +298,7 @@ class FileUploadParser(BaseParser): if 'filename*' in filename_parm: return self.get_encoded_filename(filename_parm) return force_text(filename_parm['filename']) - except (AttributeError, KeyError): + except (AttributeError, KeyError, ValueError): pass def get_encoded_filename(self, filename_parm): diff --git a/tests/test_parsers.py b/tests/test_parsers.py index d28d8bd43..1d2054aca 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -161,7 +161,9 @@ class TestFileUploadParser(TestCase): self.__replace_content_disposition('inline; filename=fallback.txt; filename*=utf-8--ÀĥƦ.txt') filename = parser.get_filename(self.stream, None, self.parser_context) - self.assertEqual(filename, 'fallback.txt') + # Malformed. Either None or 'fallback.txt' will be acceptable. + # See also https://code.djangoproject.com/ticket/24209 + self.assertIn(filename, ('fallback.txt', None)) def __replace_content_disposition(self, disposition): self.parser_context['request'].META['HTTP_CONTENT_DISPOSITION'] = disposition From f3b6eedb8aeaa23f4b48551356814837973db31c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 12:56:55 +0000 Subject: [PATCH 13/15] More sensible response caching. --- rest_framework/response.py | 7 +++- tests/test_renderers.py | 85 ++++++-------------------------------- 2 files changed, 17 insertions(+), 75 deletions(-) diff --git a/rest_framework/response.py b/rest_framework/response.py index d6ca1aad4..7f90bae10 100644 --- a/rest_framework/response.py +++ b/rest_framework/response.py @@ -81,10 +81,13 @@ class Response(SimpleTemplateResponse): def __getstate__(self): """ - Remove attributes from the response that shouldn't be cached + Remove attributes from the response that shouldn't be cached. """ state = super(Response, self).__getstate__() - for key in ('accepted_renderer', 'renderer_context', 'data'): + for key in ( + 'accepted_renderer', 'renderer_context', 'resolver_match', + 'client', 'request', 'wsgi_request', '_closable_objects' + ): if key in state: del state[key] return state diff --git a/tests/test_renderers.py b/tests/test_renderers.py index 00a24fb12..54eea8ceb 100644 --- a/tests/test_renderers.py +++ b/tests/test_renderers.py @@ -22,7 +22,6 @@ from rest_framework.test import APIRequestFactory from collections import MutableMapping import datetime import json -import pickle import re @@ -618,84 +617,24 @@ class CacheRenderTest(TestCase): urls = 'tests.test_renderers' - cache_key = 'just_a_cache_key' - - @classmethod - def _get_pickling_errors(cls, obj, seen=None): - """ Return any errors that would be raised if `obj' is pickled - Courtesy of koffie @ http://stackoverflow.com/a/7218986/109897 - """ - if seen is None: - seen = [] - try: - state = obj.__getstate__() - except AttributeError: - return - if state is None: - return - if isinstance(state, tuple): - if not isinstance(state[0], dict): - state = state[1] - else: - state = state[0].update(state[1]) - result = {} - for i in state: - try: - pickle.dumps(state[i], protocol=2) - except pickle.PicklingError: - if not state[i] in seen: - seen.append(state[i]) - result[i] = cls._get_pickling_errors(state[i], seen) - return result - - def http_resp(self, http_method, url): - """ - Simple wrapper for Client http requests - Removes the `client' and `request' attributes from as they are - added by django.test.client.Client and not part of caching - responses outside of tests. - """ - method = getattr(self.client, http_method) - resp = method(url) - resp._closable_objects = [] - del resp.client, resp.request - try: - del resp.wsgi_request - except AttributeError: - pass - return resp - - def test_obj_pickling(self): - """ - Test that responses are properly pickled - """ - resp = self.http_resp('get', '/cache') - - # Make sure that no pickling errors occurred - self.assertEqual(self._get_pickling_errors(resp), {}) - - # Unfortunately LocMem backend doesn't raise PickleErrors but returns - # None instead. - cache.set(self.cache_key, resp) - self.assertTrue(cache.get(self.cache_key) is not None) - def test_head_caching(self): """ Test caching of HEAD requests """ - resp = self.http_resp('head', '/cache') - cache.set(self.cache_key, resp) - - cached_resp = cache.get(self.cache_key) - self.assertIsInstance(cached_resp, Response) + response = self.client.head('/cache') + cache.set('key', response) + cached_response = cache.get('key') + assert isinstance(cached_response, Response) + assert cached_response.content == response.content + assert cached_response.status_code == response.status_code def test_get_caching(self): """ Test caching of GET requests """ - resp = self.http_resp('get', '/cache') - cache.set(self.cache_key, resp) - - cached_resp = cache.get(self.cache_key) - self.assertIsInstance(cached_resp, Response) - self.assertEqual(cached_resp.content, resp.content) + response = self.client.get('/cache') + cache.set('key', response) + cached_response = cache.get('key') + assert isinstance(cached_response, Response) + assert cached_response.content == response.content + assert cached_response.status_code == response.status_code From 04a5f7bf0a268d24656ef3659f85aec95fd7590a Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 13:04:29 +0000 Subject: [PATCH 14/15] Move 1.8-alpha out of 'expected failures' --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 996c3ae80..28ebfc00f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,10 +33,6 @@ env: matrix: fast_finish: true allow_failures: - - env: TOX_ENV=py34-django18alpha - - env: TOX_ENV=py33-django18alpha - - env: TOX_ENV=py32-django18alpha - - env: TOX_ENV=py27-django18alpha - env: TOX_ENV=py34-djangomaster - env: TOX_ENV=py33-djangomaster - env: TOX_ENV=py32-djangomaster From 4201c9fb01beae84fc34a5b74e138e721de42de1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Jan 2015 13:42:56 +0000 Subject: [PATCH 15/15] Drop erronous check-in --- env/pip-selfcheck.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 env/pip-selfcheck.json diff --git a/env/pip-selfcheck.json b/env/pip-selfcheck.json deleted file mode 100644 index 50cde9566..000000000 --- a/env/pip-selfcheck.json +++ /dev/null @@ -1 +0,0 @@ -{"last_check":"2015-01-23T11:37:11Z","pypi_version":"6.0.6"} \ No newline at end of file