From baa518cd890103173dd18857c609432bd47c6be4 Mon Sep 17 00:00:00 2001 From: Jharrod LaFon Date: Fri, 5 Sep 2014 15:30:01 -0700 Subject: [PATCH 01/29] Moved OAuth support out of DRF and into a separate package, per #1767 --- .travis.yml | 3 - README.md | 2 +- docs/api-guide/authentication.md | 99 ------- docs/api-guide/permissions.md | 17 -- docs/index.md | 11 +- requirements-test.txt | 3 - rest_framework/authentication.py | 183 ------------- rest_framework/compat.py | 51 ---- rest_framework/permissions.py | 28 +- tests/conftest.py | 20 -- tests/settings.py | 21 -- tests/test_authentication.py | 430 +------------------------------ 12 files changed, 6 insertions(+), 862 deletions(-) diff --git a/.travis.yml b/.travis.yml index e768e1468..017e72363 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,6 @@ install: - pip install django-guardian==1.2.3 - pip install pytest-django==2.6.1 - pip install flake8==2.2.2 - - "if [[ ${TRAVIS_PYTHON_VERSION::1} != '3' ]]; then pip install oauth2==1.5.211; fi" - - "if [[ ${TRAVIS_PYTHON_VERSION::1} != '3' ]]; then pip install django-oauth-plus==2.2.4; fi" - - "if [[ ${TRAVIS_PYTHON_VERSION::1} != '3' ]]; then pip install django-oauth2-provider==0.2.4; fi" - "if [[ ${DJANGO::11} == 'django==1.3' ]]; then pip install django-filter==0.5.4; fi" - "if [[ ${DJANGO::11} != 'django==1.3' ]]; then pip install django-filter==0.7; fi" - "if [[ ${DJANGO} == 'django==1.7' ]]; then pip install -e git+https://github.com/linovia/django-guardian.git@feature/django_1_7#egg=django-guardian-1.2.0; fi" diff --git a/README.md b/README.md index 63513f758..ebc83bf5c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Django REST framework is a powerful and flexible toolkit for building Web APIs. Some reasons you might want to use REST framework: * The [Web browseable API][sandbox] is a huge useability win for your developers. -* [Authentication policies][authentication] including [OAuth1a][oauth1-section] and [OAuth2][oauth2-section] out of the box. +* [Authentication policies][authentication] including [OAuth1a][oauth1-section] and [OAuth2][oauth2-section] through the rest-framework-oauth package. * [Serialization][serializers] that supports both [ORM][modelserializer-section] and [non-ORM][serializer-section] data sources. * Customizable all the way down - just use [regular function-based views][functionview-section] if you don't need the [more][generic-views] [powerful][viewsets] [features][routers]. * [Extensive documentation][index], and [great community support][group]. diff --git a/docs/api-guide/authentication.md b/docs/api-guide/authentication.md index 343466eee..3d4e0f722 100755 --- a/docs/api-guide/authentication.md +++ b/docs/api-guide/authentication.md @@ -247,105 +247,6 @@ Unauthenticated responses that are denied permission will result in an `HTTP 403 If you're using an AJAX style API with SessionAuthentication, you'll need to make sure you include a valid CSRF token for any "unsafe" HTTP method calls, such as `PUT`, `PATCH`, `POST` or `DELETE` requests. See the [Django CSRF documentation][csrf-ajax] for more details. -## OAuthAuthentication - -This authentication uses [OAuth 1.0a][oauth-1.0a] authentication scheme. OAuth 1.0a provides signature validation which provides a reasonable level of security over plain non-HTTPS connections. However, it may also be considered more complicated than OAuth2, as it requires clients to sign their requests. - -This authentication class depends on the optional `django-oauth-plus` and `oauth2` packages. In order to make it work you must install these packages and add `oauth_provider` to your `INSTALLED_APPS`: - - INSTALLED_APPS = ( - ... - `oauth_provider`, - ) - -Don't forget to run `syncdb` once you've added the package. - - python manage.py syncdb - -#### Getting started with django-oauth-plus - -The OAuthAuthentication class only provides token verification and signature validation for requests. It doesn't provide authorization flow for your clients. You still need to implement your own views for accessing and authorizing tokens. - -The `django-oauth-plus` package provides simple foundation for classic 'three-legged' oauth flow. Please refer to [the documentation][django-oauth-plus] for more details. - -## OAuth2Authentication - -This authentication uses [OAuth 2.0][rfc6749] authentication scheme. OAuth2 is more simple to work with than OAuth1, and provides much better security than simple token authentication. It is an unauthenticated scheme, and requires you to use an HTTPS connection. - -This authentication class depends on the optional [django-oauth2-provider][django-oauth2-provider] project. In order to make it work you must install this package and add `provider` and `provider.oauth2` to your `INSTALLED_APPS`: - - INSTALLED_APPS = ( - ... - 'provider', - 'provider.oauth2', - ) - -Then add `OAuth2Authentication` to your global `DEFAULT_AUTHENTICATION` setting: - - 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'rest_framework.authentication.OAuth2Authentication', - ), - -You must also include the following in your root `urls.py` module: - - url(r'^oauth2/', include('provider.oauth2.urls', namespace='oauth2')), - -Note that the `namespace='oauth2'` argument is required. - -Finally, sync your database. - - python manage.py syncdb - python manage.py migrate - ---- - -**Note:** If you use `OAuth2Authentication` in production you must ensure that your API is only available over `https`. - ---- - -#### Getting started with django-oauth2-provider - -The `OAuth2Authentication` class only provides token verification for requests. It doesn't provide authorization flow for your clients. - -The OAuth 2 authorization flow is taken care by the [django-oauth2-provider][django-oauth2-provider] dependency. A walkthrough is given here, but for more details you should refer to [the documentation][django-oauth2-provider-docs]. - -To get started: - -##### 1. Create a client - -You can create a client, either through the shell, or by using the Django admin. - -Go to the admin panel and create a new `Provider.Client` entry. It will create the `client_id` and `client_secret` properties for you. - -##### 2. Request an access token - -To request an access token, submit a `POST` request to the url `/oauth2/access_token` with the following fields: - -* `client_id` the client id you've just configured at the previous step. -* `client_secret` again configured at the previous step. -* `username` the username with which you want to log in. -* `password` well, that speaks for itself. - -You can use the command line to test that your local configuration is working: - - curl -X POST -d "client_id=YOUR_CLIENT_ID&client_secret=YOUR_CLIENT_SECRET&grant_type=password&username=YOUR_USERNAME&password=YOUR_PASSWORD" http://localhost:8000/oauth2/access_token/ - -You should get a response that looks something like this: - - {"access_token": "", "scope": "read", "expires_in": 86399, "refresh_token": ""} - -##### 3. Access the API - -The only thing needed to make the `OAuth2Authentication` class work is to insert the `access_token` you've received in the `Authorization` request header. - -The command line to test the authentication looks like: - - curl -H "Authorization: Bearer " http://localhost:8000/api/ - -### Alternative OAuth 2 implementations - -Note that [Django OAuth Toolkit][django-oauth-toolkit] is an alternative external package that also includes OAuth 2.0 support for REST framework. - --- # Custom authentication diff --git a/docs/api-guide/permissions.md b/docs/api-guide/permissions.md index e867a4569..a32db4a2d 100644 --- a/docs/api-guide/permissions.md +++ b/docs/api-guide/permissions.md @@ -148,21 +148,6 @@ Note that `DjangoObjectPermissions` **does not** require the `django-guardian` p As with `DjangoModelPermissions` you can use custom model permissions by overriding `DjangoModelPermissions` and setting the `.perms_map` property. Refer to the source code for details. Note that if you add a custom `view` permission for `GET`, `HEAD` and `OPTIONS` requests, you'll probably also want to consider adding the `DjangoObjectPermissionsFilter` class to ensure that list endpoints only return results including objects for which the user has appropriate view permissions. -## TokenHasReadWriteScope - -This permission class is intended for use with either of the `OAuthAuthentication` and `OAuth2Authentication` classes, and ties into the scoping that their backends provide. - -Requests with a safe methods of `GET`, `OPTIONS` or `HEAD` will be allowed if the authenticated token has read permission. - -Requests for `POST`, `PUT`, `PATCH` and `DELETE` will be allowed if the authenticated token has write permission. - -This permission class relies on the implementations of the [django-oauth-plus][django-oauth-plus] and [django-oauth2-provider][django-oauth2-provider] libraries, which both provide limited support for controlling the scope of access tokens: - -* `django-oauth-plus`: Tokens are associated with a `Resource` class which has a `name`, `url` and `is_readonly` properties. -* `django-oauth2-provider`: Tokens are associated with a bitwise `scope` attribute, that defaults to providing bitwise values for `read` and/or `write`. - -If you require more advanced scoping for your API, such as restricting tokens to accessing a subset of functionality of your API then you will need to provide a custom permission class. See the source of the `django-oauth-plus` or `django-oauth2-provider` package for more details on scoping token access. - --- # Custom permissions @@ -254,8 +239,6 @@ The [REST Condition][rest-condition] package is another extension for building c [objectpermissions]: https://docs.djangoproject.com/en/dev/topics/auth/customizing/#handling-object-permissions [guardian]: https://github.com/lukaszb/django-guardian [get_objects_for_user]: http://pythonhosted.org/django-guardian/api/guardian.shortcuts.html#get-objects-for-user -[django-oauth-plus]: http://code.larlet.fr/django-oauth-plus -[django-oauth2-provider]: https://github.com/caffeinehit/django-oauth2-provider [2.2-announcement]: ../topics/2.2-announcement.md [filtering]: filtering.md [drf-any-permissions]: https://github.com/kevin-brown/drf-any-permissions diff --git a/docs/index.md b/docs/index.md index 1888bfe4b..7dd35feab 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,7 +31,7 @@ Django REST framework is a powerful and flexible toolkit that makes it easy to b Some reasons you might want to use REST framework: * The [Web browseable API][sandbox] is a huge usability win for your developers. -* [Authentication policies][authentication] including [OAuth1a][oauth1-section] and [OAuth2][oauth2-section] out of the box. +* [Authentication policies][authentication] including [OAuth1a][oauth1-section] and [OAuth2][oauth2-section] through the rest-framework-oauth package. * [Serialization][serializers] that supports both [ORM][modelserializer-section] and [non-ORM][serializer-section] data sources. * Customizable all the way down - just use [regular function-based views][functionview-section] if you don't need the [more][generic-views] [powerful][viewsets] [features][routers]. * [Extensive documentation][index], and [great community support][group]. @@ -58,12 +58,9 @@ The following packages are optional: * [PyYAML][yaml] (3.10+) - YAML content-type support. * [defusedxml][defusedxml] (0.3+) - XML content-type support. * [django-filter][django-filter] (0.5.4+) - Filtering support. -* [django-oauth-plus][django-oauth-plus] (2.0+) and [oauth2][oauth2] (1.5.211+) - OAuth 1.0a support. -* [django-oauth2-provider][django-oauth2-provider] (0.2.3+) - OAuth 2.0 support. +* [django-restframework-oauth][django-restframework-oauth] package for OAuth 1.0a and 2.0 support. * [django-guardian][django-guardian] (1.1.1+) - Object level permissions support. -**Note**: The `oauth2` Python package is badly misnamed, and actually provides OAuth 1.0a support. Also note that packages required for both OAuth 1.0a, and OAuth 2.0 are not yet Python 3 compatible. - ## Installation Install using `pip`, including any optional packages you want... @@ -260,9 +257,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [yaml]: http://pypi.python.org/pypi/PyYAML [defusedxml]: https://pypi.python.org/pypi/defusedxml [django-filter]: http://pypi.python.org/pypi/django-filter -[oauth2]: https://github.com/simplegeo/python-oauth2 -[django-oauth-plus]: https://bitbucket.org/david/django-oauth-plus/wiki/Home -[django-oauth2-provider]: https://github.com/caffeinehit/django-oauth2-provider +[django-restframework-oauth]: https://github.com/jlafon/django-rest-framework-oauth [django-guardian]: https://github.com/lukaszb/django-guardian [0.4]: https://github.com/tomchristie/django-rest-framework/tree/0.4.X [image]: img/quickstart.png diff --git a/requirements-test.txt b/requirements-test.txt index 411daeba2..a90a1361a 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -9,7 +9,4 @@ markdown>=2.1.0 PyYAML>=3.10 defusedxml>=0.3 django-filter>=0.5.4 -django-oauth-plus>=2.2.1 -oauth2>=1.5.211 -django-oauth2-provider>=0.2.4 Pillow==2.3.0 diff --git a/rest_framework/authentication.py b/rest_framework/authentication.py index f3fec05ec..ff1c44e0b 100644 --- a/rest_framework/authentication.py +++ b/rest_framework/authentication.py @@ -3,14 +3,9 @@ Provides various authentication policies. """ from __future__ import unicode_literals import base64 - from django.contrib.auth import authenticate -from django.core.exceptions import ImproperlyConfigured from django.middleware.csrf import CsrfViewMiddleware -from django.conf import settings from rest_framework import exceptions, HTTP_HEADER_ENCODING -from rest_framework.compat import oauth, oauth_provider, oauth_provider_store -from rest_framework.compat import oauth2_provider, provider_now, check_nonce from rest_framework.authtoken.models import Token @@ -178,181 +173,3 @@ class TokenAuthentication(BaseAuthentication): def authenticate_header(self, request): return 'Token' - - -class OAuthAuthentication(BaseAuthentication): - """ - OAuth 1.0a authentication backend using `django-oauth-plus` and `oauth2`. - - Note: The `oauth2` package actually provides oauth1.0a support. Urg. - We import it from the `compat` module as `oauth`. - """ - www_authenticate_realm = 'api' - - def __init__(self, *args, **kwargs): - super(OAuthAuthentication, self).__init__(*args, **kwargs) - - if oauth is None: - raise ImproperlyConfigured( - "The 'oauth2' package could not be imported." - "It is required for use with the 'OAuthAuthentication' class.") - - if oauth_provider is None: - raise ImproperlyConfigured( - "The 'django-oauth-plus' package could not be imported." - "It is required for use with the 'OAuthAuthentication' class.") - - def authenticate(self, request): - """ - Returns two-tuple of (user, token) if authentication succeeds, - or None otherwise. - """ - try: - oauth_request = oauth_provider.utils.get_oauth_request(request) - except oauth.Error as err: - raise exceptions.AuthenticationFailed(err.message) - - if not oauth_request: - return None - - oauth_params = oauth_provider.consts.OAUTH_PARAMETERS_NAMES - - found = any(param for param in oauth_params if param in oauth_request) - missing = list(param for param in oauth_params if param not in oauth_request) - - if not found: - # OAuth authentication was not attempted. - return None - - if missing: - # OAuth was attempted but missing parameters. - msg = 'Missing parameters: %s' % (', '.join(missing)) - raise exceptions.AuthenticationFailed(msg) - - if not self.check_nonce(request, oauth_request): - msg = 'Nonce check failed' - raise exceptions.AuthenticationFailed(msg) - - try: - consumer_key = oauth_request.get_parameter('oauth_consumer_key') - consumer = oauth_provider_store.get_consumer(request, oauth_request, consumer_key) - except oauth_provider.store.InvalidConsumerError: - msg = 'Invalid consumer token: %s' % oauth_request.get_parameter('oauth_consumer_key') - raise exceptions.AuthenticationFailed(msg) - - if consumer.status != oauth_provider.consts.ACCEPTED: - msg = 'Invalid consumer key status: %s' % consumer.get_status_display() - raise exceptions.AuthenticationFailed(msg) - - try: - token_param = oauth_request.get_parameter('oauth_token') - token = oauth_provider_store.get_access_token(request, oauth_request, consumer, token_param) - except oauth_provider.store.InvalidTokenError: - msg = 'Invalid access token: %s' % oauth_request.get_parameter('oauth_token') - raise exceptions.AuthenticationFailed(msg) - - try: - self.validate_token(request, consumer, token) - except oauth.Error as err: - raise exceptions.AuthenticationFailed(err.message) - - user = token.user - - if not user.is_active: - msg = 'User inactive or deleted: %s' % user.username - raise exceptions.AuthenticationFailed(msg) - - return (token.user, token) - - def authenticate_header(self, request): - """ - If permission is denied, return a '401 Unauthorized' response, - with an appropraite 'WWW-Authenticate' header. - """ - return 'OAuth realm="%s"' % self.www_authenticate_realm - - def validate_token(self, request, consumer, token): - """ - Check the token and raise an `oauth.Error` exception if invalid. - """ - oauth_server, oauth_request = oauth_provider.utils.initialize_server_request(request) - oauth_server.verify_request(oauth_request, consumer, token) - - def check_nonce(self, request, oauth_request): - """ - Checks nonce of request, and return True if valid. - """ - oauth_nonce = oauth_request['oauth_nonce'] - oauth_timestamp = oauth_request['oauth_timestamp'] - return check_nonce(request, oauth_request, oauth_nonce, oauth_timestamp) - - -class OAuth2Authentication(BaseAuthentication): - """ - OAuth 2 authentication backend using `django-oauth2-provider` - """ - www_authenticate_realm = 'api' - allow_query_params_token = settings.DEBUG - - def __init__(self, *args, **kwargs): - super(OAuth2Authentication, self).__init__(*args, **kwargs) - - if oauth2_provider is None: - raise ImproperlyConfigured( - "The 'django-oauth2-provider' package could not be imported. " - "It is required for use with the 'OAuth2Authentication' class.") - - def authenticate(self, request): - """ - Returns two-tuple of (user, token) if authentication succeeds, - or None otherwise. - """ - - auth = get_authorization_header(request).split() - - if len(auth) == 1: - msg = 'Invalid bearer header. No credentials provided.' - raise exceptions.AuthenticationFailed(msg) - elif len(auth) > 2: - msg = 'Invalid bearer header. Token string should not contain spaces.' - raise exceptions.AuthenticationFailed(msg) - - if auth and auth[0].lower() == b'bearer': - access_token = auth[1] - elif 'access_token' in request.POST: - access_token = request.POST['access_token'] - elif 'access_token' in request.GET and self.allow_query_params_token: - access_token = request.GET['access_token'] - else: - return None - - return self.authenticate_credentials(request, access_token) - - def authenticate_credentials(self, request, access_token): - """ - Authenticate the request, given the access token. - """ - - try: - token = oauth2_provider.oauth2.models.AccessToken.objects.select_related('user') - # provider_now switches to timezone aware datetime when - # the oauth2_provider version supports to it. - token = token.get(token=access_token, expires__gt=provider_now()) - except oauth2_provider.oauth2.models.AccessToken.DoesNotExist: - raise exceptions.AuthenticationFailed('Invalid token') - - user = token.user - - if not user.is_active: - msg = 'User inactive or deleted: %s' % user.get_username() - raise exceptions.AuthenticationFailed(msg) - - return (user, token) - - def authenticate_header(self, request): - """ - Bearer is the only finalized type currently - - Check details on the `OAuth2Authentication.authenticate` method - """ - return 'Bearer realm="%s"' % self.www_authenticate_realm diff --git a/rest_framework/compat.py b/rest_framework/compat.py index fa0f0bfb1..bc5719ef8 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -182,57 +182,6 @@ except ImportError: etree = None -# OAuth2 is optional -try: - # Note: The `oauth2` package actually provides oauth1.0a support. Urg. - import oauth2 as oauth -except ImportError: - oauth = None - - -# OAuthProvider is optional -try: - import oauth_provider - from oauth_provider.store import store as oauth_provider_store - - # check_nonce's calling signature in django-oauth-plus changes sometime - # between versions 2.0 and 2.2.1 - def check_nonce(request, oauth_request, oauth_nonce, oauth_timestamp): - check_nonce_args = inspect.getargspec(oauth_provider_store.check_nonce).args - if 'timestamp' in check_nonce_args: - return oauth_provider_store.check_nonce( - request, oauth_request, oauth_nonce, oauth_timestamp - ) - return oauth_provider_store.check_nonce( - request, oauth_request, oauth_nonce - ) - -except (ImportError, ImproperlyConfigured): - oauth_provider = None - oauth_provider_store = None - check_nonce = None - - -# OAuth 2 support is optional -try: - import provider as oauth2_provider - from provider import scope as oauth2_provider_scope - from provider import constants as oauth2_constants - if oauth2_provider.__version__ in ('0.2.3', '0.2.4'): - # 0.2.3 and 0.2.4 are supported version that do not support - # timezone aware datetimes - import datetime - provider_now = datetime.datetime.now - else: - # Any other supported version does use timezone aware datetimes - from django.utils.timezone import now as provider_now -except ImportError: - oauth2_provider = None - oauth2_provider_scope = None - oauth2_constants = None - provider_now = None - - # Handle lazy strings across Py2/Py3 from django.utils.functional import Promise diff --git a/rest_framework/permissions.py b/rest_framework/permissions.py index 29f60d6de..7c4986451 100644 --- a/rest_framework/permissions.py +++ b/rest_framework/permissions.py @@ -3,8 +3,7 @@ Provides a set of pluggable permission policies. """ from __future__ import unicode_literals from django.http import Http404 -from rest_framework.compat import (get_model_name, oauth2_provider_scope, - oauth2_constants) +from rest_framework.compat import get_model_name SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] @@ -199,28 +198,3 @@ class DjangoObjectPermissions(DjangoModelPermissions): return False return True - - -class TokenHasReadWriteScope(BasePermission): - """ - The request is authenticated as a user and the token used has the right scope - """ - - def has_permission(self, request, view): - token = request.auth - read_only = request.method in SAFE_METHODS - - if not token: - return False - - if hasattr(token, 'resource'): # OAuth 1 - return read_only or not request.auth.resource.is_readonly - elif hasattr(token, 'scope'): # OAuth 2 - required = oauth2_constants.READ if read_only else oauth2_constants.WRITE - return oauth2_provider_scope.check(required, request.auth.scope) - - assert False, ( - 'TokenHasReadWriteScope requires either the' - '`OAuthAuthentication` or `OAuth2Authentication` authentication ' - 'class to be used.' - ) diff --git a/tests/conftest.py b/tests/conftest.py index 4b33e19c1..679866215 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,26 +47,6 @@ def pytest_configure(): ), ) - try: - import oauth_provider # NOQA - import oauth2 # NOQA - except ImportError: - pass - else: - settings.INSTALLED_APPS += ( - 'oauth_provider', - ) - - try: - import provider # NOQA - except ImportError: - pass - else: - settings.INSTALLED_APPS += ( - 'provider', - 'provider.oauth2', - ) - # guardian is optional try: import guardian # NOQA diff --git a/tests/settings.py b/tests/settings.py index 91c9ed09e..6a01669c3 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -101,27 +101,6 @@ INSTALLED_APPS = ( 'tests.users', ) -# OAuth is optional and won't work if there is no oauth_provider & oauth2 -try: - import oauth_provider # NOQA - import oauth2 # NOQA -except ImportError: - pass -else: - INSTALLED_APPS += ( - 'oauth_provider', - ) - -try: - import provider # NOQA -except ImportError: - pass -else: - INSTALLED_APPS += ( - 'provider', - 'provider.oauth2', - ) - # guardian is optional try: import guardian # NOQA diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 32041f9c1..ece6eff5f 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -3,8 +3,7 @@ from django.conf.urls import patterns, url, include from django.contrib.auth.models import User from django.http import HttpResponse from django.test import TestCase -from django.utils import six, unittest -from django.utils.http import urlencode +from django.utils import six from rest_framework import HTTP_HEADER_ENCODING from rest_framework import exceptions from rest_framework import permissions @@ -16,17 +15,11 @@ from rest_framework.authentication import ( TokenAuthentication, BasicAuthentication, SessionAuthentication, - OAuthAuthentication, - OAuth2Authentication ) from rest_framework.authtoken.models import Token -from rest_framework.compat import oauth2_provider, oauth2_provider_scope -from rest_framework.compat import oauth, oauth_provider from rest_framework.test import APIRequestFactory, APIClient from rest_framework.views import APIView import base64 -import time -import datetime factory = APIRequestFactory() @@ -50,37 +43,10 @@ urlpatterns = patterns( (r'^basic/$', MockView.as_view(authentication_classes=[BasicAuthentication])), (r'^token/$', MockView.as_view(authentication_classes=[TokenAuthentication])), (r'^auth-token/$', 'rest_framework.authtoken.views.obtain_auth_token'), - (r'^oauth/$', MockView.as_view(authentication_classes=[OAuthAuthentication])), - ( - r'^oauth-with-scope/$', - MockView.as_view( - authentication_classes=[OAuthAuthentication], - permission_classes=[permissions.TokenHasReadWriteScope] - ) - ), url(r'^auth/', include('rest_framework.urls', namespace='rest_framework')) ) -class OAuth2AuthenticationDebug(OAuth2Authentication): - allow_query_params_token = True - -if oauth2_provider is not None: - urlpatterns += patterns( - '', - url(r'^oauth2/', include('provider.oauth2.urls', namespace='oauth2')), - url(r'^oauth2-test/$', MockView.as_view(authentication_classes=[OAuth2Authentication])), - url(r'^oauth2-test-debug/$', MockView.as_view(authentication_classes=[OAuth2AuthenticationDebug])), - url( - r'^oauth2-with-scope-test/$', - MockView.as_view( - authentication_classes=[OAuth2Authentication], - permission_classes=[permissions.TokenHasReadWriteScope] - ) - ) - ) - - class BasicAuthTests(TestCase): """Basic authentication""" urls = 'tests.test_authentication' @@ -276,400 +242,6 @@ class IncorrectCredentialsTests(TestCase): self.assertEqual(response.data, {'detail': 'Bad credentials'}) -class OAuthTests(TestCase): - """OAuth 1.0a authentication""" - urls = 'tests.test_authentication' - - def setUp(self): - # these imports are here because oauth is optional and hiding them in try..except block or compat - # could obscure problems if something breaks - from oauth_provider.models import Consumer, Scope - from oauth_provider.models import Token as OAuthToken - from oauth_provider import consts - - self.consts = consts - - self.csrf_client = APIClient(enforce_csrf_checks=True) - self.username = 'john' - self.email = 'lennon@thebeatles.com' - self.password = 'password' - self.user = User.objects.create_user(self.username, self.email, self.password) - - self.CONSUMER_KEY = 'consumer_key' - self.CONSUMER_SECRET = 'consumer_secret' - self.TOKEN_KEY = "token_key" - self.TOKEN_SECRET = "token_secret" - - self.consumer = Consumer.objects.create( - key=self.CONSUMER_KEY, secret=self.CONSUMER_SECRET, - name='example', user=self.user, status=self.consts.ACCEPTED - ) - - self.scope = Scope.objects.create(name="resource name", url="api/") - self.token = OAuthToken.objects.create( - user=self.user, consumer=self.consumer, scope=self.scope, - token_type=OAuthToken.ACCESS, key=self.TOKEN_KEY, secret=self.TOKEN_SECRET, - is_approved=True - ) - - def _create_authorization_header(self): - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': self.token.key, - 'oauth_consumer_key': self.consumer.key - } - - req = oauth.Request(method="GET", url="http://example.com", parameters=params) - - signature_method = oauth.SignatureMethod_PLAINTEXT() - req.sign_request(signature_method, self.consumer, self.token) - - return req.to_header()["Authorization"] - - def _create_authorization_url_parameters(self): - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': self.token.key, - 'oauth_consumer_key': self.consumer.key - } - - req = oauth.Request(method="GET", url="http://example.com", parameters=params) - - signature_method = oauth.SignatureMethod_PLAINTEXT() - req.sign_request(signature_method, self.consumer, self.token) - return dict(req) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_passing_oauth(self): - """Ensure POSTing form over OAuth with correct credentials passes and does not require CSRF""" - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth/', {'example': 'example'}, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_repeated_nonce_failing_oauth(self): - """Ensure POSTing form over OAuth with repeated auth (same nonces and timestamp) credentials fails""" - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth/', {'example': 'example'}, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - # simulate reply attack auth header containes already used (nonce, timestamp) pair - response = self.csrf_client.post('/oauth/', {'example': 'example'}, HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_token_removed_failing_oauth(self): - """Ensure POSTing when there is no OAuth access token in db fails""" - self.token.delete() - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth/', {'example': 'example'}, HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_consumer_status_not_accepted_failing_oauth(self): - """Ensure POSTing when consumer status is anything other than ACCEPTED fails""" - for consumer_status in (self.consts.CANCELED, self.consts.PENDING, self.consts.REJECTED): - self.consumer.status = consumer_status - self.consumer.save() - - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth/', {'example': 'example'}, HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_with_request_token_failing_oauth(self): - """Ensure POSTing with unauthorized request token instead of access token fails""" - self.token.token_type = self.token.REQUEST - self.token.save() - - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth/', {'example': 'example'}, HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_with_urlencoded_parameters(self): - """Ensure POSTing with x-www-form-urlencoded auth parameters passes""" - params = self._create_authorization_url_parameters() - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth/', params, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_get_form_with_url_parameters(self): - """Ensure GETing with auth in url parameters passes""" - params = self._create_authorization_url_parameters() - response = self.csrf_client.get('/oauth/', params) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_hmac_sha1_signature_passes(self): - """Ensure POSTing using HMAC_SHA1 signature method passes""" - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': self.token.key, - 'oauth_consumer_key': self.consumer.key - } - - req = oauth.Request(method="POST", url="http://testserver/oauth/", parameters=params) - - signature_method = oauth.SignatureMethod_HMAC_SHA1() - req.sign_request(signature_method, self.consumer, self.token) - auth = req.to_header()["Authorization"] - - response = self.csrf_client.post('/oauth/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_get_form_with_readonly_resource_passing_auth(self): - """Ensure POSTing with a readonly scope instead of a write scope fails""" - read_only_access_token = self.token - read_only_access_token.scope.is_readonly = True - read_only_access_token.scope.save() - params = self._create_authorization_url_parameters() - response = self.csrf_client.get('/oauth-with-scope/', params) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_with_readonly_resource_failing_auth(self): - """Ensure POSTing with a readonly resource instead of a write scope fails""" - read_only_access_token = self.token - read_only_access_token.scope.is_readonly = True - read_only_access_token.scope.save() - params = self._create_authorization_url_parameters() - response = self.csrf_client.post('/oauth-with-scope/', params) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_post_form_with_write_resource_passing_auth(self): - """Ensure POSTing with a write resource succeed""" - read_write_access_token = self.token - read_write_access_token.scope.is_readonly = False - read_write_access_token.scope.save() - params = self._create_authorization_url_parameters() - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth-with-scope/', params, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_bad_consumer_key(self): - """Ensure POSTing using HMAC_SHA1 signature method passes""" - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': self.token.key, - 'oauth_consumer_key': 'badconsumerkey' - } - - req = oauth.Request(method="POST", url="http://testserver/oauth/", parameters=params) - - signature_method = oauth.SignatureMethod_HMAC_SHA1() - req.sign_request(signature_method, self.consumer, self.token) - auth = req.to_header()["Authorization"] - - response = self.csrf_client.post('/oauth/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - - @unittest.skipUnless(oauth_provider, 'django-oauth-plus not installed') - @unittest.skipUnless(oauth, 'oauth2 not installed') - def test_bad_token_key(self): - """Ensure POSTing using HMAC_SHA1 signature method passes""" - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': 'badtokenkey', - 'oauth_consumer_key': self.consumer.key - } - - req = oauth.Request(method="POST", url="http://testserver/oauth/", parameters=params) - - signature_method = oauth.SignatureMethod_HMAC_SHA1() - req.sign_request(signature_method, self.consumer, self.token) - auth = req.to_header()["Authorization"] - - response = self.csrf_client.post('/oauth/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - - -class OAuth2Tests(TestCase): - """OAuth 2.0 authentication""" - urls = 'tests.test_authentication' - - def setUp(self): - self.csrf_client = APIClient(enforce_csrf_checks=True) - self.username = 'john' - self.email = 'lennon@thebeatles.com' - self.password = 'password' - self.user = User.objects.create_user(self.username, self.email, self.password) - - self.CLIENT_ID = 'client_key' - self.CLIENT_SECRET = 'client_secret' - self.ACCESS_TOKEN = "access_token" - self.REFRESH_TOKEN = "refresh_token" - - self.oauth2_client = oauth2_provider.oauth2.models.Client.objects.create( - client_id=self.CLIENT_ID, - client_secret=self.CLIENT_SECRET, - redirect_uri='', - client_type=0, - name='example', - user=None, - ) - - self.access_token = oauth2_provider.oauth2.models.AccessToken.objects.create( - token=self.ACCESS_TOKEN, - client=self.oauth2_client, - user=self.user, - ) - self.refresh_token = oauth2_provider.oauth2.models.RefreshToken.objects.create( - user=self.user, - access_token=self.access_token, - client=self.oauth2_client - ) - - def _create_authorization_header(self, token=None): - return "Bearer {0}".format(token or self.access_token.token) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_with_wrong_authorization_header_token_type_failing(self): - """Ensure that a wrong token type lead to the correct HTTP error status code""" - auth = "Wrong token-type-obsviously" - response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_with_wrong_authorization_header_token_format_failing(self): - """Ensure that a wrong token format lead to the correct HTTP error status code""" - auth = "Bearer wrong token format" - response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_with_wrong_authorization_header_token_failing(self): - """Ensure that a wrong token lead to the correct HTTP error status code""" - auth = "Bearer wrong-token" - response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_with_wrong_authorization_header_token_missing(self): - """Ensure that a missing token lead to the correct HTTP error status code""" - auth = "Bearer" - response = self.csrf_client.get('/oauth2-test/', {}, HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 401) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_passing_auth(self): - """Ensure GETing form over OAuth with correct client credentials succeed""" - auth = self._create_authorization_header() - response = self.csrf_client.get('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_passing_auth_url_transport(self): - """Ensure GETing form over OAuth with correct client credentials in form data succeed""" - response = self.csrf_client.post( - '/oauth2-test/', - data={'access_token': self.access_token.token} - ) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_passing_auth_url_transport(self): - """Ensure GETing form over OAuth with correct client credentials in query succeed when DEBUG is True""" - query = urlencode({'access_token': self.access_token.token}) - response = self.csrf_client.get('/oauth2-test-debug/?%s' % query) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_get_form_failing_auth_url_transport(self): - """Ensure GETing form over OAuth with correct client credentials in query fails when DEBUG is False""" - query = urlencode({'access_token': self.access_token.token}) - response = self.csrf_client.get('/oauth2-test/?%s' % query) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_passing_auth(self): - """Ensure POSTing form over OAuth with correct credentials passes and does not require CSRF""" - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_token_removed_failing_auth(self): - """Ensure POSTing when there is no OAuth access token in db fails""" - self.access_token.delete() - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_with_refresh_token_failing_auth(self): - """Ensure POSTing with refresh token instead of access token fails""" - auth = self._create_authorization_header(token=self.refresh_token.token) - response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_with_expired_access_token_failing_auth(self): - """Ensure POSTing with expired access token fails with an 'Invalid token' error""" - self.access_token.expires = datetime.datetime.now() - datetime.timedelta(seconds=10) # 10 seconds late - self.access_token.save() - auth = self._create_authorization_header() - response = self.csrf_client.post('/oauth2-test/', HTTP_AUTHORIZATION=auth) - self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) - self.assertIn('Invalid token', response.content) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_with_invalid_scope_failing_auth(self): - """Ensure POSTing with a readonly scope instead of a write scope fails""" - read_only_access_token = self.access_token - read_only_access_token.scope = oauth2_provider_scope.SCOPE_NAME_DICT['read'] - read_only_access_token.save() - auth = self._create_authorization_header(token=read_only_access_token.token) - response = self.csrf_client.get('/oauth2-with-scope-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - response = self.csrf_client.post('/oauth2-with-scope-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - - @unittest.skipUnless(oauth2_provider, 'django-oauth2-provider not installed') - def test_post_form_with_valid_scope_passing_auth(self): - """Ensure POSTing with a write scope succeed""" - read_write_access_token = self.access_token - read_write_access_token.scope = oauth2_provider_scope.SCOPE_NAME_DICT['write'] - read_write_access_token.save() - auth = self._create_authorization_header(token=read_write_access_token.token) - response = self.csrf_client.post('/oauth2-with-scope-test/', HTTP_AUTHORIZATION=auth) - self.assertEqual(response.status_code, 200) - - class FailingAuthAccessedInRenderer(TestCase): def setUp(self): class AuthAccessingRenderer(renderers.BaseRenderer): From afaa52a378705b7f0475d5ece04a2cf49af4b7c2 Mon Sep 17 00:00:00 2001 From: Jharrod LaFon Date: Fri, 5 Sep 2014 15:42:29 -0700 Subject: [PATCH 02/29] Removes OAuth dependencies from tox configurations --- tox.ini | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tox.ini b/tox.ini index d40a70799..a502af972 100644 --- a/tox.ini +++ b/tox.ini @@ -45,9 +45,6 @@ basepython = python2.7 deps = Django==1.7 django-filter==0.7 defusedxml==0.3 - # django-oauth-plus==2.2.1 - # oauth2==1.5.211 - # django-oauth2-provider==0.2.4 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 @@ -81,9 +78,6 @@ basepython = python2.7 deps = Django==1.6.3 django-filter==0.7 defusedxml==0.3 - django-oauth-plus==2.2.1 - oauth2==1.5.211 - django-oauth2-provider==0.2.4 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 @@ -93,9 +87,6 @@ basepython = python2.6 deps = Django==1.6.3 django-filter==0.7 defusedxml==0.3 - django-oauth-plus==2.2.1 - oauth2==1.5.211 - django-oauth2-provider==0.2.4 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 @@ -129,9 +120,6 @@ basepython = python2.7 deps = django==1.5.6 django-filter==0.7 defusedxml==0.3 - django-oauth-plus==2.2.1 - oauth2==1.5.211 - django-oauth2-provider==0.2.3 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 @@ -141,9 +129,6 @@ basepython = python2.6 deps = django==1.5.6 django-filter==0.7 defusedxml==0.3 - django-oauth-plus==2.2.1 - oauth2==1.5.211 - django-oauth2-provider==0.2.3 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 @@ -153,9 +138,6 @@ basepython = python2.7 deps = django==1.4.11 django-filter==0.7 defusedxml==0.3 - django-oauth-plus==2.2.1 - oauth2==1.5.211 - django-oauth2-provider==0.2.3 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 @@ -165,9 +147,6 @@ basepython = python2.6 deps = django==1.4.11 django-filter==0.7 defusedxml==0.3 - django-oauth-plus==2.2.1 - oauth2==1.5.211 - django-oauth2-provider==0.2.3 django-guardian==1.2.3 Pillow==2.3.0 pytest-django==2.6.1 From 731c8421afe3093a78cdabb9c3cc28fa52cd1c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sat, 29 Nov 2014 14:43:05 -0400 Subject: [PATCH 03/29] Remove YAML support from core --- README.md | 49 +++++++++-------- docs/api-guide/parsers.md | 24 +++------ docs/api-guide/renderers.md | 42 ++------------- docs/api-guide/settings.md | 4 +- docs/api-guide/testing.md | 4 +- docs/index.md | 2 - docs/tutorial/2-requests-and-responses.md | 2 +- requirements-test.txt | 1 - rest_framework/compat.py | 7 --- rest_framework/parsers.py | 25 +-------- rest_framework/renderers.py | 25 +-------- rest_framework/settings.py | 4 +- rest_framework/utils/encoders.py | 65 +---------------------- tests/test_renderers.py | 57 ++------------------ tests/test_templatetags.py | 13 +---- tox.ini | 1 - 16 files changed, 52 insertions(+), 273 deletions(-) diff --git a/README.md b/README.md index c86bb65ff..aafcb29b9 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Add `'rest_framework'` to your `INSTALLED_APPS` setting. Let's take a look at a quick example of using REST framework to build a simple model-backed API for accessing users and groups. -Startup up a new project like so... +Startup up a new project like so... pip install django pip install djangorestframework @@ -79,7 +79,7 @@ class UserViewSet(viewsets.ModelViewSet): queryset = User.objects.all() serializer_class = UserSerializer - + # Routers provide a way of automatically determining the URL conf. router = routers.DefaultRouter() router.register(r'users', UserViewSet) @@ -100,7 +100,7 @@ Add the following to your `settings.py` module: ```python INSTALLED_APPS = ( ... # Make sure to include the default installed apps here. - 'rest_framework', + 'rest_framework', ) REST_FRAMEWORK = { @@ -123,10 +123,10 @@ You can also interact with the API using command line tools such as [`curl`](htt $ curl -H 'Accept: application/json; indent=4' -u admin:password http://127.0.0.1:8000/users/ [ { - "url": "http://127.0.0.1:8000/users/1/", - "username": "admin", - "email": "admin@example.com", - "is_staff": true, + "url": "http://127.0.0.1:8000/users/1/", + "username": "admin", + "email": "admin@example.com", + "is_staff": true, } ] @@ -134,10 +134,10 @@ Or to create a new user: $ curl -X POST -d username=new -d email=new@example.com -d is_staff=false -H 'Accept: application/json; indent=4' -u admin:password http://127.0.0.1:8000/users/ { - "url": "http://127.0.0.1:8000/users/2/", - "username": "new", - "email": "new@example.com", - "is_staff": false, + "url": "http://127.0.0.1:8000/users/2/", + "username": "new", + "email": "new@example.com", + "is_staff": false, } # Documentation & Support @@ -159,24 +159,24 @@ Send a description of the issue via email to [rest-framework-security@googlegrou Copyright (c) 2011-2014, Tom Christie All rights reserved. -Redistribution and use in source and binary forms, with or without +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -Redistributions of source code must retain the above copyright notice, this +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @@ -214,7 +214,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [docs]: http://www.django-rest-framework.org/ [urlobject]: https://github.com/zacharyvoase/urlobject [markdown]: http://pypi.python.org/pypi/Markdown/ -[pyyaml]: http://pypi.python.org/pypi/PyYAML [defusedxml]: https://pypi.python.org/pypi/defusedxml [django-filter]: http://pypi.python.org/pypi/django-filter [security-mail]: mailto:rest-framework-security@googlegroups.com diff --git a/docs/api-guide/parsers.md b/docs/api-guide/parsers.md index 73e3a7057..1e134c772 100644 --- a/docs/api-guide/parsers.md +++ b/docs/api-guide/parsers.md @@ -26,26 +26,26 @@ As an example, if you are sending `json` encoded data using jQuery with the [.aj ## Setting the parsers -The default set of parsers may be set globally, using the `DEFAULT_PARSER_CLASSES` setting. For example, the following settings would allow requests with `YAML` content. +The default set of parsers may be set globally, using the `DEFAULT_PARSER_CLASSES` setting. For example, the following settings would allow requests with `JSON` content. REST_FRAMEWORK = { 'DEFAULT_PARSER_CLASSES': ( - 'rest_framework.parsers.YAMLParser', + 'rest_framework.parsers.JSONParser', ) } You can also set the parsers used for an individual view, or viewset, using the `APIView` class based views. - from rest_framework.parsers import YAMLParser + from rest_framework.parsers import JSONParser from rest_framework.response import Response from rest_framework.views import APIView class ExampleView(APIView): """ - A view that can accept POST requests with YAML content. + A view that can accept POST requests with JSON content. """ - parser_classes = (YAMLParser,) + parser_classes = (JSONParser,) def post(self, request, format=None): return Response({'received data': request.data}) @@ -53,10 +53,10 @@ using the `APIView` class based views. Or, if you're using the `@api_view` decorator with function based views. @api_view(['POST']) - @parser_classes((YAMLParser,)) + @parser_classes((JSONParser,)) def example_view(request, format=None): """ - A view that can accept POST requests with YAML content. + A view that can accept POST requests with JSON content. """ return Response({'received data': request.data}) @@ -70,14 +70,6 @@ Parses `JSON` request content. **.media_type**: `application/json` -## YAMLParser - -Parses `YAML` request content. - -Requires the `pyyaml` package to be installed. - -**.media_type**: `application/yaml` - ## XMLParser Parses REST framework's default style of `XML` request content. @@ -161,7 +153,7 @@ By default this will include the following keys: `view`, `request`, `args`, `kwa ## Example -The following is an example plaintext parser that will populate the `request.data` property with a string representing the body of the request. +The following is an example plaintext parser that will populate the `request.data` property with a string representing the body of the request. class PlainTextParser(BaseParser): """ diff --git a/docs/api-guide/renderers.md b/docs/api-guide/renderers.md index 035ec1d27..aa8da0886 100644 --- a/docs/api-guide/renderers.md +++ b/docs/api-guide/renderers.md @@ -18,11 +18,11 @@ For more information see the documentation on [content negotiation][conneg]. ## Setting the renderers -The default set of renderers may be set globally, using the `DEFAULT_RENDERER_CLASSES` setting. For example, the following settings would use `YAML` as the main media type and also include the self describing API. +The default set of renderers may be set globally, using the `DEFAULT_RENDERER_CLASSES` setting. For example, the following settings would use `JSON` as the main media type and also include the self describing API. REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.YAMLRenderer', + 'rest_framework.renderers.JSONRenderer', 'rest_framework.renderers.BrowsableAPIRenderer', ) } @@ -31,15 +31,15 @@ You can also set the renderers used for an individual view, or viewset, using the `APIView` class based views. from django.contrib.auth.models import User - from rest_framework.renderers import JSONRenderer, YAMLRenderer + from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework.views import APIView class UserCountView(APIView): """ - A view that returns the count of active users, in JSON or YAML. + A view that returns the count of active users in JSON. """ - renderer_classes = (JSONRenderer, YAMLRenderer) + renderer_classes = (JSONRenderer, ) def get(self, request, format=None): user_count = User.objects.filter(active=True).count() @@ -113,38 +113,6 @@ The `jsonp` approach is essentially a browser hack, and is [only appropriate for **.charset**: `utf-8` -## YAMLRenderer - -Renders the request data into `YAML`. - -Requires the `pyyaml` package to be installed. - -Note that non-ascii characters will be rendered using `\uXXXX` character escape. For example: - - unicode black star: "\u2605" - -**.media_type**: `application/yaml` - -**.format**: `'.yaml'` - -**.charset**: `utf-8` - -## UnicodeYAMLRenderer - -Renders the request data into `YAML`. - -Requires the `pyyaml` package to be installed. - -Note that non-ascii characters will not be character escaped. For example: - - unicode black star: ★ - -**.media_type**: `application/yaml` - -**.format**: `'.yaml'` - -**.charset**: `utf-8` - ## XMLRenderer Renders REST framework's default style of `XML` response content. diff --git a/docs/api-guide/settings.md b/docs/api-guide/settings.md index 9005511b7..623d89fbc 100644 --- a/docs/api-guide/settings.md +++ b/docs/api-guide/settings.md @@ -12,10 +12,10 @@ For example your project's `settings.py` file might include something like this: REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.YAMLRenderer', + 'rest_framework.renderers.JSONRenderer', ), 'DEFAULT_PARSER_CLASSES': ( - 'rest_framework.parsers.YAMLParser', + 'rest_framework.parsers.JSONParser', ) } diff --git a/docs/api-guide/testing.md b/docs/api-guide/testing.md index d059fdab5..cd8c7820a 100644 --- a/docs/api-guide/testing.md +++ b/docs/api-guide/testing.md @@ -255,14 +255,14 @@ The default format used to make test requests may be set using the `TEST_REQUEST If you need to test requests using something other than multipart or json requests, you can do so by setting the `TEST_REQUEST_RENDERER_CLASSES` setting. -For example, to add support for using `format='yaml'` in test requests, you might have something like this in your `settings.py` file. +For example, to add support for using `format='html'` in test requests, you might have something like this in your `settings.py` file. REST_FRAMEWORK = { ... 'TEST_REQUEST_RENDERER_CLASSES': ( 'rest_framework.renderers.MultiPartRenderer', 'rest_framework.renderers.JSONRenderer', - 'rest_framework.renderers.YAMLRenderer' + 'rest_framework.renderers.TemplateHTMLRenderer' ) } diff --git a/docs/index.md b/docs/index.md index b5257c734..c2836dbb9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -54,7 +54,6 @@ REST framework requires the following: The following packages are optional: * [Markdown][markdown] (2.1.0+) - Markdown support for the browsable API. -* [PyYAML][yaml] (3.10+) - YAML content-type support. * [defusedxml][defusedxml] (0.3+) - XML content-type support. * [django-filter][django-filter] (0.5.4+) - Filtering support. * [django-oauth-plus][django-oauth-plus] (2.0+) and [oauth2][oauth2] (1.5.211+) - OAuth 1.0a support. @@ -258,7 +257,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [mozilla]: http://www.mozilla.org/en-US/about/ [eventbrite]: https://www.eventbrite.co.uk/about/ [markdown]: http://pypi.python.org/pypi/Markdown/ -[yaml]: http://pypi.python.org/pypi/PyYAML [defusedxml]: https://pypi.python.org/pypi/defusedxml [django-filter]: http://pypi.python.org/pypi/django-filter [oauth2]: https://github.com/simplegeo/python-oauth2 diff --git a/docs/tutorial/2-requests-and-responses.md b/docs/tutorial/2-requests-and-responses.md index f377c7122..06a684b17 100644 --- a/docs/tutorial/2-requests-and-responses.md +++ b/docs/tutorial/2-requests-and-responses.md @@ -92,7 +92,7 @@ Here is the view for an individual snippet, in the `views.py` module. This should all feel very familiar - it is not a lot different from working with regular Django views. -Notice that we're no longer explicitly tying our requests or responses to a given content type. `request.data` can handle incoming `json` requests, but it can also handle `yaml` and other formats. Similarly we're returning response objects with data, but allowing REST framework to render the response into the correct content type for us. +Notice that we're no longer explicitly tying our requests or responses to a given content type. `request.data` can handle incoming `json` requests, but it can also handle other formats. Similarly we're returning response objects with data, but allowing REST framework to render the response into the correct content type for us. ## Adding optional format suffixes to our URLs diff --git a/requirements-test.txt b/requirements-test.txt index 06c8849a8..bd09211ea 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,7 +6,6 @@ flake8==2.2.2 # Optional packages markdown>=2.1.0 -PyYAML>=3.10 defusedxml>=0.3 django-guardian==1.2.4 django-filter>=0.5.4 diff --git a/rest_framework/compat.py b/rest_framework/compat.py index 5bd85e743..52db96257 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -237,13 +237,6 @@ except ImportError: apply_markdown = None -# Yaml is optional -try: - import yaml -except ImportError: - yaml = None - - # XML is optional try: import defusedxml.ElementTree as etree diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index ccb82f03b..e6bb75f6d 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -12,7 +12,7 @@ from django.http import QueryDict from django.http.multipartparser import MultiPartParser as DjangoMultiPartParser from django.http.multipartparser import MultiPartParserError, parse_header, ChunkIter from django.utils import six -from rest_framework.compat import etree, yaml, force_text, urlparse +from rest_framework.compat import etree, force_text, urlparse from rest_framework.exceptions import ParseError from rest_framework import renderers import json @@ -65,29 +65,6 @@ class JSONParser(BaseParser): raise ParseError('JSON parse error - %s' % six.text_type(exc)) -class YAMLParser(BaseParser): - """ - Parses YAML-serialized data. - """ - - media_type = 'application/yaml' - - def parse(self, stream, media_type=None, parser_context=None): - """ - Parses the incoming bytestream as YAML and returns the resulting data. - """ - assert yaml, 'YAMLParser requires pyyaml to be installed' - - parser_context = parser_context or {} - encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) - - try: - data = stream.read().decode(encoding) - return yaml.safe_load(data) - except (ValueError, yaml.parser.ParserError) as exc: - raise ParseError('YAML parse error - %s' % six.text_type(exc)) - - class FormParser(BaseParser): """ Parser for form data. diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index e87d16d0d..a6e4f1bb9 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -19,7 +19,7 @@ from django.utils import six from django.utils.xmlutils import SimplerXMLGenerator from rest_framework import exceptions, serializers, status, VERSION from rest_framework.compat import ( - SHORT_SEPARATORS, LONG_SEPARATORS, StringIO, smart_text, yaml + SHORT_SEPARATORS, LONG_SEPARATORS, StringIO, smart_text ) from rest_framework.exceptions import ParseError from rest_framework.settings import api_settings @@ -189,29 +189,6 @@ class XMLRenderer(BaseRenderer): xml.characters(smart_text(data)) -class YAMLRenderer(BaseRenderer): - """ - Renderer which serializes to YAML. - """ - - media_type = 'application/yaml' - format = 'yaml' - encoder = encoders.SafeDumper - charset = 'utf-8' - ensure_ascii = False - - def render(self, data, accepted_media_type=None, renderer_context=None): - """ - Renders `data` into serialized YAML. - """ - assert yaml, 'YAMLRenderer requires pyyaml to be installed' - - if data is None: - return '' - - return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder, allow_unicode=not self.ensure_ascii) - - class TemplateHTMLRenderer(BaseRenderer): """ An HTML renderer for use with templates. diff --git a/rest_framework/settings.py b/rest_framework/settings.py index 1e8c27fc3..3abc1fe85 100644 --- a/rest_framework/settings.py +++ b/rest_framework/settings.py @@ -5,11 +5,11 @@ For example your project's `settings.py` file might look like this: REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', - 'rest_framework.renderers.YAMLRenderer', + 'rest_framework.renderers.TemplateHTMLRenderer', ) 'DEFAULT_PARSER_CLASSES': ( 'rest_framework.parsers.JSONParser', - 'rest_framework.parsers.YAMLParser', + 'rest_framework.parsers.TemplateHTMLRenderer', ) } diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index 4d6bb3a34..2c97f1d7a 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -5,10 +5,9 @@ from __future__ import unicode_literals from django.db.models.query import QuerySet from django.utils import six, timezone from django.utils.functional import Promise -from rest_framework.compat import force_text, OrderedDict +from rest_framework.compat import force_text import datetime import decimal -import types import json @@ -56,65 +55,3 @@ class JSONEncoder(json.JSONEncoder): elif hasattr(obj, '__iter__'): return tuple(item for item in obj) return super(JSONEncoder, self).default(obj) - - -try: - import yaml -except ImportError: - SafeDumper = None -else: - # Adapted from http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py - class SafeDumper(yaml.SafeDumper): - """ - Handles decimals as strings. - Handles OrderedDicts as usual dicts, but preserves field order, rather - than the usual behaviour of sorting the keys. - """ - def represent_decimal(self, data): - return self.represent_scalar('tag:yaml.org,2002:str', six.text_type(data)) - - def represent_mapping(self, tag, mapping, flow_style=None): - value = [] - node = yaml.MappingNode(tag, value, flow_style=flow_style) - if self.alias_key is not None: - self.represented_objects[self.alias_key] = node - best_style = True - if hasattr(mapping, 'items'): - mapping = list(mapping.items()) - if not isinstance(mapping, OrderedDict): - mapping.sort() - for item_key, item_value in mapping: - node_key = self.represent_data(item_key) - node_value = self.represent_data(item_value) - if not (isinstance(node_key, yaml.ScalarNode) and not node_key.style): - best_style = False - if not (isinstance(node_value, yaml.ScalarNode) and not node_value.style): - best_style = False - value.append((node_key, node_value)) - if flow_style is None: - if self.default_flow_style is not None: - node.flow_style = self.default_flow_style - else: - node.flow_style = best_style - return node - - SafeDumper.add_representer( - decimal.Decimal, - SafeDumper.represent_decimal - ) - SafeDumper.add_representer( - OrderedDict, - yaml.representer.SafeRepresenter.represent_dict - ) - # SafeDumper.add_representer( - # DictWithMetadata, - # yaml.representer.SafeRepresenter.represent_dict - # ) - # SafeDumper.add_representer( - # OrderedDictWithMetadata, - # yaml.representer.SafeRepresenter.represent_dict - # ) - SafeDumper.add_representer( - types.GeneratorType, - yaml.representer.SafeRepresenter.represent_list - ) diff --git a/tests/test_renderers.py b/tests/test_renderers.py index 416d7f224..0603f800b 100644 --- a/tests/test_renderers.py +++ b/tests/test_renderers.py @@ -9,12 +9,12 @@ from django.test import TestCase from django.utils import six, unittest from django.utils.translation import ugettext_lazy as _ from rest_framework import status, permissions -from rest_framework.compat import yaml, etree, StringIO, BytesIO +from rest_framework.compat import etree, StringIO from rest_framework.response import Response from rest_framework.views import APIView -from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \ - XMLRenderer, JSONPRenderer, BrowsableAPIRenderer -from rest_framework.parsers import YAMLParser, XMLParser +from rest_framework.renderers import BaseRenderer, JSONRenderer, XMLRenderer, \ + JSONPRenderer, BrowsableAPIRenderer +from rest_framework.parsers import XMLParser from rest_framework.settings import api_settings from rest_framework.test import APIRequestFactory from collections import MutableMapping @@ -452,55 +452,6 @@ class JSONPRendererTests(TestCase): ) -if yaml: - _yaml_repr = 'foo: [bar, baz]\n' - - class YAMLRendererTests(TestCase): - """ - Tests specific to the YAML Renderer - """ - - def test_render(self): - """ - Test basic YAML rendering. - """ - obj = {'foo': ['bar', 'baz']} - renderer = YAMLRenderer() - content = renderer.render(obj, 'application/yaml') - self.assertEqual(content.decode('utf-8'), _yaml_repr) - - def test_render_and_parse(self): - """ - Test rendering and then parsing returns the original object. - IE obj -> render -> parse -> obj. - """ - obj = {'foo': ['bar', 'baz']} - - renderer = YAMLRenderer() - parser = YAMLParser() - - content = renderer.render(obj, 'application/yaml') - data = parser.parse(BytesIO(content)) - self.assertEqual(obj, data) - - def test_render_decimal(self): - """ - Test YAML decimal rendering. - """ - renderer = YAMLRenderer() - content = renderer.render({'field': Decimal('111.2')}, 'application/yaml') - self.assertYAMLContains(content.decode('utf-8'), "field: '111.2'") - - def assertYAMLContains(self, content, string): - self.assertTrue(string in content, '%r not in %r' % (string, content)) - - def test_proper_encoding(self): - obj = {'countries': ['United Kingdom', 'France', 'España']} - renderer = YAMLRenderer() - content = renderer.render(obj, 'application/yaml') - self.assertEqual(content.strip(), 'countries: [United Kingdom, France, España]'.encode('utf-8')) - - class XMLRendererTestCase(TestCase): """ Tests specific to the XML Renderer diff --git a/tests/test_templatetags.py b/tests/test_templatetags.py index b04a937e0..0cee91f19 100644 --- a/tests/test_templatetags.py +++ b/tests/test_templatetags.py @@ -54,7 +54,7 @@ class Issue1386Tests(TestCase): class URLizerTests(TestCase): """ - Test if both JSON and YAML URLs are transformed into links well + Test if JSON URLs are transformed into links well """ def _urlize_dict_check(self, data): """ @@ -73,14 +73,3 @@ class URLizerTests(TestCase): data['"foo_set": [\n "http://api/foos/1/"\n], '] = \ '"foo_set": [\n "http://api/foos/1/"\n], ' self._urlize_dict_check(data) - - def test_yaml_with_url(self): - """ - Test if YAML URLs are transformed into links well - """ - data = {} - data['''{users: 'http://api/users/'}'''] = \ - '''{users: 'http://api/users/'}''' - data['''foo_set: ['http://api/foos/1/']'''] = \ - '''foo_set: ['http://api/foos/1/']''' - self._urlize_dict_check(data) diff --git a/tox.ini b/tox.ini index d5cb9ef94..edfeb33d8 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,6 @@ deps = django-filter==0.7 defusedxml==0.3 markdown>=2.1.0 - PyYAML>=3.10 [testenv:py27-flake8] deps = From fe745b96163282e492f17a6b003418b81350333f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sat, 29 Nov 2014 14:55:33 -0400 Subject: [PATCH 04/29] Remove JSONP support from core --- docs/api-guide/renderers.md | 28 ++---------------- rest_framework/renderers.py | 34 ---------------------- tests/test_renderers.py | 58 +------------------------------------ 3 files changed, 3 insertions(+), 117 deletions(-) diff --git a/docs/api-guide/renderers.md b/docs/api-guide/renderers.md index 035ec1d27..a77b9db26 100644 --- a/docs/api-guide/renderers.md +++ b/docs/api-guide/renderers.md @@ -49,10 +49,10 @@ using the `APIView` class based views. Or, if you're using the `@api_view` decorator with function based views. @api_view(['GET']) - @renderer_classes((JSONRenderer, JSONPRenderer)) + @renderer_classes((JSONRenderer,)) def user_count_view(request, format=None): """ - A view that returns the count of active users, in JSON or JSONp. + A view that returns the count of active users in JSON. """ user_count = User.objects.filter(active=True).count() content = {'user_count': user_count} @@ -93,26 +93,6 @@ The default JSON encoding style can be altered using the `UNICODE_JSON` and `COM **.charset**: `None` -## JSONPRenderer - -Renders the request data into `JSONP`. The `JSONP` media type provides a mechanism of allowing cross-domain AJAX requests, by wrapping a `JSON` response in a javascript callback. - -The javascript callback function must be set by the client including a `callback` URL query parameter. For example `http://example.com/api/users?callback=jsonpCallback`. If the callback function is not explicitly set by the client it will default to `'callback'`. - ---- - -**Warning**: If you require cross-domain AJAX requests, you should almost certainly be using the more modern approach of [CORS][cors] as an alternative to `JSONP`. See the [CORS documentation][cors-docs] for more details. - -The `jsonp` approach is essentially a browser hack, and is [only appropriate for globally readable API endpoints][jsonp-security], where `GET` requests are unauthenticated and do not require any user permissions. - ---- - -**.media_type**: `application/javascript` - -**.format**: `'.jsonp'` - -**.charset**: `utf-8` - ## YAMLRenderer Renders the request data into `YAML`. @@ -433,10 +413,6 @@ Comma-separated values are a plain-text tabular data format, that can be easily [cite]: https://docs.djangoproject.com/en/dev/ref/template-response/#the-rendering-process [conneg]: content-negotiation.md [browser-accept-headers]: http://www.gethifi.com/blog/browser-rest-http-accept-headers -[rfc4627]: http://www.ietf.org/rfc/rfc4627.txt -[cors]: http://www.w3.org/TR/cors/ -[cors-docs]: ../topics/ajax-csrf-cors.md -[jsonp-security]: http://stackoverflow.com/questions/613962/is-jsonp-safe-to-use [testing]: testing.md [HATEOAS]: http://timelessrepo.com/haters-gonna-hateoas [quote]: http://roy.gbiv.com/untangled/2008/rest-apis-must-be-hypertext-driven diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index e87d16d0d..ab6f251c5 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -106,40 +106,6 @@ class JSONRenderer(BaseRenderer): return ret -class JSONPRenderer(JSONRenderer): - """ - Renderer which serializes to json, - wrapping the json output in a callback function. - """ - - media_type = 'application/javascript' - format = 'jsonp' - callback_parameter = 'callback' - default_callback = 'callback' - charset = 'utf-8' - - def get_callback(self, renderer_context): - """ - Determine the name of the callback to wrap around the json output. - """ - request = renderer_context.get('request', None) - params = request and request.query_params or {} - return params.get(self.callback_parameter, self.default_callback) - - def render(self, data, accepted_media_type=None, renderer_context=None): - """ - Renders into jsonp, wrapping the json output in a callback function. - - Clients may set the callback function name using a query parameter - on the URL, for example: ?callback=exampleCallbackName - """ - renderer_context = renderer_context or {} - callback = self.get_callback(renderer_context) - json = super(JSONPRenderer, self).render(data, accepted_media_type, - renderer_context) - return callback.encode(self.charset) + b'(' + json + b');' - - class XMLRenderer(BaseRenderer): """ Renderer which serializes to XML. diff --git a/tests/test_renderers.py b/tests/test_renderers.py index 416d7f224..15f15dcd2 100644 --- a/tests/test_renderers.py +++ b/tests/test_renderers.py @@ -13,7 +13,7 @@ from rest_framework.compat import yaml, etree, StringIO, BytesIO from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \ - XMLRenderer, JSONPRenderer, BrowsableAPIRenderer + XMLRenderer, BrowsableAPIRenderer from rest_framework.parsers import YAMLParser, XMLParser from rest_framework.settings import api_settings from rest_framework.test import APIRequestFactory @@ -106,8 +106,6 @@ urlpatterns = patterns( url(r'^.*\.(?P.+)$', MockView.as_view(renderer_classes=[RendererA, RendererB])), url(r'^$', MockView.as_view(renderer_classes=[RendererA, RendererB])), url(r'^cache$', MockGETView.as_view()), - url(r'^jsonp/jsonrenderer$', MockGETView.as_view(renderer_classes=[JSONRenderer, JSONPRenderer])), - url(r'^jsonp/nojsonrenderer$', MockGETView.as_view(renderer_classes=[JSONPRenderer])), url(r'^parseerror$', MockPOSTView.as_view(renderer_classes=[JSONRenderer, BrowsableAPIRenderer])), url(r'^html$', HTMLView.as_view()), url(r'^html1$', HTMLView1.as_view()), @@ -398,60 +396,6 @@ class AsciiJSONRendererTests(TestCase): self.assertEqual(content, '{"countries":["United Kingdom","France","Espa\\u00f1a"]}'.encode('utf-8')) -class JSONPRendererTests(TestCase): - """ - Tests specific to the JSONP Renderer - """ - - urls = 'tests.test_renderers' - - def test_without_callback_with_json_renderer(self): - """ - Test JSONP rendering with View JSON Renderer. - """ - resp = self.client.get( - '/jsonp/jsonrenderer', - HTTP_ACCEPT='application/javascript' - ) - self.assertEqual(resp.status_code, status.HTTP_200_OK) - self.assertEqual(resp['Content-Type'], 'application/javascript; charset=utf-8') - self.assertEqual( - resp.content, - ('callback(%s);' % _flat_repr).encode('ascii') - ) - - def test_without_callback_without_json_renderer(self): - """ - Test JSONP rendering without View JSON Renderer. - """ - resp = self.client.get( - '/jsonp/nojsonrenderer', - HTTP_ACCEPT='application/javascript' - ) - self.assertEqual(resp.status_code, status.HTTP_200_OK) - self.assertEqual(resp['Content-Type'], 'application/javascript; charset=utf-8') - self.assertEqual( - resp.content, - ('callback(%s);' % _flat_repr).encode('ascii') - ) - - def test_with_callback(self): - """ - Test JSONP rendering with callback function name. - """ - callback_func = 'myjsonpcallback' - resp = self.client.get( - '/jsonp/nojsonrenderer?callback=' + callback_func, - HTTP_ACCEPT='application/javascript' - ) - self.assertEqual(resp.status_code, status.HTTP_200_OK) - self.assertEqual(resp['Content-Type'], 'application/javascript; charset=utf-8') - self.assertEqual( - resp.content, - ('%s(%s);' % (callback_func, _flat_repr)).encode('ascii') - ) - - if yaml: _yaml_repr = 'foo: [bar, baz]\n' From 7f9dc736728baf92a3198a7f90bd302fff240373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=CC=81=20Padilla?= Date: Sat, 29 Nov 2014 14:50:51 -0400 Subject: [PATCH 05/29] Remove XML support from core --- README.md | 49 ++++++++--------- docs/api-guide/parsers.md | 14 +---- docs/api-guide/renderers.md | 14 ----- docs/index.md | 2 - requirements-test.txt | 1 - rest_framework/compat.py | 7 --- rest_framework/parsers.py | 76 +------------------------ rest_framework/renderers.py | 54 +----------------- tests/test_parsers.py | 62 +-------------------- tests/test_renderers.py | 107 ++---------------------------------- 10 files changed, 32 insertions(+), 354 deletions(-) diff --git a/README.md b/README.md index c86bb65ff..83d16030d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Add `'rest_framework'` to your `INSTALLED_APPS` setting. Let's take a look at a quick example of using REST framework to build a simple model-backed API for accessing users and groups. -Startup up a new project like so... +Startup up a new project like so... pip install django pip install djangorestframework @@ -79,7 +79,7 @@ class UserViewSet(viewsets.ModelViewSet): queryset = User.objects.all() serializer_class = UserSerializer - + # Routers provide a way of automatically determining the URL conf. router = routers.DefaultRouter() router.register(r'users', UserViewSet) @@ -100,7 +100,7 @@ Add the following to your `settings.py` module: ```python INSTALLED_APPS = ( ... # Make sure to include the default installed apps here. - 'rest_framework', + 'rest_framework', ) REST_FRAMEWORK = { @@ -123,10 +123,10 @@ You can also interact with the API using command line tools such as [`curl`](htt $ curl -H 'Accept: application/json; indent=4' -u admin:password http://127.0.0.1:8000/users/ [ { - "url": "http://127.0.0.1:8000/users/1/", - "username": "admin", - "email": "admin@example.com", - "is_staff": true, + "url": "http://127.0.0.1:8000/users/1/", + "username": "admin", + "email": "admin@example.com", + "is_staff": true, } ] @@ -134,10 +134,10 @@ Or to create a new user: $ curl -X POST -d username=new -d email=new@example.com -d is_staff=false -H 'Accept: application/json; indent=4' -u admin:password http://127.0.0.1:8000/users/ { - "url": "http://127.0.0.1:8000/users/2/", - "username": "new", - "email": "new@example.com", - "is_staff": false, + "url": "http://127.0.0.1:8000/users/2/", + "username": "new", + "email": "new@example.com", + "is_staff": false, } # Documentation & Support @@ -159,24 +159,24 @@ Send a description of the issue via email to [rest-framework-security@googlegrou Copyright (c) 2011-2014, Tom Christie All rights reserved. -Redistribution and use in source and binary forms, with or without +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -Redistributions of source code must retain the above copyright notice, this +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @@ -215,6 +215,5 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [urlobject]: https://github.com/zacharyvoase/urlobject [markdown]: http://pypi.python.org/pypi/Markdown/ [pyyaml]: http://pypi.python.org/pypi/PyYAML -[defusedxml]: https://pypi.python.org/pypi/defusedxml [django-filter]: http://pypi.python.org/pypi/django-filter [security-mail]: mailto:rest-framework-security@googlegroups.com diff --git a/docs/api-guide/parsers.md b/docs/api-guide/parsers.md index 73e3a7057..32819146e 100644 --- a/docs/api-guide/parsers.md +++ b/docs/api-guide/parsers.md @@ -78,18 +78,6 @@ Requires the `pyyaml` package to be installed. **.media_type**: `application/yaml` -## XMLParser - -Parses REST framework's default style of `XML` request content. - -Note that the `XML` markup language is typically used as the base language for more strictly defined domain-specific languages, such as `RSS`, `Atom`, and `XHTML`. - -If you are considering using `XML` for your API, you may want to consider implementing a custom renderer and parser for your specific requirements, and using an existing domain-specific media-type, or creating your own custom XML-based media-type. - -Requires the `defusedxml` package to be installed. - -**.media_type**: `application/xml` - ## FormParser Parses HTML form content. `request.data` will be populated with a `QueryDict` of data. @@ -161,7 +149,7 @@ By default this will include the following keys: `view`, `request`, `args`, `kwa ## Example -The following is an example plaintext parser that will populate the `request.data` property with a string representing the body of the request. +The following is an example plaintext parser that will populate the `request.data` property with a string representing the body of the request. class PlainTextParser(BaseParser): """ diff --git a/docs/api-guide/renderers.md b/docs/api-guide/renderers.md index 035ec1d27..47bf2e601 100644 --- a/docs/api-guide/renderers.md +++ b/docs/api-guide/renderers.md @@ -145,20 +145,6 @@ Note that non-ascii characters will not be character escaped. For example: **.charset**: `utf-8` -## XMLRenderer - -Renders REST framework's default style of `XML` response content. - -Note that the `XML` markup language is used typically used as the base language for more strictly defined domain-specific languages, such as `RSS`, `Atom`, and `XHTML`. - -If you are considering using `XML` for your API, you may want to consider implementing a custom renderer and parser for your specific requirements, and using an existing domain-specific media-type, or creating your own custom XML-based media-type. - -**.media_type**: `application/xml` - -**.format**: `'.xml'` - -**.charset**: `utf-8` - ## TemplateHTMLRenderer Renders data to HTML, using Django's standard template rendering. diff --git a/docs/index.md b/docs/index.md index b5257c734..3b75821bd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -55,7 +55,6 @@ The following packages are optional: * [Markdown][markdown] (2.1.0+) - Markdown support for the browsable API. * [PyYAML][yaml] (3.10+) - YAML content-type support. -* [defusedxml][defusedxml] (0.3+) - XML content-type support. * [django-filter][django-filter] (0.5.4+) - Filtering support. * [django-oauth-plus][django-oauth-plus] (2.0+) and [oauth2][oauth2] (1.5.211+) - OAuth 1.0a support. * [django-oauth2-provider][django-oauth2-provider] (0.2.3+) - OAuth 2.0 support. @@ -259,7 +258,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [eventbrite]: https://www.eventbrite.co.uk/about/ [markdown]: http://pypi.python.org/pypi/Markdown/ [yaml]: http://pypi.python.org/pypi/PyYAML -[defusedxml]: https://pypi.python.org/pypi/defusedxml [django-filter]: http://pypi.python.org/pypi/django-filter [oauth2]: https://github.com/simplegeo/python-oauth2 [django-oauth-plus]: https://bitbucket.org/david/django-oauth-plus/wiki/Home diff --git a/requirements-test.txt b/requirements-test.txt index 06c8849a8..75cffb9b7 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -7,7 +7,6 @@ flake8==2.2.2 # Optional packages markdown>=2.1.0 PyYAML>=3.10 -defusedxml>=0.3 django-guardian==1.2.4 django-filter>=0.5.4 django-oauth-plus>=2.2.1 diff --git a/rest_framework/compat.py b/rest_framework/compat.py index 5bd85e743..899dd2b48 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -244,13 +244,6 @@ except ImportError: yaml = None -# XML is optional -try: - import defusedxml.ElementTree as etree -except ImportError: - etree = None - - # OAuth2 is optional try: # Note: The `oauth2` package actually provides oauth1.0a support. Urg. diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index ccb82f03b..6d0e932bd 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -12,12 +12,10 @@ from django.http import QueryDict from django.http.multipartparser import MultiPartParser as DjangoMultiPartParser from django.http.multipartparser import MultiPartParserError, parse_header, ChunkIter from django.utils import six -from rest_framework.compat import etree, yaml, force_text, urlparse +from rest_framework.compat import yaml, force_text, urlparse from rest_framework.exceptions import ParseError from rest_framework import renderers import json -import datetime -import decimal class DataAndFiles(object): @@ -136,78 +134,6 @@ class MultiPartParser(BaseParser): raise ParseError('Multipart form parse error - %s' % six.text_type(exc)) -class XMLParser(BaseParser): - """ - XML parser. - """ - - media_type = 'application/xml' - - def parse(self, stream, media_type=None, parser_context=None): - """ - Parses the incoming bytestream as XML and returns the resulting data. - """ - assert etree, 'XMLParser requires defusedxml to be installed' - - parser_context = parser_context or {} - encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) - parser = etree.DefusedXMLParser(encoding=encoding) - try: - tree = etree.parse(stream, parser=parser, forbid_dtd=True) - except (etree.ParseError, ValueError) as exc: - raise ParseError('XML parse error - %s' % six.text_type(exc)) - data = self._xml_convert(tree.getroot()) - - return data - - def _xml_convert(self, element): - """ - convert the xml `element` into the corresponding python object - """ - - children = list(element) - - if len(children) == 0: - return self._type_convert(element.text) - else: - # if the fist child tag is list-item means all children are list-item - if children[0].tag == "list-item": - data = [] - for child in children: - data.append(self._xml_convert(child)) - else: - data = {} - for child in children: - data[child.tag] = self._xml_convert(child) - - return data - - def _type_convert(self, value): - """ - Converts the value returned by the XMl parse into the equivalent - Python type - """ - if value is None: - return value - - try: - return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') - except ValueError: - pass - - try: - return int(value) - except ValueError: - pass - - try: - return decimal.Decimal(value) - except decimal.InvalidOperation: - pass - - return value - - class FileUploadParser(BaseParser): """ Parser for file upload data. diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index e87d16d0d..dd49ae828 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -16,11 +16,8 @@ from django.http.multipartparser import parse_header from django.template import Context, RequestContext, loader, Template from django.test.client import encode_multipart from django.utils import six -from django.utils.xmlutils import SimplerXMLGenerator from rest_framework import exceptions, serializers, status, VERSION -from rest_framework.compat import ( - SHORT_SEPARATORS, LONG_SEPARATORS, StringIO, smart_text, yaml -) +from rest_framework.compat import SHORT_SEPARATORS, LONG_SEPARATORS, yaml from rest_framework.exceptions import ParseError from rest_framework.settings import api_settings from rest_framework.request import is_form_media_type, override_method @@ -140,55 +137,6 @@ class JSONPRenderer(JSONRenderer): return callback.encode(self.charset) + b'(' + json + b');' -class XMLRenderer(BaseRenderer): - """ - Renderer which serializes to XML. - """ - - media_type = 'application/xml' - format = 'xml' - charset = 'utf-8' - - def render(self, data, accepted_media_type=None, renderer_context=None): - """ - Renders `data` into serialized XML. - """ - if data is None: - return '' - - stream = StringIO() - - xml = SimplerXMLGenerator(stream, self.charset) - xml.startDocument() - xml.startElement("root", {}) - - self._to_xml(xml, data) - - xml.endElement("root") - xml.endDocument() - return stream.getvalue() - - def _to_xml(self, xml, data): - if isinstance(data, (list, tuple)): - for item in data: - xml.startElement("list-item", {}) - self._to_xml(xml, item) - xml.endElement("list-item") - - elif isinstance(data, dict): - for key, value in six.iteritems(data): - xml.startElement(key, {}) - self._to_xml(xml, value) - xml.endElement(key) - - elif data is None: - # Don't output any value - pass - - else: - xml.characters(smart_text(data)) - - class YAMLRenderer(BaseRenderer): """ Renderer which serializes to YAML. diff --git a/tests/test_parsers.py b/tests/test_parsers.py index 3f2672df0..32fb05955 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from rest_framework.compat import StringIO from django import forms from django.core.files.uploadhandler import MemoryFileUploadHandler from django.test import TestCase -from django.utils import unittest -from rest_framework.compat import etree +from rest_framework.compat import StringIO from rest_framework.parsers import FormParser, FileUploadParser -from rest_framework.parsers import XMLParser -import datetime class Form(forms.Form): @@ -31,62 +27,6 @@ class TestFormParser(TestCase): self.assertEqual(Form(data).is_valid(), True) -class TestXMLParser(TestCase): - def setUp(self): - self._input = StringIO( - '' - '' - '121.0' - 'dasd' - '' - '2011-12-25 12:45:00' - '' - ) - self._data = { - 'field_a': 121, - 'field_b': 'dasd', - 'field_c': None, - 'field_d': datetime.datetime(2011, 12, 25, 12, 45, 00) - } - self._complex_data_input = StringIO( - '' - '' - '2011-12-25 12:45:00' - '' - '1first' - '2second' - '' - 'name' - '' - ) - self._complex_data = { - "creation_date": datetime.datetime(2011, 12, 25, 12, 45, 00), - "name": "name", - "sub_data_list": [ - { - "sub_id": 1, - "sub_name": "first" - }, - { - "sub_id": 2, - "sub_name": "second" - } - ] - } - - @unittest.skipUnless(etree, 'defusedxml not installed') - def test_parse(self): - parser = XMLParser() - data = parser.parse(self._input) - self.assertEqual(data, self._data) - - @unittest.skipUnless(etree, 'defusedxml not installed') - def test_complex_data_parse(self): - parser = XMLParser() - data = parser.parse(self._complex_data_input) - self.assertEqual(data, self._complex_data) - - class TestFileUploadParser(TestCase): def setUp(self): class MockRequest(object): diff --git a/tests/test_renderers.py b/tests/test_renderers.py index 416d7f224..1eec37dc3 100644 --- a/tests/test_renderers.py +++ b/tests/test_renderers.py @@ -6,19 +6,18 @@ from django.conf.urls import patterns, url, include from django.core.cache import cache from django.db import models from django.test import TestCase -from django.utils import six, unittest +from django.utils import six from django.utils.translation import ugettext_lazy as _ from rest_framework import status, permissions -from rest_framework.compat import yaml, etree, StringIO, BytesIO +from rest_framework.compat import yaml, BytesIO from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \ - XMLRenderer, JSONPRenderer, BrowsableAPIRenderer -from rest_framework.parsers import YAMLParser, XMLParser + JSONPRenderer, BrowsableAPIRenderer +from rest_framework.parsers import YAMLParser from rest_framework.settings import api_settings from rest_framework.test import APIRequestFactory from collections import MutableMapping -import datetime import json import pickle import re @@ -501,104 +500,6 @@ if yaml: self.assertEqual(content.strip(), 'countries: [United Kingdom, France, España]'.encode('utf-8')) -class XMLRendererTestCase(TestCase): - """ - Tests specific to the XML Renderer - """ - - _complex_data = { - "creation_date": datetime.datetime(2011, 12, 25, 12, 45, 00), - "name": "name", - "sub_data_list": [ - { - "sub_id": 1, - "sub_name": "first" - }, - { - "sub_id": 2, - "sub_name": "second" - } - ] - } - - def test_render_string(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render({'field': 'astring'}, 'application/xml') - self.assertXMLContains(content, 'astring') - - def test_render_integer(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render({'field': 111}, 'application/xml') - self.assertXMLContains(content, '111') - - def test_render_datetime(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render({ - 'field': datetime.datetime(2011, 12, 25, 12, 45, 00) - }, 'application/xml') - self.assertXMLContains(content, '2011-12-25 12:45:00') - - def test_render_float(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render({'field': 123.4}, 'application/xml') - self.assertXMLContains(content, '123.4') - - def test_render_decimal(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render({'field': Decimal('111.2')}, 'application/xml') - self.assertXMLContains(content, '111.2') - - def test_render_none(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render({'field': None}, 'application/xml') - self.assertXMLContains(content, '') - - def test_render_complex_data(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = renderer.render(self._complex_data, 'application/xml') - self.assertXMLContains(content, 'first') - self.assertXMLContains(content, 'second') - - @unittest.skipUnless(etree, 'defusedxml not installed') - def test_render_and_parse_complex_data(self): - """ - Test XML rendering. - """ - renderer = XMLRenderer() - content = StringIO(renderer.render(self._complex_data, 'application/xml')) - - parser = XMLParser() - complex_data_out = parser.parse(content) - error_msg = "complex data differs!IN:\n %s \n\n OUT:\n %s" % (repr(self._complex_data), repr(complex_data_out)) - self.assertEqual(self._complex_data, complex_data_out, error_msg) - - def assertXMLContains(self, xml, string): - self.assertTrue(xml.startswith('\n')) - self.assertTrue(xml.endswith('')) - self.assertTrue(string in xml, '%r not in %r' % (string, xml)) - - # Tests for caching issue, #346 class CacheRenderTest(TestCase): """ From 26131a7aea39bb517393b3b6774372d6aebd6885 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 12 Dec 2014 15:59:11 +0000 Subject: [PATCH 06/29] Fix dependancies --- requirements.txt | 5 ----- tox.ini | 8 ++++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index f282d3baf..474df7168 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,10 +9,5 @@ flake8==2.2.2 # Optional packages markdown>=2.1.0 -PyYAML>=3.10 -defusedxml>=0.3 django-guardian==1.2.4 django-filter>=0.5.4 -django-oauth-plus>=2.2.1 -oauth2>=1.5.211 -django-oauth2-provider>=0.2.4 diff --git a/tox.ini b/tox.ini index f129ff3ab..9fb8586c6 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,8 @@ envlist = py27-{flake8,docs}, {py26,py27}-django14, {py26,py27,py32,py33,py34}-django{15,16}, - {py27,py32,py33,py34}-django{17,master} + {py27,py32,py33,py34}-django17, + {py27,py32,py33,py34}-djangomaster [testenv] commands = ./runtests.py --fast @@ -15,10 +16,9 @@ deps = django16: Django==1.6.8 django17: Django==1.7.1 djangomaster: https://github.com/django/django/zipball/master - {py26,py27}-django{14,15,16,17}: django-guardian==1.2.3 + django-guardian==1.2.4 pytest-django==2.6.1 - django-filter==0.7 - defusedxml==0.3 + django-filter==0.9.1 markdown>=2.1.0 [testenv:py27-flake8] From 0d109c90a74bc575efa6d497a6501aef2b837983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sat, 13 Dec 2014 18:18:00 -0400 Subject: [PATCH 07/29] Add context to exception handler #2236 Same context as renderers which include: the view, args, kwargs, and request. This provides enough contextual information to the exception handlers to handle errors better. In a use case like #1671, a custom handler would allow Sentry to log the request properly. --- rest_framework/views.py | 5 +++-- tests/test_views.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rest_framework/views.py b/rest_framework/views.py index bc870417f..07e713939 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -46,7 +46,7 @@ def get_view_description(view_cls, html=False): return description -def exception_handler(exc): +def exception_handler(exc, context=None): """ Returns the response that should be used for any given exception. @@ -369,7 +369,8 @@ class APIView(View): else: exc.status_code = status.HTTP_403_FORBIDDEN - response = self.settings.EXCEPTION_HANDLER(exc) + context = self.get_renderer_context() + response = self.settings.EXCEPTION_HANDLER(exc, context) if response is None: raise diff --git a/tests/test_views.py b/tests/test_views.py index 77b113ee5..e9b75f065 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -121,7 +121,12 @@ class TestCustomExceptionHandler(TestCase): def setUp(self): self.DEFAULT_HANDLER = api_settings.EXCEPTION_HANDLER - def exception_handler(exc): + def exception_handler(exc, context=None): + self.assertTrue('args' in context) + self.assertTrue('kwargs' in context) + self.assertTrue('request' in context) + self.assertTrue('view' in context) + return Response('Error!', status=status.HTTP_400_BAD_REQUEST) api_settings.EXCEPTION_HANDLER = exception_handler From e8c0766568cb20a5357c5e6823283f0c187b35b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sat, 13 Dec 2014 20:54:35 -0400 Subject: [PATCH 08/29] Support handlers with and without context --- rest_framework/views.py | 10 ++++++++-- tests/test_views.py | 27 +++++++++++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/rest_framework/views.py b/rest_framework/views.py index 07e713939..3ece66e68 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -2,6 +2,7 @@ Provides an APIView class that is the base of all views in REST framework. """ from __future__ import unicode_literals +import inspect from django.core.exceptions import PermissionDenied from django.http import Http404 @@ -369,8 +370,13 @@ class APIView(View): else: exc.status_code = status.HTTP_403_FORBIDDEN - context = self.get_renderer_context() - response = self.settings.EXCEPTION_HANDLER(exc, context) + exception_handler = self.settings.EXCEPTION_HANDLER + + if 'context' in inspect.getargspec(exception_handler).args: + context = self.get_renderer_context() + response = exception_handler(exc, context) + else: + response = exception_handler(exc) if response is None: raise diff --git a/tests/test_views.py b/tests/test_views.py index e9b75f065..9952248fc 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -6,6 +6,7 @@ from django.test import TestCase from rest_framework import status from rest_framework.decorators import api_view from rest_framework.response import Response +from rest_framework.request import Request from rest_framework.settings import api_settings from rest_framework.test import APIRequestFactory from rest_framework.views import APIView @@ -121,12 +122,7 @@ class TestCustomExceptionHandler(TestCase): def setUp(self): self.DEFAULT_HANDLER = api_settings.EXCEPTION_HANDLER - def exception_handler(exc, context=None): - self.assertTrue('args' in context) - self.assertTrue('kwargs' in context) - self.assertTrue('request' in context) - self.assertTrue('view' in context) - + def exception_handler(exc): return Response('Error!', status=status.HTTP_400_BAD_REQUEST) api_settings.EXCEPTION_HANDLER = exception_handler @@ -151,3 +147,22 @@ class TestCustomExceptionHandler(TestCase): expected = 'Error!' self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(response.data, expected) + + def test_context_exception_handler(self): + def exception_handler(exc, context=None): + self.assertEqual(context['args'], ()) + self.assertEqual(context['kwargs'], {}) + self.assertTrue(isinstance(context['request'], Request)) + self.assertTrue(isinstance(context['view'], ErrorView)) + + return Response('Error!', status=status.HTTP_400_BAD_REQUEST) + + api_settings.EXCEPTION_HANDLER = exception_handler + + view = ErrorView.as_view() + + request = factory.get('/', content_type='application/json') + response = view(request) + expected = 'Error!' + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data, expected) From 3f85b476fa2ddb8a205c03cea6684fca257dbd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sun, 14 Dec 2014 10:15:13 -0400 Subject: [PATCH 09/29] Remove test --- tests/test_views.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/test_views.py b/tests/test_views.py index 9952248fc..77b113ee5 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -6,7 +6,6 @@ from django.test import TestCase from rest_framework import status from rest_framework.decorators import api_view from rest_framework.response import Response -from rest_framework.request import Request from rest_framework.settings import api_settings from rest_framework.test import APIRequestFactory from rest_framework.views import APIView @@ -147,22 +146,3 @@ class TestCustomExceptionHandler(TestCase): expected = 'Error!' self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(response.data, expected) - - def test_context_exception_handler(self): - def exception_handler(exc, context=None): - self.assertEqual(context['args'], ()) - self.assertEqual(context['kwargs'], {}) - self.assertTrue(isinstance(context['request'], Request)) - self.assertTrue(isinstance(context['view'], ErrorView)) - - return Response('Error!', status=status.HTTP_400_BAD_REQUEST) - - api_settings.EXCEPTION_HANDLER = exception_handler - - view = ErrorView.as_view() - - request = factory.get('/', content_type='application/json') - response = view(request) - expected = 'Error!' - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertEqual(response.data, expected) From 478c8d724b846b370c897548f8ee89f1128e12c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sun, 14 Dec 2014 10:16:52 -0400 Subject: [PATCH 10/29] Update docs --- docs/api-guide/exceptions.md | 4 ++-- docs/api-guide/settings.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md index 467ad9709..31a8431bc 100644 --- a/docs/api-guide/exceptions.md +++ b/docs/api-guide/exceptions.md @@ -51,10 +51,10 @@ In order to alter the style of the response, you could write the following custo from rest_framework.views import exception_handler - def custom_exception_handler(exc): + def custom_exception_handler(exc, context): # Call REST framework's default exception handler first, # to get the standard error response. - response = exception_handler(exc) + response = exception_handler(exc, context) # Now add the HTTP status code to the response. if response is not None: diff --git a/docs/api-guide/settings.md b/docs/api-guide/settings.md index 9005511b7..2c4f84237 100644 --- a/docs/api-guide/settings.md +++ b/docs/api-guide/settings.md @@ -393,7 +393,7 @@ This setting can be changed to support error responses other than the default `{ This should be a function with the following signature: - exception_handler(exc) + exception_handler(exc, context) * `exc`: The exception. From fd003fcefaee964e744ded0aec1ae76715889378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sun, 14 Dec 2014 15:03:20 -0400 Subject: [PATCH 11/29] Add pending deprecation warning message --- rest_framework/views.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/rest_framework/views.py b/rest_framework/views.py index 3ece66e68..37889d1b4 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -3,6 +3,7 @@ Provides an APIView class that is the base of all views in REST framework. """ from __future__ import unicode_literals import inspect +import warnings from django.core.exceptions import PermissionDenied from django.http import Http404 @@ -370,13 +371,16 @@ class APIView(View): else: exc.status_code = status.HTTP_403_FORBIDDEN - exception_handler = self.settings.EXCEPTION_HANDLER - - if 'context' in inspect.getargspec(exception_handler).args: - context = self.get_renderer_context() - response = exception_handler(exc, context) + if len(inspect.getargspec(self.settings.EXCEPTION_HANDLER).args) == 1: + warnings.warn( + 'The `exception_handler(exc)` call signature is deprecated. ' + 'Use `exception_handler(exc, context) instead.', + PendingDeprecationWarning + ) + response = self.settings.EXCEPTION_HANDLER(exc) else: - response = exception_handler(exc) + context = self.get_renderer_context() + response = self.settings.EXCEPTION_HANDLER(exc, context) if response is None: raise From 89e9fc98d6e7407e6f7715fa2680df7c94221105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sun, 14 Dec 2014 15:20:44 -0400 Subject: [PATCH 12/29] Reuse exception_handler variable throughout --- rest_framework/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rest_framework/views.py b/rest_framework/views.py index 37889d1b4..c2e19bf42 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -371,16 +371,18 @@ class APIView(View): else: exc.status_code = status.HTTP_403_FORBIDDEN - if len(inspect.getargspec(self.settings.EXCEPTION_HANDLER).args) == 1: + exception_handler = self.settings.EXCEPTION_HANDLER + + if len(inspect.getargspec(exception_handler).args) == 1: warnings.warn( 'The `exception_handler(exc)` call signature is deprecated. ' 'Use `exception_handler(exc, context) instead.', PendingDeprecationWarning ) - response = self.settings.EXCEPTION_HANDLER(exc) + response = exception_handler(exc) else: context = self.get_renderer_context() - response = self.settings.EXCEPTION_HANDLER(exc, context) + response = exception_handler(exc, context) if response is None: raise From 26c223a34f7e0cc21d37c6302e53d547dae252dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sun, 14 Dec 2014 16:43:58 -0400 Subject: [PATCH 13/29] Add get_exception_handler_context() --- rest_framework/views.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/rest_framework/views.py b/rest_framework/views.py index c2e19bf42..80a13a1a9 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -186,6 +186,18 @@ class APIView(View): 'request': getattr(self, 'request', None) } + def get_exception_handler_context(self): + """ + Returns a dict that is passed through to EXCEPTION_HANDLER, + as the `context` argument. + """ + return { + 'view': self, + 'args': getattr(self, 'args', ()), + 'kwargs': getattr(self, 'kwargs', {}), + 'request': getattr(self, 'request', None) + } + def get_view_name(self): """ Return the view name, as used in OPTIONS responses and in the @@ -381,7 +393,7 @@ class APIView(View): ) response = exception_handler(exc) else: - context = self.get_renderer_context() + context = self.get_exception_handler_context() response = exception_handler(exc, context) if response is None: From 4ebd8770b94ecb8fe8fb41fe8daa4309b33b9952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Sun, 14 Dec 2014 20:47:33 -0400 Subject: [PATCH 14/29] Update excepteion_handler signature --- rest_framework/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_framework/views.py b/rest_framework/views.py index 80a13a1a9..b39724c2f 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -48,7 +48,7 @@ def get_view_description(view_cls, html=False): return description -def exception_handler(exc, context=None): +def exception_handler(exc, context): """ Returns the response that should be used for any given exception. From 5e7c9687c7e11b6adfe2fc534eb0504e67ca9fc9 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 15 Dec 2014 09:13:02 +0000 Subject: [PATCH 15/29] First pass at serializer repr bug --- rest_framework/utils/representation.py | 3 ++- tests/test_serializer.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/rest_framework/utils/representation.py b/rest_framework/utils/representation.py index 3f17a8b9b..0fdb4775d 100644 --- a/rest_framework/utils/representation.py +++ b/rest_framework/utils/representation.py @@ -2,6 +2,7 @@ Helper functions for creating user-friendly representations of serializer classes and serializer fields. """ +from __future__ import unicode_literals from django.db import models from django.utils.encoding import force_text from django.utils.functional import Promise @@ -24,7 +25,7 @@ def smart_repr(value): if isinstance(value, Promise) and value._delegate_text: value = force_text(value) - value = repr(value) + value = repr(value).decode('utf-8') # Representations like u'help text' # should simply be presented as 'help text' diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 56b390956..48fcc83bb 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import unicode_literals from rest_framework import serializers import pytest @@ -197,3 +198,19 @@ class TestIncorrectlyConfigured: "The serializer field might be named incorrectly and not match any attribute or key on the `ExampleObject` instance.\n" "Original exception text was:" ) + + +class TestUnicodeRepr: + def test_unicode_repr(self): + class ExampleSerializer(serializers.Serializer): + example = serializers.CharField() + + class ExampleObject: + def __init__(self): + self.example = '한국' + def __repr__(self): + return self.example.encode('utf8') + + instance = ExampleObject() + serializer = ExampleSerializer(instance) + repr(serializer) From 6e51e4f5cdec4f4580360a487d7bf5ebdef08709 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 16 Dec 2014 15:34:19 +0000 Subject: [PATCH 16/29] Versioning first pass --- docs/api-guide/versioning.md | 9 +++ rest_framework/reverse.py | 12 ++++ rest_framework/settings.py | 7 ++- rest_framework/versioning.py | 96 ++++++++++++++++++++++++++++++++ rest_framework/views.py | 27 +++++++-- tests/test_versioning.py | 104 +++++++++++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 7 deletions(-) create mode 100644 docs/api-guide/versioning.md create mode 100644 rest_framework/versioning.py create mode 100644 tests/test_versioning.py diff --git a/docs/api-guide/versioning.md b/docs/api-guide/versioning.md new file mode 100644 index 000000000..df8148941 --- /dev/null +++ b/docs/api-guide/versioning.md @@ -0,0 +1,9 @@ +source: versioning.py + +# Versioning + +> Versioning an interface is just a "polite" way to kill deployed clients. +> +> — [Roy Fielding][cite]. + +[cite]: http://www.slideshare.net/evolve_conference/201308-fielding-evolve/31 \ No newline at end of file diff --git a/rest_framework/reverse.py b/rest_framework/reverse.py index a74e8aa2d..8fcca55ba 100644 --- a/rest_framework/reverse.py +++ b/rest_framework/reverse.py @@ -8,6 +8,18 @@ from django.utils.functional import lazy def reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra): + """ + If versioning is being used then we pass any `reverse` calls through + to the versioning scheme instance, so that the resulting URL + can be modified if needed. + """ + scheme = getattr(request, 'versioning_scheme', None) + if scheme is not None: + return scheme.reverse(viewname, args, kwargs, request, format, **extra) + return _reverse(viewname, args, kwargs, request, format, **extra) + + +def _reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra): """ Same as `django.core.urlresolvers.reverse`, but optionally takes a request and returns a fully qualified URL, using the request to get the base URL. diff --git a/rest_framework/settings.py b/rest_framework/settings.py index 0aac6d43e..b17f5fccf 100644 --- a/rest_framework/settings.py +++ b/rest_framework/settings.py @@ -46,6 +46,7 @@ DEFAULTS = { 'DEFAULT_THROTTLE_CLASSES': (), 'DEFAULT_CONTENT_NEGOTIATION_CLASS': 'rest_framework.negotiation.DefaultContentNegotiation', 'DEFAULT_METADATA_CLASS': 'rest_framework.metadata.SimpleMetadata', + 'DEFAULT_VERSIONING_CLASS': None, # Generic view behavior 'DEFAULT_MODEL_SERIALIZER_CLASS': 'rest_framework.serializers.ModelSerializer', @@ -124,7 +125,7 @@ IMPORT_STRINGS = ( 'DEFAULT_THROTTLE_CLASSES', 'DEFAULT_CONTENT_NEGOTIATION_CLASS', 'DEFAULT_METADATA_CLASS', - 'DEFAULT_MODEL_SERIALIZER_CLASS', + 'DEFAULT_VERSIONING_CLASS', 'DEFAULT_PAGINATION_SERIALIZER_CLASS', 'DEFAULT_FILTER_BACKENDS', 'EXCEPTION_HANDLER', @@ -141,7 +142,9 @@ def perform_import(val, setting_name): If the given setting is a string import notation, then perform the necessary import or imports. """ - if isinstance(val, six.string_types): + if val is None: + return None + elif isinstance(val, six.string_types): return import_from_string(val, setting_name) elif isinstance(val, (list, tuple)): return [import_from_string(item, setting_name) for item in val] diff --git a/rest_framework/versioning.py b/rest_framework/versioning.py new file mode 100644 index 000000000..2ca8efff4 --- /dev/null +++ b/rest_framework/versioning.py @@ -0,0 +1,96 @@ +# coding: utf-8 +from __future__ import unicode_literals +from rest_framework.reverse import _reverse +from rest_framework.utils.mediatypes import _MediaType +import re + + +class BaseVersioning(object): + def determine_version(self, request, *args, **kwargs): + msg = '{cls}.determine_version() must be implemented.' + raise NotImplemented(msg.format( + cls=self.__class__.__name__ + )) + + def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): + return _reverse(viewname, args, kwargs, request, format, **extra) + + +class QueryParameterVersioning(BaseVersioning): + """ + GET /something/?version=0.1 HTTP/1.1 + Host: example.com + Accept: application/json + """ + default_version = None + version_param = 'version' + + def determine_version(self, request, *args, **kwargs): + return request.query_params.get(self.version_param) + + def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): + url = super(QueryParameterVersioning, self).reverse( + viewname, args, kwargs, request, format, **kwargs + ) + if request.version is not None: + return replace_query_param(url, self.version_param, request.version) + return url + + +class HostNameVersioning(BaseVersioning): + """ + GET /something/ HTTP/1.1 + Host: v1.example.com + Accept: application/json + """ + default_version = None + hostname_regex = re.compile(r'^([a-zA-Z0-9]+)\.[a-zA-Z0-9]+\.[a-zA-Z0-9]+$') + + def determine_version(self, request, *args, **kwargs): + hostname, seperator, port = request.get_host().partition(':') + match = self.hostname_regex.match(hostname) + if not match: + return self.default_version + return match.group(1) + + # We don't need to implement `reverse`, as the hostname will already be + # preserved as part of the standard `reverse` implementation. + + +class AcceptHeaderVersioning(BaseVersioning): + """ + GET /something/ HTTP/1.1 + Host: example.com + Accept: application/json; version=1.0 + """ + default_version = None + version_param = 'version' + + def determine_version(self, request, *args, **kwargs): + media_type = _MediaType(request.accepted_media_type) + return media_type.params.get(self.version_param, self.default_version) + + # We don't need to implement `reverse`, as the versioning is based + # on the `Accept` header, not on the request URL. + + +class URLPathVersioning(BaseVersioning): + """ + GET /1.0/something/ HTTP/1.1 + Host: example.com + Accept: application/json + """ + default_version = None + version_param = 'version' + + def determine_version(self, request, *args, **kwargs): + return kwargs.get(self.version_param, self.default_version) + + def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): + if request.version is not None: + kwargs = {} if (kwargs is None) else kwargs + kwargs[self.version_param] = request.version + + return super(URLPathVersioning, self).reverse( + viewname, args, kwargs, request, format, **kwargs + ) diff --git a/rest_framework/views.py b/rest_framework/views.py index b39724c2f..12bb78bd9 100644 --- a/rest_framework/views.py +++ b/rest_framework/views.py @@ -95,6 +95,7 @@ class APIView(View): permission_classes = api_settings.DEFAULT_PERMISSION_CLASSES content_negotiation_class = api_settings.DEFAULT_CONTENT_NEGOTIATION_CLASS metadata_class = api_settings.DEFAULT_METADATA_CLASS + versioning_class = api_settings.DEFAULT_VERSIONING_CLASS # Allow dependency injection of other settings to make testing easier. settings = api_settings @@ -314,6 +315,16 @@ class APIView(View): if not throttle.allow_request(request, self): self.throttled(request, throttle.wait()) + def determine_version(self, request, *args, **kwargs): + """ + If versioning is being used, then determine any API version for the + incoming request. Returns a two-tuple of (version, versioning_scheme) + """ + if self.versioning_class is None: + return (None, None) + scheme = self.versioning_class() + return (scheme.determine_version(request, *args, **kwargs), scheme) + # Dispatch methods def initialize_request(self, request, *args, **kwargs): @@ -322,11 +333,13 @@ class APIView(View): """ parser_context = self.get_parser_context(request) - return Request(request, - parsers=self.get_parsers(), - authenticators=self.get_authenticators(), - negotiator=self.get_content_negotiator(), - parser_context=parser_context) + return Request( + request, + parsers=self.get_parsers(), + authenticators=self.get_authenticators(), + negotiator=self.get_content_negotiator(), + parser_context=parser_context + ) def initial(self, request, *args, **kwargs): """ @@ -343,6 +356,10 @@ class APIView(View): neg = self.perform_content_negotiation(request) request.accepted_renderer, request.accepted_media_type = neg + # Determine the API version, if versioning is in use. + version, scheme = self.determine_version(request, *args, **kwargs) + request.version, request.versioning_scheme = version, scheme + def finalize_response(self, request, response, *args, **kwargs): """ Returns the final response object. diff --git a/tests/test_versioning.py b/tests/test_versioning.py new file mode 100644 index 000000000..d90b29a1b --- /dev/null +++ b/tests/test_versioning.py @@ -0,0 +1,104 @@ +from django.conf.urls import url +from rest_framework import versioning +from rest_framework.decorators import APIView +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.test import APIRequestFactory, APITestCase + + +class RequestVersionView(APIView): + def get(self, request, *args, **kwargs): + return Response({'version': request.version}) + +class ReverseView(APIView): + def get(self, request, *args, **kwargs): + return Response({'url': reverse('another', request=request)}) + + +factory = APIRequestFactory() + +mock_view = lambda request: None + +urlpatterns = [ + url(r'^another/$', mock_view, name='another') +] + + +class TestRequestVersion: + def test_unversioned(self): + view = RequestVersionView.as_view() + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'version': None} + + def test_query_param_versioning(self): + scheme = versioning.QueryParameterVersioning + view = RequestVersionView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/?version=1.2.3') + response = view(request) + assert response.data == {'version': '1.2.3'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'version': None} + + def test_host_name_versioning(self): + scheme = versioning.HostNameVersioning + view = RequestVersionView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/', HTTP_HOST='v1.example.org') + response = view(request) + assert response.data == {'version': 'v1'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'version': None} + + def test_accept_header_versioning(self): + scheme = versioning.AcceptHeaderVersioning + view = RequestVersionView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/', HTTP_ACCEPT='application/json; version=1.2.3') + response = view(request) + assert response.data == {'version': '1.2.3'} + + request = factory.get('/endpoint/', HTTP_ACCEPT='application/json') + response = view(request) + assert response.data == {'version': None} + + def test_url_path_versioning(self): + scheme = versioning.URLPathVersioning + view = RequestVersionView.as_view(versioning_class=scheme) + + request = factory.get('/1.2.3/endpoint/') + response = view(request, version='1.2.3') + assert response.data == {'version': '1.2.3'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'version': None} + + +class TestURLReversing(APITestCase): + urls = 'tests.test_versioning' + + def test_reverse_unversioned(self): + view = ReverseView.as_view() + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'url': 'http://testserver/another/'} + + def test_reverse_host_name_versioning(self): + scheme = versioning.HostNameVersioning + view = ReverseView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/', HTTP_HOST='v1.example.org') + response = view(request) + assert response.data == {'url': 'http://v1.example.org/another/'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'url': 'http://testserver/another/'} From 4e91ec61339838426e246e20ef062c963a78c4e1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 16 Dec 2014 16:14:08 +0000 Subject: [PATCH 17/29] Added NamespaceVersioning --- rest_framework/versioning.py | 31 +++++++++++++++-- tests/test_versioning.py | 67 ++++++++++++++++++++++++++++++++++-- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/rest_framework/versioning.py b/rest_framework/versioning.py index 2ca8efff4..42df8b2c0 100644 --- a/rest_framework/versioning.py +++ b/rest_framework/versioning.py @@ -1,6 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals from rest_framework.reverse import _reverse +from rest_framework.templatetags.rest_framework import replace_query_param from rest_framework.utils.mediatypes import _MediaType import re @@ -30,7 +31,7 @@ class QueryParameterVersioning(BaseVersioning): def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): url = super(QueryParameterVersioning, self).reverse( - viewname, args, kwargs, request, format, **kwargs + viewname, args, kwargs, request, format, **extra ) if request.version is not None: return replace_query_param(url, self.version_param, request.version) @@ -92,5 +93,31 @@ class URLPathVersioning(BaseVersioning): kwargs[self.version_param] = request.version return super(URLPathVersioning, self).reverse( - viewname, args, kwargs, request, format, **kwargs + viewname, args, kwargs, request, format, **extra + ) + + +class NamespaceVersioning(BaseVersioning): + """ + To the client this is the same style as `URLPathVersioning`. + The difference is in the backend - this implementation uses + Django's URL namespaces to determine the version. + + GET /1.0/something/ HTTP/1.1 + Host: example.com + Accept: application/json + """ + default_version = None + + def determine_version(self, request, *args, **kwargs): + resolver_match = getattr(request, 'resolver_match', None) + if (resolver_match is None or not resolver_match.namespace): + return self.default_version + return resolver_match.namespace + + def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): + if request.version is not None: + viewname = request.version + ':' + viewname + return super(NamespaceVersioning, self).reverse( + viewname, args, kwargs, request, format, **extra ) diff --git a/tests/test_versioning.py b/tests/test_versioning.py index d90b29a1b..eaac5dfb3 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.conf.urls import include, url from rest_framework import versioning from rest_framework.decorators import APIView from rest_framework.response import Response @@ -10,6 +10,7 @@ class RequestVersionView(APIView): def get(self, request, *args, **kwargs): return Response({'version': request.version}) + class ReverseView(APIView): def get(self, request, *args, **kwargs): return Response({'url': reverse('another', request=request)}) @@ -19,8 +20,14 @@ factory = APIRequestFactory() mock_view = lambda request: None +included_patterns = [ + url(r'^namespaced/$', mock_view, name='another'), +] + urlpatterns = [ - url(r'^another/$', mock_view, name='another') + url(r'^v1/', include(included_patterns, namespace='v1')), + url(r'^another/$', mock_view, name='another'), + url(r'^(?P[^/]+)/another/$', mock_view, name='another') ] @@ -80,6 +87,22 @@ class TestRequestVersion: response = view(request) assert response.data == {'version': None} + def test_namespace_versioning(self): + class FakeResolverMatch: + namespace = 'v1' + + scheme = versioning.NamespaceVersioning + view = RequestVersionView.as_view(versioning_class=scheme) + + request = factory.get('/v1/endpoint/') + request.resolver_match = FakeResolverMatch + response = view(request, version='v1') + assert response.data == {'version': 'v1'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'version': None} + class TestURLReversing(APITestCase): urls = 'tests.test_versioning' @@ -91,6 +114,18 @@ class TestURLReversing(APITestCase): response = view(request) assert response.data == {'url': 'http://testserver/another/'} + def test_reverse_query_param_versioning(self): + scheme = versioning.QueryParameterVersioning + view = ReverseView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/?version=v1') + response = view(request) + assert response.data == {'url': 'http://testserver/another/?version=v1'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'url': 'http://testserver/another/'} + def test_reverse_host_name_versioning(self): scheme = versioning.HostNameVersioning view = ReverseView.as_view(versioning_class=scheme) @@ -102,3 +137,31 @@ class TestURLReversing(APITestCase): request = factory.get('/endpoint/') response = view(request) assert response.data == {'url': 'http://testserver/another/'} + + def test_reverse_url_path_versioning(self): + scheme = versioning.URLPathVersioning + view = ReverseView.as_view(versioning_class=scheme) + + request = factory.get('/v1/endpoint/') + response = view(request, version='v1') + assert response.data == {'url': 'http://testserver/v1/another/'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'url': 'http://testserver/another/'} + + def test_namespace_versioning(self): + class FakeResolverMatch: + namespace = 'v1' + + scheme = versioning.NamespaceVersioning + view = ReverseView.as_view(versioning_class=scheme) + + request = factory.get('/v1/endpoint/') + request.resolver_match = FakeResolverMatch + response = view(request, version='v1') + assert response.data == {'url': 'http://testserver/v1/namespaced/'} + + request = factory.get('/endpoint/') + response = view(request) + assert response.data == {'url': 'http://testserver/another/'} From fe9647ce92b61b57dc64604241352bf269d65af7 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 16 Dec 2014 16:37:32 +0000 Subject: [PATCH 18/29] AcceptHeaderVersioning to return unicode strings. --- rest_framework/compat.py | 13 +++++++++---- rest_framework/versioning.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/rest_framework/compat.py b/rest_framework/compat.py index c5242343e..3c8fb0da4 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -5,15 +5,13 @@ versions of django/python, and compatibility wrappers around optional packages. # flake8: noqa from __future__ import unicode_literals - -import inspect - from django.core.exceptions import ImproperlyConfigured +from django.conf import settings from django.utils.encoding import force_text from django.utils.six.moves.urllib import parse as urlparse -from django.conf import settings from django.utils import six import django +import inspect def unicode_repr(instance): @@ -33,6 +31,13 @@ def unicode_to_repr(value): return value +def unicode_http_header(value): + # Coerce HTTP header value to unicode. + if isinstance(value, six.binary_type): + return value.decode('iso-8859-1') + return value + + # OrderedDict only available in Python 2.7. # This will always be the case in Django 1.7 and above, as these versions # no longer support Python 2.6. diff --git a/rest_framework/versioning.py b/rest_framework/versioning.py index 42df8b2c0..9a27cb081 100644 --- a/rest_framework/versioning.py +++ b/rest_framework/versioning.py @@ -1,5 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals +from rest_framework.compat import unicode_http_header from rest_framework.reverse import _reverse from rest_framework.templatetags.rest_framework import replace_query_param from rest_framework.utils.mediatypes import _MediaType @@ -69,7 +70,8 @@ class AcceptHeaderVersioning(BaseVersioning): def determine_version(self, request, *args, **kwargs): media_type = _MediaType(request.accepted_media_type) - return media_type.params.get(self.version_param, self.default_version) + version = media_type.params.get(self.version_param, self.default_version) + return unicode_http_header(version) # We don't need to implement `reverse`, as the versioning is based # on the `Accept` header, not on the request URL. @@ -77,6 +79,17 @@ class AcceptHeaderVersioning(BaseVersioning): class URLPathVersioning(BaseVersioning): """ + To the client this is the same style as `NamespaceVersioning`. + The difference is in the backend - this implementation uses + Django's URL keyword arguments to determine the version. + + An example URL conf for two views that accept two different versions. + + urlpatterns = [ + url(r'^(?P{v1,v2})/users/$', users_list, name='users-list'), + url(r'^(?P{v1,v2})/users/(?P[0-9]+)/$', users_detail, name='users-detail') + ] + GET /1.0/something/ HTTP/1.1 Host: example.com Accept: application/json @@ -103,6 +116,20 @@ class NamespaceVersioning(BaseVersioning): The difference is in the backend - this implementation uses Django's URL namespaces to determine the version. + An example URL conf that is namespaced into two seperate versions + + # users/urls.py + urlpatterns = [ + url(r'^/users/$', users_list, name='users-list'), + url(r'^/users/(?P[0-9]+)/$', users_detail, name='users-detail') + ] + + # urls.py + urlpatterns = [ + url(r'^v1/', include('users.urls', namespace='v1')), + url(r'^v2/', include('users.urls', namespace='v2')) + ] + GET /1.0/something/ HTTP/1.1 Host: example.com Accept: application/json From 70bd3a32f7cf57543e8ec08fddf001a718e40c7f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 16 Dec 2014 20:01:01 +0000 Subject: [PATCH 19/29] Minor comment tweak --- rest_framework/versioning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_framework/versioning.py b/rest_framework/versioning.py index 9a27cb081..223d0f613 100644 --- a/rest_framework/versioning.py +++ b/rest_framework/versioning.py @@ -56,7 +56,7 @@ class HostNameVersioning(BaseVersioning): return match.group(1) # We don't need to implement `reverse`, as the hostname will already be - # preserved as part of the standard `reverse` implementation. + # preserved as part of the REST framework `reverse` implementation. class AcceptHeaderVersioning(BaseVersioning): From 05a6eaec8aebdca2248b9e1069a15769fd85a480 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 17 Dec 2014 12:41:46 +0000 Subject: [PATCH 20/29] More docs, plus 'ALLOWED_VERSIONS' setting. --- docs/api-guide/exceptions.md | 16 +++ docs/api-guide/settings.md | 22 ++++ docs/api-guide/versioning.md | 195 ++++++++++++++++++++++++++++++++++- docs/index.md | 2 + mkdocs.yml | 1 + rest_framework/exceptions.py | 5 + rest_framework/settings.py | 5 + rest_framework/versioning.py | 120 ++++++++++++--------- tests/test_versioning.py | 60 ++++++++++- 9 files changed, 375 insertions(+), 51 deletions(-) diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md index 31a8431bc..50bd14dd2 100644 --- a/docs/api-guide/exceptions.md +++ b/docs/api-guide/exceptions.md @@ -128,6 +128,14 @@ Raised when an authenticated request fails the permission checks. By default this exception results in a response with the HTTP status code "403 Forbidden". +## NotFound + +**Signature:** `NotFound(detail=None)` + +Raised when a resource does not exists at the given URL. This exception is equivalent to the standard `Http404` Django exception. + +By default this exception results in a response with the HTTP status code "404 Not Found". + ## MethodNotAllowed **Signature:** `MethodNotAllowed(method, detail=None)` @@ -136,6 +144,14 @@ Raised when an incoming request occurs that does not map to a handler method on By default this exception results in a response with the HTTP status code "405 Method Not Allowed". +## NotAcceptable + +**Signature:** `NotAcceptable(detail=None)` + +Raised when an incoming request occurs with an `Accept` header that cannot be satisfied by any of the available renderers. + +By default this exception results in a response with the HTTP status code "406 Not Acceptable". + ## UnsupportedMediaType **Signature:** `UnsupportedMediaType(media_type, detail=None)` diff --git a/docs/api-guide/settings.md b/docs/api-guide/settings.md index 9efeda7fa..5af429d16 100644 --- a/docs/api-guide/settings.md +++ b/docs/api-guide/settings.md @@ -166,6 +166,28 @@ Default: `ordering` --- +## Versioning settings + +#### DEFAULT_VERSION + +The value that should be used for `request.version` when no versioning information is present. + +Default: `None` + +#### ALLOWED_VERSIONS + +If set, this value will restrict the set of versions that may be returned by the versioning scheme, and will raise an error if the provided version if not in this set. + +Default: `None` + +#### VERSION_PARAMETER + +The string that should used for any versioning parameters, such as in the media type or URL query parameters. + +Default: `'version'` + +--- + ## Authentication settings *The following settings control the behavior of unauthenticated requests.* diff --git a/docs/api-guide/versioning.md b/docs/api-guide/versioning.md index df8148941..92380cc0e 100644 --- a/docs/api-guide/versioning.md +++ b/docs/api-guide/versioning.md @@ -6,4 +6,197 @@ source: versioning.py > > — [Roy Fielding][cite]. -[cite]: http://www.slideshare.net/evolve_conference/201308-fielding-evolve/31 \ No newline at end of file +API versioning allows you to alter behavior between different clients. REST framework provides for a number of different versioning schemes. + +Versioning is determined by the incoming client request, and may either be based on the request URL, or based on the request headers. + +## Versioning with REST framework + +When API versioning is enabled, the `request.version` attribute will contain a string that corresponds to the version requested in the incoming client request. + +By default, versioning is not enabled, and `request.version` will always return `None`. + +#### Varying behavior based on the version + +How you vary the API behavior is up to you, but one example you might typically want is to switch to a different serialization style in a newer version. For example: + + def get_serializer_class(self): + if self.request.version == 'v1': + return AccountSerializerVersion1 + return AccountSerializer + +#### Reversing URLs for versioned APIs + +The `reverse` function included by REST framework ties in with the versioning scheme. You need to make sure to include the current `request` as a keyword argument, like so. + + reverse('bookings-list', request=request) + +The above function will apply any URL transformations appropriate to the request version. For example: + +* If `NamespacedVersioning` was being used, and the API version was 'v1', then the URL lookup used would be `'v1:bookings-list'`, which might resolve to a URL like `http://example.org/v1/bookings/`. +* If `QueryParameterVersioning` was being used, and the API version was `1.0`, then the returned URL might be something like `http://example.org/bookings/?version=1.0` + +#### Versioned APIs and hyperlinked serializers + +When using hyperlinked serialization styles together with a URL based versioning scheme make sure to include the request as context to the serializer. + + def get(self, request): + queryset = Booking.objects.all() + serializer = BookingsSerializer(queryset, many=True, context={'request': request}) + return Response({'all_bookings': serializer.data}) + +Doing so will allow any returned URLs to include the appropriate versioning. + +## Configuring the versioning scheme + +The versioning scheme is defined by the `DEFAULT_VERSIONING_CLASS` settings key. + + REST_FRAMEWORK = { + 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning' + } + +Unless it is explicitly set, the value for `DEFAULT_VERSIONING_CLASS` will be `None`. In this case the `request.version` attribute will always return `None`. + +You can also set the versioning scheme on an individual view. Typically you won't need to do this, as it makes more sense to have a single versioning scheme used globally. If you do need to do so, use the `versioning_class` attribute. + + class ProfileList(APIView): + versioning_class = versioning.QueryParameterVersioning + +#### Other versioning settings + +The following settings keys are also used to control versioning: + +* `DEFAULT_VERSION`. The value that should be used for `request.version` when no versioning information is present. Defaults to `None`. +* `ALLOWED_VERSIONS`. If set, this value will restrict the set of versions that may be returned by the versioning scheme, and will raise an error if the provided version if not in this set. Defaults to `None`. +* `VERSION_PARAMETER`. The string that should used for any versioning parameters, such as in the media type or URL query parameters. Defaults to `'version'`. + +--- + +# API Reference + +## AcceptHeaderVersioning + +This scheme requires the client to specify the version as part of the media type in the `Accept` header. The version is included as a media type parameter, that supplements the main media type. + +Here's an example HTTP request using the accept header versioning style. + + GET /bookings/ HTTP/1.1 + Host: example.com + Accept: application/json; version=1.0 + +In the example request above `request.version` attribute would return the string `'1.0'`. + +Versioning based on accept headers is [generally considered][klabnik-guidelines] as [best practice][heroku-guidelines], although other styles may be suitable depending on your client requirements. + +#### Using accept headers with vendor media types + +Strictly speaking the `json` media type is not specified as [including additional parameters][json-parameters]. If you are building a well-specified public API you might consider using a [vendor media type][vendor-media-type]. To do so, configure your renderers to use a JSON based renderer with a custom media type: + + class BookingsAPIRenderer(JSONRenderer): + media_type = 'application/vnd.megacorp.bookings+json' + +Your client requests would now look like this: + + GET /bookings/ HTTP/1.1 + Host: example.com + Accept: application/vnd.megacorp.bookings+json; version=1.0 + +## URLParameterVersioning + +This scheme requires the client to specify the version as part of the URL path. + + GET /v1/bookings/ HTTP/1.1 + Host: example.com + Accept: application/json + +Your URL conf must include a pattern that matches the version with a `'version'` keyword argument, so that this information is available to the versioning scheme. + + urlpatterns = [ + url( + r'^(?P{v1,v2})/bookings/$', + bookings_list, + name='bookings-list' + ), + url( + r'^(?P{v1,v2})/bookings/(?P[0-9]+)/$', + bookings_detail, + name='bookings-detail' + ) + ] + +## NamespaceVersioning + +To the client, this scheme is the same as `URLParameterVersioning`. The only difference is how it is configured in your Django application, as it uses URL namespacing, instead of URL keyword arguments. + + GET /v1/something/ HTTP/1.1 + Host: example.com + Accept: application/json + +With this scheme the `request.version` attribute is determined based on the `namespace` that matches the incoming request path. + +In the following example we're giving a set of views two different possible URL prefixes, each under a different namespace: + + # bookings/urls.py + urlpatterns = [ + url(r'^$', bookings_list, name='bookings-list'), + url(r'^(?P[0-9]+)/$', bookings_detail, name='bookings-detail') + ] + + # urls.py + urlpatterns = [ + url(r'^v1/bookings/', include('bookings.urls', namespace='v1')), + url(r'^v2/bookings/', include('bookings.urls', namespace='v2')) + ] + +Both `URLParameterVersioning` and `NamespaceVersioning` are reasonable if you just need a simple versioning scheme. The `URLParameterVersioning` approach might be better suitable for small ad-hoc projects, and the `NaemspaceVersioning` is probably easier to manage for larger projects. + +## HostNameVersioning + +The hostname versioning scheme requires the client to specify the requested version as part of the hostname in the URL. + +For example the following is an HTTP request to the `http://v1.example.com/bookings/` URL: + + GET /bookings/ HTTP/1.1 + Host: v1.example.com + Accept: application/json + +By default this implementation expects the hostname to match this simple regular expression: + + ^([a-zA-Z0-9]+)\.[a-zA-Z0-9]+\.[a-zA-Z0-9]+$ + +Note that the first group is enclosed in brackets, indicating that this is the matched portion of the hostname. + +The `HostNameVersioning` scheme can be awkward to use in debug mode as you will typically be accessing a raw IP address such as `127.0.0.1`. There are various online services which you to [access localhost with a custom subdomain][lvh] which you may find helpful in this case. + +Hostname based versioning can be particularly useful if you have requirements to route incoming requests to different servers based on the version, as you can configure different DNS records for different API versions. + +## QueryParameterVersioning + +This scheme is a simple style that includes the version as a query parameter in the URL. For example: + + GET /something/?version=0.1 HTTP/1.1 + Host: example.com + Accept: application/json + +--- + +# Custom versioning schemes + +To implement a custom versioning scheme, subclass `BaseVersioning` and override the `.determine_version` method. + +## Example + +The following example uses a custom `X-API-Version` header to determine the requested version. + + class XAPIVersionScheme(versioning.BaseVersioning): + def determine_version(self, request, *args, **kwargs): + return request.META.get('HTTP_X_API_VERSION', None) + +If your versioning scheme is based on the request URL, you will also want to alter how versioned URLs are determined. In order to do so you should override the `.reverse()` method on the class. See the source code for examples. + +[cite]: http://www.slideshare.net/evolve_conference/201308-fielding-evolve/31 +[klabnik-guidelines]: http://blog.steveklabnik.com/posts/2011-07-03-nobody-understands-rest-or-http#i_want_my_api_to_be_versioned +[heroku-guidelines]: https://github.com/interagent/http-api-design#version-with-accepts-header +[json-parameters]: http://tools.ietf.org/html/rfc4627#section-6 +[vendor-media-type]: http://en.wikipedia.org/wiki/Internet_media_type#Vendor_tree +[lvh]: https://reinteractive.net/posts/199-developing-and-testing-rails-applications-with-subdomains diff --git a/docs/index.md b/docs/index.md index 502d352cb..14cf30acb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -175,6 +175,7 @@ The API guide is your complete reference manual to all the functionality provide * [Throttling][throttling] * [Filtering][filtering] * [Pagination][pagination] +* [Versioning][versioning] * [Content negotiation][contentnegotiation] * [Format suffixes][formatsuffixes] * [Returning URLs][reverse] @@ -294,6 +295,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [throttling]: api-guide/throttling.md [filtering]: api-guide/filtering.md [pagination]: api-guide/pagination.md +[versioning]: api-guide/versioning.md [contentnegotiation]: api-guide/content-negotiation.md [formatsuffixes]: api-guide/format-suffixes.md [reverse]: api-guide/reverse.md diff --git a/mkdocs.yml b/mkdocs.yml index 9513f04f7..c2d6bb524 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ pages: - ['api-guide/throttling.md', 'API Guide', 'Throttling'] - ['api-guide/filtering.md', 'API Guide', 'Filtering'] - ['api-guide/pagination.md', 'API Guide', 'Pagination'] + - ['api-guide/versioning.md', 'API Guide', 'Versioning'] - ['api-guide/content-negotiation.md', 'API Guide', 'Content negotiation'] - ['api-guide/format-suffixes.md', 'API Guide', 'Format suffixes'] - ['api-guide/reverse.md', 'API Guide', 'Returning URLs'] diff --git a/rest_framework/exceptions.py b/rest_framework/exceptions.py index be41d08d9..238934dbe 100644 --- a/rest_framework/exceptions.py +++ b/rest_framework/exceptions.py @@ -89,6 +89,11 @@ class PermissionDenied(APIException): default_detail = _('You do not have permission to perform this action.') +class NotFound(APIException): + status_code = status.HTTP_404_NOT_FOUND + default_detail = _('Not found') + + class MethodNotAllowed(APIException): status_code = status.HTTP_405_METHOD_NOT_ALLOWED default_detail = _("Method '%s' not allowed.") diff --git a/rest_framework/settings.py b/rest_framework/settings.py index da3be38dd..877d461be 100644 --- a/rest_framework/settings.py +++ b/rest_framework/settings.py @@ -68,6 +68,11 @@ DEFAULTS = { 'SEARCH_PARAM': 'search', 'ORDERING_PARAM': 'ordering', + # Versioning + 'DEFAULT_VERSION': None, + 'ALLOWED_VERSIONS': None, + 'VERSION_PARAM': 'version', + # Authentication 'UNAUTHENTICATED_USER': 'django.contrib.auth.models.AnonymousUser', 'UNAUTHENTICATED_TOKEN': None, diff --git a/rest_framework/versioning.py b/rest_framework/versioning.py index 223d0f613..440efd139 100644 --- a/rest_framework/versioning.py +++ b/rest_framework/versioning.py @@ -1,13 +1,20 @@ # coding: utf-8 from __future__ import unicode_literals +from django.utils.translation import ugettext_lazy as _ +from rest_framework import exceptions from rest_framework.compat import unicode_http_header from rest_framework.reverse import _reverse +from rest_framework.settings import api_settings from rest_framework.templatetags.rest_framework import replace_query_param from rest_framework.utils.mediatypes import _MediaType import re class BaseVersioning(object): + default_version = api_settings.DEFAULT_VERSION + allowed_versions = api_settings.ALLOWED_VERSIONS + version_param = api_settings.VERSION_PARAM + def determine_version(self, request, *args, **kwargs): msg = '{cls}.determine_version() must be implemented.' raise NotImplemented(msg.format( @@ -17,46 +24,10 @@ class BaseVersioning(object): def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): return _reverse(viewname, args, kwargs, request, format, **extra) - -class QueryParameterVersioning(BaseVersioning): - """ - GET /something/?version=0.1 HTTP/1.1 - Host: example.com - Accept: application/json - """ - default_version = None - version_param = 'version' - - def determine_version(self, request, *args, **kwargs): - return request.query_params.get(self.version_param) - - def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): - url = super(QueryParameterVersioning, self).reverse( - viewname, args, kwargs, request, format, **extra - ) - if request.version is not None: - return replace_query_param(url, self.version_param, request.version) - return url - - -class HostNameVersioning(BaseVersioning): - """ - GET /something/ HTTP/1.1 - Host: v1.example.com - Accept: application/json - """ - default_version = None - hostname_regex = re.compile(r'^([a-zA-Z0-9]+)\.[a-zA-Z0-9]+\.[a-zA-Z0-9]+$') - - def determine_version(self, request, *args, **kwargs): - hostname, seperator, port = request.get_host().partition(':') - match = self.hostname_regex.match(hostname) - if not match: - return self.default_version - return match.group(1) - - # We don't need to implement `reverse`, as the hostname will already be - # preserved as part of the REST framework `reverse` implementation. + def is_allowed_version(self, version): + if not self.allowed_versions: + return True + return (version == self.default_version) or (version in self.allowed_versions) class AcceptHeaderVersioning(BaseVersioning): @@ -65,13 +36,15 @@ class AcceptHeaderVersioning(BaseVersioning): Host: example.com Accept: application/json; version=1.0 """ - default_version = None - version_param = 'version' + invalid_version_message = _("Invalid version in 'Accept' header.") def determine_version(self, request, *args, **kwargs): media_type = _MediaType(request.accepted_media_type) version = media_type.params.get(self.version_param, self.default_version) - return unicode_http_header(version) + version = unicode_http_header(version) + if not self.is_allowed_version(version): + raise exceptions.NotAcceptable(self.invalid_version_message) + return version # We don't need to implement `reverse`, as the versioning is based # on the `Accept` header, not on the request URL. @@ -94,11 +67,13 @@ class URLPathVersioning(BaseVersioning): Host: example.com Accept: application/json """ - default_version = None - version_param = 'version' + invalid_version_message = _('Invalid version in URL path.') def determine_version(self, request, *args, **kwargs): - return kwargs.get(self.version_param, self.default_version) + version = kwargs.get(self.version_param, self.default_version) + if not self.is_allowed_version(version): + raise exceptions.NotFound(self.invalid_version_message) + return version def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): if request.version is not None: @@ -134,13 +109,16 @@ class NamespaceVersioning(BaseVersioning): Host: example.com Accept: application/json """ - default_version = None + invalid_version_message = _('Invalid version in URL path.') def determine_version(self, request, *args, **kwargs): resolver_match = getattr(request, 'resolver_match', None) if (resolver_match is None or not resolver_match.namespace): return self.default_version - return resolver_match.namespace + version = resolver_match.namespace + if not self.is_allowed_version(version): + raise exceptions.NotFound(self.invalid_version_message) + return version def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): if request.version is not None: @@ -148,3 +126,49 @@ class NamespaceVersioning(BaseVersioning): return super(NamespaceVersioning, self).reverse( viewname, args, kwargs, request, format, **extra ) + + +class HostNameVersioning(BaseVersioning): + """ + GET /something/ HTTP/1.1 + Host: v1.example.com + Accept: application/json + """ + hostname_regex = re.compile(r'^([a-zA-Z0-9]+)\.[a-zA-Z0-9]+\.[a-zA-Z0-9]+$') + invalid_version_message = _('Invalid version in hostname.') + + def determine_version(self, request, *args, **kwargs): + hostname, seperator, port = request.get_host().partition(':') + match = self.hostname_regex.match(hostname) + if not match: + return self.default_version + version = match.group(1) + if not self.is_allowed_version(version): + raise exceptions.NotFound(self.invalid_version_message) + return version + + # We don't need to implement `reverse`, as the hostname will already be + # preserved as part of the REST framework `reverse` implementation. + + +class QueryParameterVersioning(BaseVersioning): + """ + GET /something/?version=0.1 HTTP/1.1 + Host: example.com + Accept: application/json + """ + invalid_version_message = _('Invalid version in query parameter.') + + def determine_version(self, request, *args, **kwargs): + version = request.query_params.get(self.version_param) + if not self.is_allowed_version(version): + raise exceptions.NotFound(self.invalid_version_message) + return version + + def reverse(self, viewname, args=None, kwargs=None, request=None, format=None, **extra): + url = super(QueryParameterVersioning, self).reverse( + viewname, args, kwargs, request, format, **extra + ) + if request.version is not None: + return replace_query_param(url, self.version_param, request.version) + return url diff --git a/tests/test_versioning.py b/tests/test_versioning.py index eaac5dfb3..c44f727d2 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -1,5 +1,5 @@ from django.conf.urls import include, url -from rest_framework import versioning +from rest_framework import status, versioning from rest_framework.decorators import APIView from rest_framework.response import Response from rest_framework.reverse import reverse @@ -16,6 +16,16 @@ class ReverseView(APIView): return Response({'url': reverse('another', request=request)}) +class RequestInvalidVersionView(APIView): + def determine_version(self, request, *args, **kwargs): + scheme = self.versioning_class() + scheme.allowed_versions = ('v1', 'v2') + return (scheme.determine_version(request, *args, **kwargs), scheme) + + def get(self, request, *args, **kwargs): + return Response({'version': request.version}) + + factory = APIRequestFactory() mock_view = lambda request: None @@ -150,7 +160,7 @@ class TestURLReversing(APITestCase): response = view(request) assert response.data == {'url': 'http://testserver/another/'} - def test_namespace_versioning(self): + def test_reverse_namespace_versioning(self): class FakeResolverMatch: namespace = 'v1' @@ -165,3 +175,49 @@ class TestURLReversing(APITestCase): request = factory.get('/endpoint/') response = view(request) assert response.data == {'url': 'http://testserver/another/'} + + +class TestInvalidVersion: + def test_invalid_query_param_versioning(self): + scheme = versioning.QueryParameterVersioning + view = RequestInvalidVersionView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/?version=v3') + response = view(request) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_invalid_host_name_versioning(self): + scheme = versioning.HostNameVersioning + view = RequestInvalidVersionView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/', HTTP_HOST='v3.example.org') + response = view(request) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_invalid_accept_header_versioning(self): + scheme = versioning.AcceptHeaderVersioning + view = RequestInvalidVersionView.as_view(versioning_class=scheme) + + request = factory.get('/endpoint/', HTTP_ACCEPT='application/json; version=v3') + response = view(request) + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + + def test_invalid_url_path_versioning(self): + scheme = versioning.URLPathVersioning + view = RequestInvalidVersionView.as_view(versioning_class=scheme) + + request = factory.get('/v3/endpoint/') + response = view(request, version='v3') + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_invalid_namespace_versioning(self): + class FakeResolverMatch: + namespace = 'v3' + + scheme = versioning.NamespaceVersioning + view = RequestInvalidVersionView.as_view(versioning_class=scheme) + + request = factory.get('/v3/endpoint/') + request.resolver_match = FakeResolverMatch + response = view(request, version='v3') + assert response.status_code == status.HTTP_404_NOT_FOUND From 5830f7e13817210f5c6d955ad4fedfaa492aa209 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 10:15:36 +0000 Subject: [PATCH 21/29] get_unique_together_validators and get_unique_for_date_validators --- rest_framework/serializers.py | 51 ++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 8de22f4b9..55828b03e 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -792,14 +792,33 @@ class ModelSerializer(Serializer): return instance def get_validators(self): + """ + Determine the set of validators to use when instantiating serializer. + """ # If the validators have been declared explicitly then use that. validators = getattr(getattr(self, 'Meta', None), 'validators', None) if validators is not None: return validators - # Determine the default set of validators. - validators = [] - model_class = self.Meta.model + # Otherwise use the default set of validators. + return ( + self.get_unique_together_validators() + + self.get_unique_for_date_validators() + ) + + def get_unique_together_validators(self): + """ + Determine a default set of validators for any unique_together contraints. + """ + model_class_inheritance_tree = ( + [self.Meta.model] + + list(self.Meta.model._meta.parents.keys()) + ) + + # The field names we're passing though here only include fields + # which may map onto a model field. Any dotted field name lookups + # cannot map to a field, and must be a traversal, so we're not + # including those. field_names = set([ field.source for field in self.fields.values() if (field.source != '*') and ('.' not in field.source) @@ -807,7 +826,8 @@ class ModelSerializer(Serializer): # Note that we make sure to check `unique_together` both on the # base model class, but also on any parent classes. - for parent_class in [model_class] + list(model_class._meta.parents.keys()): + validators = [] + for parent_class in model_class_inheritance_tree: for unique_together in parent_class._meta.unique_together: if field_names.issuperset(set(unique_together)): validator = UniqueTogetherValidator( @@ -815,13 +835,26 @@ class ModelSerializer(Serializer): fields=unique_together ) validators.append(validator) + return validators + + def get_unique_for_date_validators(self): + """ + Determine a default set of validators for the following contraints: + + * unique_for_date + * unique_for_month + * unique_for_year + """ + info = model_meta.get_field_info(self.Meta.model) + default_manager = self.Meta.model._default_manager + field_names = [field.source for field in self.fields.values()] + + validators = [] - # Add any unique_for_date/unique_for_month/unique_for_year constraints. - info = model_meta.get_field_info(model_class) for field_name, field in info.fields_and_pk.items(): if field.unique_for_date and field_name in field_names: validator = UniqueForDateValidator( - queryset=model_class._default_manager, + queryset=default_manager, field=field_name, date_field=field.unique_for_date ) @@ -829,7 +862,7 @@ class ModelSerializer(Serializer): if field.unique_for_month and field_name in field_names: validator = UniqueForMonthValidator( - queryset=model_class._default_manager, + queryset=default_manager, field=field_name, date_field=field.unique_for_month ) @@ -837,7 +870,7 @@ class ModelSerializer(Serializer): if field.unique_for_year and field_name in field_names: validator = UniqueForYearValidator( - queryset=model_class._default_manager, + queryset=default_manager, field=field_name, date_field=field.unique_for_year ) From 6d907cde9a90aad76acb00482a1d70550bb95ccd Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 12:18:40 +0000 Subject: [PATCH 22/29] get_field_names, get_default_field_names --- rest_framework/serializers.py | 102 +++++++++++++++++++++++---------- tests/test_model_serializer.py | 8 +-- 2 files changed, 76 insertions(+), 34 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 52ea5b0b6..b391a94eb 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -883,41 +883,14 @@ class ModelSerializer(Serializer): ret = OrderedDict() model = getattr(self.Meta, 'model') - fields = getattr(self.Meta, 'fields', None) - exclude = getattr(self.Meta, 'exclude', None) depth = getattr(self.Meta, 'depth', 0) extra_kwargs = getattr(self.Meta, 'extra_kwargs', {}) - - if fields and not isinstance(fields, (list, tuple)): - raise TypeError( - 'The `fields` option must be a list or tuple. Got %s.' % - type(fields).__name__ - ) - - if exclude and not isinstance(exclude, (list, tuple)): - raise TypeError( - 'The `exclude` option must be a list or tuple. Got %s.' % - type(exclude).__name__ - ) - - assert not (fields and exclude), "Cannot set both 'fields' and 'exclude'." - extra_kwargs = self._include_additional_options(extra_kwargs) # Retrieve metadata about fields & relationships on the model class. info = model_meta.get_field_info(model) - # Use the default set of field names if none is supplied explicitly. - if fields is None: - fields = self._get_default_field_names(declared_fields, info) - exclude = getattr(self.Meta, 'exclude', None) - if exclude is not None: - for field_name in exclude: - assert field_name in fields, ( - 'The field in the `exclude` option must be a model field. Got %s.' % - field_name - ) - fields.remove(field_name) + fields = self.get_field_names(declared_fields, info) # Determine the set of model fields, and the fields that they map to. # We actually only need this to deal with the slightly awkward case @@ -1133,7 +1106,72 @@ class ModelSerializer(Serializer): return extra_kwargs - def _get_default_field_names(self, declared_fields, model_info): + def get_field_names(self, declared_fields, info): + """ + Returns the list of all field names that should be created when + instantiating this serializer class. This is based on the default + set of fields, but also takes into account the `Meta.fields` or + `Meta.exclude` options if they have been specified. + """ + fields = getattr(self.Meta, 'fields', None) + exclude = getattr(self.Meta, 'exclude', None) + + if fields and not isinstance(fields, (list, tuple)): + raise TypeError( + 'The `fields` option must be a list or tuple. Got %s.' % + type(fields).__name__ + ) + + if exclude and not isinstance(exclude, (list, tuple)): + raise TypeError( + 'The `exclude` option must be a list or tuple. Got %s.' % + type(exclude).__name__ + ) + + assert not (fields and exclude), ( + "Cannot set both 'fields' and 'exclude' options on " + "serializer {serializer_class}.".format( + serializer_class=self.__class__.__name__ + ) + ) + + if fields is not None: + # Ensure that all declared fields have also been included in the + # `Meta.fields` option. + for field_name in declared_fields: + assert field_name in fields, ( + "The field '{field_name}' was declared on serializer " + "{serializer_class}, but has not been included in the " + "'fields' option.".format( + field_name=field_name, + serializer_class=self.__class__.__name__ + ) + ) + return fields + + # Use the default set of field names if `Meta.fields` is not specified. + fields = self.get_default_field_names(declared_fields, info) + + if exclude is not None: + # If `Meta.exclude` is included, then remove those fields. + for field_name in exclude: + assert field_name in fields, ( + "The field '{field_name}' was include on serializer " + "{serializer_class} in the 'exclude' option, but does " + "not match any model field.".format( + field_name=field_name, + serializer_class=self.__class__.__name__ + ) + ) + fields.remove(field_name) + + return fields + + def get_default_field_names(self, declared_fields, model_info): + """ + Return the default list of field names that will be used if the + `Meta.fields` option is not specified. + """ return ( [model_info.pk.name] + list(declared_fields.keys()) + @@ -1160,7 +1198,11 @@ class HyperlinkedModelSerializer(ModelSerializer): """ _related_class = HyperlinkedRelatedField - def _get_default_field_names(self, declared_fields, model_info): + def get_default_field_names(self, declared_fields, model_info): + """ + Return the default list of field names that will be used if the + `Meta.fields` option is not specified. + """ return ( [api_settings.URL_FIELD_NAME] + list(declared_fields.keys()) + diff --git a/tests/test_model_serializer.py b/tests/test_model_serializer.py index da79164af..5c56c8dbb 100644 --- a/tests/test_model_serializer.py +++ b/tests/test_model_serializer.py @@ -221,11 +221,11 @@ class TestRegularFieldMappings(TestCase): model = RegularFieldsModel fields = ('auto_field',) - with self.assertRaises(ImproperlyConfigured) as excinfo: + with self.assertRaises(AssertionError) as excinfo: TestSerializer().fields expected = ( - 'Field `missing` has been declared on serializer ' - '`TestSerializer`, but is missing from `Meta.fields`.' + "The field 'missing' was declared on serializer TestSerializer, " + "but has not been included in the 'fields' option." ) assert str(excinfo.exception) == expected @@ -607,5 +607,5 @@ class TestSerializerMetaClass(TestCase): exception = result.exception self.assertEqual( str(exception), - "Cannot set both 'fields' and 'exclude'." + "Cannot set both 'fields' and 'exclude' options on serializer ExampleSerializer." ) From 1a84943a006abffb7e1b3b3ff55441c7a1132fa2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 12:27:50 +0000 Subject: [PATCH 23/29] get_extra_kwargs --- rest_framework/serializers.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index b391a94eb..d4b0926e6 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -884,13 +884,12 @@ class ModelSerializer(Serializer): ret = OrderedDict() model = getattr(self.Meta, 'model') depth = getattr(self.Meta, 'depth', 0) - extra_kwargs = getattr(self.Meta, 'extra_kwargs', {}) - extra_kwargs = self._include_additional_options(extra_kwargs) # Retrieve metadata about fields & relationships on the model class. info = model_meta.get_field_info(model) fields = self.get_field_names(declared_fields, info) + extra_kwargs = self.get_extra_kwargs() # Determine the set of model fields, and the fields that they map to. # We actually only need this to deal with the slightly awkward case @@ -1024,17 +1023,6 @@ class ModelSerializer(Serializer): (field_name, model.__class__.__name__) ) - # Check that any fields declared on the class are - # also explicitly included in `Meta.fields`. - missing_fields = set(declared_fields.keys()) - set(fields) - if missing_fields: - missing_field = list(missing_fields)[0] - raise ImproperlyConfigured( - 'Field `%s` has been declared on serializer `%s`, but ' - 'is missing from `Meta.fields`.' % - (missing_field, self.__class__.__name__) - ) - # Populate any kwargs defined in `Meta.extra_kwargs` extras = extra_kwargs.get(field_name, {}) if extras.get('read_only', False): @@ -1058,7 +1046,13 @@ class ModelSerializer(Serializer): return ret - def _include_additional_options(self, extra_kwargs): + def get_extra_kwargs(self): + """ + Return a dictionary mapping field names to a dictionary of + additional keyword arguments. + """ + extra_kwargs = getattr(self.Meta, 'extra_kwargs', {}) + read_only_fields = getattr(self.Meta, 'read_only_fields', None) if read_only_fields is not None: for field_name in read_only_fields: From caa13181244ce3c074f647510bb38d7b0c8b4c70 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 13:13:20 +0000 Subject: [PATCH 24/29] get_uniqueness_field_options first pass --- rest_framework/serializers.py | 174 +++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 78 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index d4b0926e6..5e9cbe361 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -888,89 +888,19 @@ class ModelSerializer(Serializer): # Retrieve metadata about fields & relationships on the model class. info = model_meta.get_field_info(model) - fields = self.get_field_names(declared_fields, info) + field_names = self.get_field_names(declared_fields, info) extra_kwargs = self.get_extra_kwargs() - # Determine the set of model fields, and the fields that they map to. - # We actually only need this to deal with the slightly awkward case - # of supporting `unique_for_date`/`unique_for_month`/`unique_for_year`. - model_field_mapping = {} - for field_name in fields: - if field_name in declared_fields: - field = declared_fields[field_name] - source = field.source or field_name + model_fields = self.get_model_fields(field_names, declared_fields, extra_kwargs) + uniqueness_extra_kwargs, hidden_fields = self.get_uniqueness_field_options(field_names, model_fields) + for key, value in uniqueness_extra_kwargs.items(): + if key in extra_kwargs: + extra_kwargs[key].update(value) else: - try: - source = extra_kwargs[field_name]['source'] - except KeyError: - source = field_name - # Model fields will always have a simple source mapping, - # they can't be nested attribute lookups. - if '.' not in source and source != '*': - model_field_mapping[source] = field_name - - # Determine if we need any additional `HiddenField` or extra keyword - # arguments to deal with `unique_for` dates that are required to - # be in the input data in order to validate it. - hidden_fields = {} - unique_constraint_names = set() - - for model_field_name, field_name in model_field_mapping.items(): - try: - model_field = model._meta.get_field(model_field_name) - except FieldDoesNotExist: - continue - - # Include each of the `unique_for_*` field names. - unique_constraint_names |= set([ - model_field.unique_for_date, - model_field.unique_for_month, - model_field.unique_for_year - ]) - - unique_constraint_names -= set([None]) - - # Include each of the `unique_together` field names, - # so long as all the field names are included on the serializer. - for parent_class in [model] + list(model._meta.parents.keys()): - for unique_together_list in parent_class._meta.unique_together: - if set(fields).issuperset(set(unique_together_list)): - unique_constraint_names |= set(unique_together_list) - - # Now we have all the field names that have uniqueness constraints - # applied, we can add the extra 'required=...' or 'default=...' - # arguments that are appropriate to these fields, or add a `HiddenField` for it. - for unique_constraint_name in unique_constraint_names: - # Get the model field that is referred too. - unique_constraint_field = model._meta.get_field(unique_constraint_name) - - if getattr(unique_constraint_field, 'auto_now_add', None): - default = CreateOnlyDefault(timezone.now) - elif getattr(unique_constraint_field, 'auto_now', None): - default = timezone.now - elif unique_constraint_field.has_default(): - default = unique_constraint_field.default - else: - default = empty - - if unique_constraint_name in model_field_mapping: - # The corresponding field is present in the serializer - if unique_constraint_name not in extra_kwargs: - extra_kwargs[unique_constraint_name] = {} - if default is empty: - if 'required' not in extra_kwargs[unique_constraint_name]: - extra_kwargs[unique_constraint_name]['required'] = True - else: - if 'default' not in extra_kwargs[unique_constraint_name]: - extra_kwargs[unique_constraint_name]['default'] = default - elif default is not empty: - # The corresponding field is not present in the, - # serializer. We have a default to use for it, so - # add in a hidden field that populates it. - hidden_fields[unique_constraint_name] = HiddenField(default=default) + extra_kwargs[key] = value # Now determine the fields that should be included on the serializer. - for field_name in fields: + for field_name in field_names: if field_name in declared_fields: # Field is explicitly declared on the class, use that. ret[field_name] = declared_fields[field_name] @@ -1046,6 +976,94 @@ class ModelSerializer(Serializer): return ret + def get_model_fields(self, field_names, declared_fields, extra_kwargs): + # Returns all the model fields that are being mapped to by fields + # on the serializer class. + # Returned as a dict of 'model field name' -> 'model field' + model = getattr(self.Meta, 'model') + model_fields = {} + + for field_name in field_names: + if field_name in declared_fields: + # If the field is declared on the serializer + field = declared_fields[field_name] + source = field.source or field_name + else: + try: + source = extra_kwargs[field_name]['source'] + except KeyError: + source = field_name + + if '.' in source or source == '*': + # Model fields will always have a simple source mapping, + # they can't be nested attribute lookups. + continue + + try: + model_fields[source] = model._meta.get_field(source) + except FieldDoesNotExist: + pass + + return model_fields + + def get_uniqueness_field_options(self, field_names, model_fields): + model = getattr(self.Meta, 'model') + + # Determine if we need any additional `HiddenField` or extra keyword + # arguments to deal with `unique_for` dates that are required to + # be in the input data in order to validate it. + unique_constraint_names = set() + + for model_field in model_fields.values(): + # Include each of the `unique_for_*` field names. + unique_constraint_names |= set([ + model_field.unique_for_date, + model_field.unique_for_month, + model_field.unique_for_year + ]) + + unique_constraint_names -= set([None]) + + # Include each of the `unique_together` field names, + # so long as all the field names are included on the serializer. + for parent_class in [model] + list(model._meta.parents.keys()): + for unique_together_list in parent_class._meta.unique_together: + if set(field_names).issuperset(set(unique_together_list)): + unique_constraint_names |= set(unique_together_list) + + # Now we have all the field names that have uniqueness constraints + # applied, we can add the extra 'required=...' or 'default=...' + # arguments that are appropriate to these fields, or add a `HiddenField` for it. + hidden_fields = {} + extra_kwargs = {} + + for unique_constraint_name in unique_constraint_names: + # Get the model field that is referred too. + unique_constraint_field = model._meta.get_field(unique_constraint_name) + + if getattr(unique_constraint_field, 'auto_now_add', None): + default = CreateOnlyDefault(timezone.now) + elif getattr(unique_constraint_field, 'auto_now', None): + default = timezone.now + elif unique_constraint_field.has_default(): + default = unique_constraint_field.default + else: + default = empty + + if unique_constraint_name in model_fields: + # The corresponding field is present in the serializer + if default is empty: + extra_kwargs[unique_constraint_name] = {'required': True} + else: + extra_kwargs[unique_constraint_name] = {'default': default} + elif default is not empty: + # The corresponding field is not present in the, + # serializer. We have a default to use for it, so + # add in a hidden field that populates it. + hidden_fields[unique_constraint_name] = HiddenField(default=default) + + return extra_kwargs, hidden_fields + def get_extra_kwargs(self): """ Return a dictionary mapping field names to a dictionary of From 4a112fc3a616238b7995b3a442ae236116364ceb Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 14:51:45 +0000 Subject: [PATCH 25/29] Clean up --- rest_framework/serializers.py | 63 +++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 5e9cbe361..093b0eb51 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -722,6 +722,8 @@ class ModelSerializer(Serializer): }) _related_class = PrimaryKeyRelatedField + # Default `create` and `update` behavior... + def create(self, validated_data): """ We have a bit of extra checking around this in order to provide @@ -791,6 +793,8 @@ class ModelSerializer(Serializer): return instance + # Determine the validators to apply... + def get_validators(self): """ Determine the set of validators to use when instantiating serializer. @@ -878,28 +882,26 @@ class ModelSerializer(Serializer): return validators + # Determine the fields to apply... + def get_fields(self): declared_fields = copy.deepcopy(self._declared_fields) - - ret = OrderedDict() model = getattr(self.Meta, 'model') depth = getattr(self.Meta, 'depth', 0) # Retrieve metadata about fields & relationships on the model class. info = model_meta.get_field_info(model) - field_names = self.get_field_names(declared_fields, info) - extra_kwargs = self.get_extra_kwargs() - model_fields = self.get_model_fields(field_names, declared_fields, extra_kwargs) - uniqueness_extra_kwargs, hidden_fields = self.get_uniqueness_field_options(field_names, model_fields) - for key, value in uniqueness_extra_kwargs.items(): - if key in extra_kwargs: - extra_kwargs[key].update(value) - else: - extra_kwargs[key] = value + # Determine any extra field arguments and hidden fields that + # should be included + extra_kwargs = self.get_extra_kwargs() + extra_kwargs, hidden_fields = self.get_uniqueness_extra_kwargs( + field_names, declared_fields, extra_kwargs + ) # Now determine the fields that should be included on the serializer. + ret = OrderedDict() for field_name in field_names: if field_name in declared_fields: # Field is explicitly declared on the class, use that. @@ -971,15 +973,17 @@ class ModelSerializer(Serializer): # Create the serializer field. ret[field_name] = field_cls(**kwargs) - for field_name, field in hidden_fields.items(): - ret[field_name] = field + ret.update(hidden_fields) return ret - def get_model_fields(self, field_names, declared_fields, extra_kwargs): - # Returns all the model fields that are being mapped to by fields - # on the serializer class. - # Returned as a dict of 'model field name' -> 'model field' + def _get_model_fields(self, field_names, declared_fields, extra_kwargs): + """ + Returns all the model fields that are being mapped to by fields + on the serializer class. + Returned as a dict of 'model field name' -> 'model field'. + Used internally by `get_uniqueness_field_options`. + """ model = getattr(self.Meta, 'model') model_fields = {} @@ -1006,8 +1010,18 @@ class ModelSerializer(Serializer): return model_fields - def get_uniqueness_field_options(self, field_names, model_fields): + def get_uniqueness_extra_kwargs(self, field_names, declared_fields, extra_kwargs): + """ + Return any additional field options that need to be included as a + result of uniqueness constraints on the model. This is returned as + a two-tuple of: + + ('dict of updated extra kwargs', 'mapping of hidden fields') + """ model = getattr(self.Meta, 'model') + model_fields = self._get_model_fields( + field_names, declared_fields, extra_kwargs + ) # Determine if we need any additional `HiddenField` or extra keyword # arguments to deal with `unique_for` dates that are required to @@ -1035,7 +1049,7 @@ class ModelSerializer(Serializer): # applied, we can add the extra 'required=...' or 'default=...' # arguments that are appropriate to these fields, or add a `HiddenField` for it. hidden_fields = {} - extra_kwargs = {} + uniqueness_extra_kwargs = {} for unique_constraint_name in unique_constraint_names: # Get the model field that is referred too. @@ -1053,15 +1067,22 @@ class ModelSerializer(Serializer): if unique_constraint_name in model_fields: # The corresponding field is present in the serializer if default is empty: - extra_kwargs[unique_constraint_name] = {'required': True} + uniqueness_extra_kwargs[unique_constraint_name] = {'required': True} else: - extra_kwargs[unique_constraint_name] = {'default': default} + uniqueness_extra_kwargs[unique_constraint_name] = {'default': default} elif default is not empty: # The corresponding field is not present in the, # serializer. We have a default to use for it, so # add in a hidden field that populates it. hidden_fields[unique_constraint_name] = HiddenField(default=default) + # Update `extra_kwargs` with any new options. + for key, value in uniqueness_extra_kwargs.items(): + if key in extra_kwargs: + extra_kwargs[key].update(value) + else: + extra_kwargs[key] = value + return extra_kwargs, hidden_fields def get_extra_kwargs(self): From f72928ea982cfe2127288dd6dc52f8006638b0c3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 15:09:57 +0000 Subject: [PATCH 26/29] build_field, build_final_kwargs --- rest_framework/serializers.py | 135 +++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 59 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 1f76c4c19..80ad10f0a 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -908,75 +908,92 @@ class ModelSerializer(Serializer): ret[field_name] = declared_fields[field_name] continue - elif field_name in info.fields_and_pk: - # Create regular model fields. - model_field = info.fields_and_pk[field_name] - field_cls = self._field_mapping[model_field] - kwargs = get_field_kwargs(field_name, model_field) - if 'choices' in kwargs: - # Fields with choices get coerced into `ChoiceField` - # instead of using their regular typed field. - field_cls = ChoiceField - if not issubclass(field_cls, ModelField): - # `model_field` is only valid for the fallback case of - # `ModelField`, which is used when no other typed field - # matched to the model field. - kwargs.pop('model_field', None) - if not issubclass(field_cls, CharField) and not issubclass(field_cls, ChoiceField): - # `allow_blank` is only valid for textual fields. - kwargs.pop('allow_blank', None) - - elif field_name in info.relations: - # Create forward and reverse relationships. - relation_info = info.relations[field_name] - if depth: - field_cls = self._get_nested_class(depth, relation_info) - kwargs = get_nested_relation_kwargs(relation_info) - else: - field_cls = self._related_class - kwargs = get_relation_kwargs(field_name, relation_info) - # `view_name` is only valid for hyperlinked relationships. - if not issubclass(field_cls, HyperlinkedRelatedField): - kwargs.pop('view_name', None) - - elif hasattr(model, field_name): - # Create a read only field for model methods and properties. - field_cls = ReadOnlyField - kwargs = {} - - elif field_name == api_settings.URL_FIELD_NAME: - # Create the URL field. - field_cls = HyperlinkedIdentityField - kwargs = get_url_kwargs(model) - - else: - raise ImproperlyConfigured( - 'Field name `%s` is not valid for model `%s`.' % - (field_name, model.__class__.__name__) - ) + # Determine the serializer field class and keyword arguments. + field_cls, kwargs = self.build_field(field_name, info, model, depth) # Populate any kwargs defined in `Meta.extra_kwargs` - extras = extra_kwargs.get(field_name, {}) - if extras.get('read_only', False): - for attr in [ - 'required', 'default', 'allow_blank', 'allow_null', - 'min_length', 'max_length', 'min_value', 'max_value', - 'validators', 'queryset' - ]: - kwargs.pop(attr, None) - - if extras.get('default') and kwargs.get('required') is False: - kwargs.pop('required') - - kwargs.update(extras) + kwargs = self.build_final_kwargs(kwargs, extra_kwargs, field_name) # Create the serializer field. ret[field_name] = field_cls(**kwargs) + # Add in any hidden fields. ret.update(hidden_fields) return ret + def build_field(self, field_name, info, model, depth): + if field_name in info.fields_and_pk: + # Create regular model fields. + model_field = info.fields_and_pk[field_name] + field_cls = self._field_mapping[model_field] + kwargs = get_field_kwargs(field_name, model_field) + if 'choices' in kwargs: + # Fields with choices get coerced into `ChoiceField` + # instead of using their regular typed field. + field_cls = ChoiceField + if not issubclass(field_cls, ModelField): + # `model_field` is only valid for the fallback case of + # `ModelField`, which is used when no other typed field + # matched to the model field. + kwargs.pop('model_field', None) + if not issubclass(field_cls, CharField) and not issubclass(field_cls, ChoiceField): + # `allow_blank` is only valid for textual fields. + kwargs.pop('allow_blank', None) + + elif field_name in info.relations: + # Create forward and reverse relationships. + relation_info = info.relations[field_name] + if depth: + field_cls = self._get_nested_class(depth, relation_info) + kwargs = get_nested_relation_kwargs(relation_info) + else: + field_cls = self._related_class + kwargs = get_relation_kwargs(field_name, relation_info) + # `view_name` is only valid for hyperlinked relationships. + if not issubclass(field_cls, HyperlinkedRelatedField): + kwargs.pop('view_name', None) + + elif hasattr(model, field_name): + # Create a read only field for model methods and properties. + field_cls = ReadOnlyField + kwargs = {} + + elif field_name == api_settings.URL_FIELD_NAME: + # Create the URL field. + field_cls = HyperlinkedIdentityField + kwargs = get_url_kwargs(model) + + else: + raise ImproperlyConfigured( + 'Field name `%s` is not valid for model `%s`.' % + (field_name, model.__class__.__name__) + ) + + return field_cls, kwargs + + def build_final_kwargs(self, kwargs, extra_kwargs, field_name): + """ + Include an 'extra_kwargs' that have been included for this field, + possibly removing any incompatible existing keyword arguments. + """ + extras = extra_kwargs.get(field_name, {}) + + if extras.get('read_only', False): + for attr in [ + 'required', 'default', 'allow_blank', 'allow_null', + 'min_length', 'max_length', 'min_value', 'max_value', + 'validators', 'queryset' + ]: + kwargs.pop(attr, None) + + if extras.get('default') and kwargs.get('required') is False: + kwargs.pop('required') + + kwargs.update(extras) + + return kwargs + def _get_model_fields(self, field_names, declared_fields, extra_kwargs): """ Returns all the model fields that are being mapped to by fields From 75e81b82545704bac8afdf3270ba9f6c8da09c27 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 15:35:52 +0000 Subject: [PATCH 27/29] build_*_field methods --- rest_framework/serializers.py | 785 ++++++++++++++++++---------------- 1 file changed, 424 insertions(+), 361 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 80ad10f0a..a983d3fc7 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -696,7 +696,7 @@ class ModelSerializer(Serializer): you need you should either declare the extra/differing fields explicitly on the serializer class, or simply use a `Serializer` class. """ - _field_mapping = ClassLookupDict({ + serializer_field_mapping = { models.AutoField: IntegerField, models.BigIntegerField: IntegerField, models.BooleanField: BooleanField, @@ -719,8 +719,8 @@ class ModelSerializer(Serializer): models.TextField: CharField, models.TimeField: TimeField, models.URLField: URLField, - }) - _related_class = PrimaryKeyRelatedField + } + serializer_related_class = PrimaryKeyRelatedField # Default `create` and `update` behavior... @@ -793,6 +793,417 @@ class ModelSerializer(Serializer): return instance + # Determine the fields to apply... + + def get_fields(self): + """ + Return the dict of field names -> field instances that should be + used for `self.fields` when instantiating the serializer. + """ + declared_fields = copy.deepcopy(self._declared_fields) + model = getattr(self.Meta, 'model') + depth = getattr(self.Meta, 'depth', 0) + + # Retrieve metadata about fields & relationships on the model class. + info = model_meta.get_field_info(model) + field_names = self.get_field_names(declared_fields, info) + + # Determine any extra field arguments and hidden fields that + # should be included + extra_kwargs = self.get_extra_kwargs() + extra_kwargs, hidden_fields = self.get_uniqueness_extra_kwargs( + field_names, declared_fields, extra_kwargs + ) + + # Now determine the fields that should be included on the serializer. + ret = OrderedDict() + for field_name in field_names: + if field_name in declared_fields: + # Field is explicitly declared on the class, use that. + ret[field_name] = declared_fields[field_name] + continue + + # Determine the serializer field class and keyword arguments. + field_cls, kwargs = self.build_field(field_name, info, model, depth) + + # Populate any kwargs defined in `Meta.extra_kwargs` + kwargs = self.build_field_kwargs(kwargs, extra_kwargs, field_name) + + # Create the serializer field. + ret[field_name] = field_cls(**kwargs) + + # Add in any hidden fields. + ret.update(hidden_fields) + + return ret + + # Methods for determining the set of field names to include... + + def get_field_names(self, declared_fields, info): + """ + Returns the list of all field names that should be created when + instantiating this serializer class. This is based on the default + set of fields, but also takes into account the `Meta.fields` or + `Meta.exclude` options if they have been specified. + """ + fields = getattr(self.Meta, 'fields', None) + exclude = getattr(self.Meta, 'exclude', None) + + if fields and not isinstance(fields, (list, tuple)): + raise TypeError( + 'The `fields` option must be a list or tuple. Got %s.' % + type(fields).__name__ + ) + + if exclude and not isinstance(exclude, (list, tuple)): + raise TypeError( + 'The `exclude` option must be a list or tuple. Got %s.' % + type(exclude).__name__ + ) + + assert not (fields and exclude), ( + "Cannot set both 'fields' and 'exclude' options on " + "serializer {serializer_class}.".format( + serializer_class=self.__class__.__name__ + ) + ) + + if fields is not None: + # Ensure that all declared fields have also been included in the + # `Meta.fields` option. + for field_name in declared_fields: + assert field_name in fields, ( + "The field '{field_name}' was declared on serializer " + "{serializer_class}, but has not been included in the " + "'fields' option.".format( + field_name=field_name, + serializer_class=self.__class__.__name__ + ) + ) + return fields + + # Use the default set of field names if `Meta.fields` is not specified. + fields = self.get_default_field_names(declared_fields, info) + + if exclude is not None: + # If `Meta.exclude` is included, then remove those fields. + for field_name in exclude: + assert field_name in fields, ( + "The field '{field_name}' was include on serializer " + "{serializer_class} in the 'exclude' option, but does " + "not match any model field.".format( + field_name=field_name, + serializer_class=self.__class__.__name__ + ) + ) + fields.remove(field_name) + + return fields + + def get_default_field_names(self, declared_fields, model_info): + """ + Return the default list of field names that will be used if the + `Meta.fields` option is not specified. + """ + return ( + [model_info.pk.name] + + list(declared_fields.keys()) + + list(model_info.fields.keys()) + + list(model_info.forward_relations.keys()) + ) + + # Methods for constructing serializer fields... + + def build_field(self, field_name, info, model, nested_depth): + """ + Return a two tuple of (cls, kwargs) to build a serializer field with. + """ + if field_name in info.fields_and_pk: + return self.build_standard_field(field_name, info, model) + + elif field_name in info.relations: + if not nested_depth: + return self.build_relational_field(field_name, info, model) + else: + return self.build_nested_field(field_name, info, model, nested_depth) + + elif hasattr(model, field_name): + return self.build_property_field(field_name, info, model) + + elif field_name == api_settings.URL_FIELD_NAME: + return self.build_url_field(field_name, info, model) + + return self.build_unknown_field(field_name, info, model) + + def build_standard_field(self, field_name, info, model): + """ + Create regular model fields. + """ + field_mapping = ClassLookupDict(self.serializer_field_mapping) + model_field = info.fields_and_pk[field_name] + + field_cls = field_mapping[model_field] + kwargs = get_field_kwargs(field_name, model_field) + + if 'choices' in kwargs: + # Fields with choices get coerced into `ChoiceField` + # instead of using their regular typed field. + field_cls = ChoiceField + if not issubclass(field_cls, ModelField): + # `model_field` is only valid for the fallback case of + # `ModelField`, which is used when no other typed field + # matched to the model field. + kwargs.pop('model_field', None) + if not issubclass(field_cls, CharField) and not issubclass(field_cls, ChoiceField): + # `allow_blank` is only valid for textual fields. + kwargs.pop('allow_blank', None) + + return field_cls, kwargs + + def build_relational_field(self, field_name, info, model): + """ + Create fields for forward and reverse relationships. + """ + relation_info = info.relations[field_name] + + field_cls = self.serializer_related_class + kwargs = get_relation_kwargs(field_name, relation_info) + + # `view_name` is only valid for hyperlinked relationships. + if not issubclass(field_cls, HyperlinkedRelatedField): + kwargs.pop('view_name', None) + + return field_cls, kwargs + + def build_nested_field(self, field_name, info, model, nested_depth): + """ + Create nested fields for forward and reverse relationships. + """ + relation_info = info.relations[field_name] + + class NestedSerializer(ModelSerializer): + class Meta: + model = relation_info.related + depth = nested_depth - 1 + + field_cls = NestedSerializer + kwargs = get_nested_relation_kwargs(relation_info) + + return field_cls, kwargs + + def build_property_field(self, field_name, info, model): + """ + Create a read only field for model methods and properties. + """ + field_cls = ReadOnlyField + kwargs = {} + + return field_cls, kwargs + + def build_url_field(self, field_name, info, model): + """ + Create a field representing the object's own URL. + """ + field_cls = HyperlinkedIdentityField + kwargs = get_url_kwargs(model) + + return field_cls, kwargs + + def build_unknown_field(self, field_name, info, model): + """ + Raise an error on any unknown fields. + """ + raise ImproperlyConfigured( + 'Field name `%s` is not valid for model `%s`.' % + (field_name, model.__class__.__name__) + ) + + def build_field_kwargs(self, kwargs, extra_kwargs, field_name): + """ + Include an 'extra_kwargs' that have been included for this field, + possibly removing any incompatible existing keyword arguments. + """ + extras = extra_kwargs.get(field_name, {}) + + if extras.get('read_only', False): + for attr in [ + 'required', 'default', 'allow_blank', 'allow_null', + 'min_length', 'max_length', 'min_value', 'max_value', + 'validators', 'queryset' + ]: + kwargs.pop(attr, None) + + if extras.get('default') and kwargs.get('required') is False: + kwargs.pop('required') + + kwargs.update(extras) + + return kwargs + + # Methods for determining additional keyword arguments to apply... + + def get_extra_kwargs(self): + """ + Return a dictionary mapping field names to a dictionary of + additional keyword arguments. + """ + extra_kwargs = getattr(self.Meta, 'extra_kwargs', {}) + + read_only_fields = getattr(self.Meta, 'read_only_fields', None) + if read_only_fields is not None: + for field_name in read_only_fields: + kwargs = extra_kwargs.get(field_name, {}) + kwargs['read_only'] = True + extra_kwargs[field_name] = kwargs + + # These are all pending deprecation. + write_only_fields = getattr(self.Meta, 'write_only_fields', None) + if write_only_fields is not None: + warnings.warn( + "The `Meta.write_only_fields` option is pending deprecation. " + "Use `Meta.extra_kwargs={: {'write_only': True}}` instead.", + PendingDeprecationWarning, + stacklevel=3 + ) + for field_name in write_only_fields: + kwargs = extra_kwargs.get(field_name, {}) + kwargs['write_only'] = True + extra_kwargs[field_name] = kwargs + + view_name = getattr(self.Meta, 'view_name', None) + if view_name is not None: + warnings.warn( + "The `Meta.view_name` option is pending deprecation. " + "Use `Meta.extra_kwargs={'url': {'view_name': ...}}` instead.", + PendingDeprecationWarning, + stacklevel=3 + ) + kwargs = extra_kwargs.get(api_settings.URL_FIELD_NAME, {}) + kwargs['view_name'] = view_name + extra_kwargs[api_settings.URL_FIELD_NAME] = kwargs + + lookup_field = getattr(self.Meta, 'lookup_field', None) + if lookup_field is not None: + warnings.warn( + "The `Meta.lookup_field` option is pending deprecation. " + "Use `Meta.extra_kwargs={'url': {'lookup_field': ...}}` instead.", + PendingDeprecationWarning, + stacklevel=3 + ) + kwargs = extra_kwargs.get(api_settings.URL_FIELD_NAME, {}) + kwargs['lookup_field'] = lookup_field + extra_kwargs[api_settings.URL_FIELD_NAME] = kwargs + + return extra_kwargs + + def get_uniqueness_extra_kwargs(self, field_names, declared_fields, extra_kwargs): + """ + Return any additional field options that need to be included as a + result of uniqueness constraints on the model. This is returned as + a two-tuple of: + + ('dict of updated extra kwargs', 'mapping of hidden fields') + """ + model = getattr(self.Meta, 'model') + model_fields = self._get_model_fields( + field_names, declared_fields, extra_kwargs + ) + + # Determine if we need any additional `HiddenField` or extra keyword + # arguments to deal with `unique_for` dates that are required to + # be in the input data in order to validate it. + unique_constraint_names = set() + + for model_field in model_fields.values(): + # Include each of the `unique_for_*` field names. + unique_constraint_names |= set([ + model_field.unique_for_date, + model_field.unique_for_month, + model_field.unique_for_year + ]) + + unique_constraint_names -= set([None]) + + # Include each of the `unique_together` field names, + # so long as all the field names are included on the serializer. + for parent_class in [model] + list(model._meta.parents.keys()): + for unique_together_list in parent_class._meta.unique_together: + if set(field_names).issuperset(set(unique_together_list)): + unique_constraint_names |= set(unique_together_list) + + # Now we have all the field names that have uniqueness constraints + # applied, we can add the extra 'required=...' or 'default=...' + # arguments that are appropriate to these fields, or add a `HiddenField` for it. + hidden_fields = {} + uniqueness_extra_kwargs = {} + + for unique_constraint_name in unique_constraint_names: + # Get the model field that is referred too. + unique_constraint_field = model._meta.get_field(unique_constraint_name) + + if getattr(unique_constraint_field, 'auto_now_add', None): + default = CreateOnlyDefault(timezone.now) + elif getattr(unique_constraint_field, 'auto_now', None): + default = timezone.now + elif unique_constraint_field.has_default(): + default = unique_constraint_field.default + else: + default = empty + + if unique_constraint_name in model_fields: + # The corresponding field is present in the serializer + if default is empty: + uniqueness_extra_kwargs[unique_constraint_name] = {'required': True} + else: + uniqueness_extra_kwargs[unique_constraint_name] = {'default': default} + elif default is not empty: + # The corresponding field is not present in the, + # serializer. We have a default to use for it, so + # add in a hidden field that populates it. + hidden_fields[unique_constraint_name] = HiddenField(default=default) + + # Update `extra_kwargs` with any new options. + for key, value in uniqueness_extra_kwargs.items(): + if key in extra_kwargs: + extra_kwargs[key].update(value) + else: + extra_kwargs[key] = value + + return extra_kwargs, hidden_fields + + def _get_model_fields(self, field_names, declared_fields, extra_kwargs): + """ + Returns all the model fields that are being mapped to by fields + on the serializer class. + Returned as a dict of 'model field name' -> 'model field'. + Used internally by `get_uniqueness_field_options`. + """ + model = getattr(self.Meta, 'model') + model_fields = {} + + for field_name in field_names: + if field_name in declared_fields: + # If the field is declared on the serializer + field = declared_fields[field_name] + source = field.source or field_name + else: + try: + source = extra_kwargs[field_name]['source'] + except KeyError: + source = field_name + + if '.' in source or source == '*': + # Model fields will always have a simple source mapping, + # they can't be nested attribute lookups. + continue + + try: + model_fields[source] = model._meta.get_field(source) + except FieldDoesNotExist: + pass + + return model_fields + # Determine the validators to apply... def get_validators(self): @@ -882,361 +1293,6 @@ class ModelSerializer(Serializer): return validators - # Determine the fields to apply... - - def get_fields(self): - declared_fields = copy.deepcopy(self._declared_fields) - model = getattr(self.Meta, 'model') - depth = getattr(self.Meta, 'depth', 0) - - # Retrieve metadata about fields & relationships on the model class. - info = model_meta.get_field_info(model) - field_names = self.get_field_names(declared_fields, info) - - # Determine any extra field arguments and hidden fields that - # should be included - extra_kwargs = self.get_extra_kwargs() - extra_kwargs, hidden_fields = self.get_uniqueness_extra_kwargs( - field_names, declared_fields, extra_kwargs - ) - - # Now determine the fields that should be included on the serializer. - ret = OrderedDict() - for field_name in field_names: - if field_name in declared_fields: - # Field is explicitly declared on the class, use that. - ret[field_name] = declared_fields[field_name] - continue - - # Determine the serializer field class and keyword arguments. - field_cls, kwargs = self.build_field(field_name, info, model, depth) - - # Populate any kwargs defined in `Meta.extra_kwargs` - kwargs = self.build_final_kwargs(kwargs, extra_kwargs, field_name) - - # Create the serializer field. - ret[field_name] = field_cls(**kwargs) - - # Add in any hidden fields. - ret.update(hidden_fields) - - return ret - - def build_field(self, field_name, info, model, depth): - if field_name in info.fields_and_pk: - # Create regular model fields. - model_field = info.fields_and_pk[field_name] - field_cls = self._field_mapping[model_field] - kwargs = get_field_kwargs(field_name, model_field) - if 'choices' in kwargs: - # Fields with choices get coerced into `ChoiceField` - # instead of using their regular typed field. - field_cls = ChoiceField - if not issubclass(field_cls, ModelField): - # `model_field` is only valid for the fallback case of - # `ModelField`, which is used when no other typed field - # matched to the model field. - kwargs.pop('model_field', None) - if not issubclass(field_cls, CharField) and not issubclass(field_cls, ChoiceField): - # `allow_blank` is only valid for textual fields. - kwargs.pop('allow_blank', None) - - elif field_name in info.relations: - # Create forward and reverse relationships. - relation_info = info.relations[field_name] - if depth: - field_cls = self._get_nested_class(depth, relation_info) - kwargs = get_nested_relation_kwargs(relation_info) - else: - field_cls = self._related_class - kwargs = get_relation_kwargs(field_name, relation_info) - # `view_name` is only valid for hyperlinked relationships. - if not issubclass(field_cls, HyperlinkedRelatedField): - kwargs.pop('view_name', None) - - elif hasattr(model, field_name): - # Create a read only field for model methods and properties. - field_cls = ReadOnlyField - kwargs = {} - - elif field_name == api_settings.URL_FIELD_NAME: - # Create the URL field. - field_cls = HyperlinkedIdentityField - kwargs = get_url_kwargs(model) - - else: - raise ImproperlyConfigured( - 'Field name `%s` is not valid for model `%s`.' % - (field_name, model.__class__.__name__) - ) - - return field_cls, kwargs - - def build_final_kwargs(self, kwargs, extra_kwargs, field_name): - """ - Include an 'extra_kwargs' that have been included for this field, - possibly removing any incompatible existing keyword arguments. - """ - extras = extra_kwargs.get(field_name, {}) - - if extras.get('read_only', False): - for attr in [ - 'required', 'default', 'allow_blank', 'allow_null', - 'min_length', 'max_length', 'min_value', 'max_value', - 'validators', 'queryset' - ]: - kwargs.pop(attr, None) - - if extras.get('default') and kwargs.get('required') is False: - kwargs.pop('required') - - kwargs.update(extras) - - return kwargs - - def _get_model_fields(self, field_names, declared_fields, extra_kwargs): - """ - Returns all the model fields that are being mapped to by fields - on the serializer class. - Returned as a dict of 'model field name' -> 'model field'. - Used internally by `get_uniqueness_field_options`. - """ - model = getattr(self.Meta, 'model') - model_fields = {} - - for field_name in field_names: - if field_name in declared_fields: - # If the field is declared on the serializer - field = declared_fields[field_name] - source = field.source or field_name - else: - try: - source = extra_kwargs[field_name]['source'] - except KeyError: - source = field_name - - if '.' in source or source == '*': - # Model fields will always have a simple source mapping, - # they can't be nested attribute lookups. - continue - - try: - model_fields[source] = model._meta.get_field(source) - except FieldDoesNotExist: - pass - - return model_fields - - def get_uniqueness_extra_kwargs(self, field_names, declared_fields, extra_kwargs): - """ - Return any additional field options that need to be included as a - result of uniqueness constraints on the model. This is returned as - a two-tuple of: - - ('dict of updated extra kwargs', 'mapping of hidden fields') - """ - model = getattr(self.Meta, 'model') - model_fields = self._get_model_fields( - field_names, declared_fields, extra_kwargs - ) - - # Determine if we need any additional `HiddenField` or extra keyword - # arguments to deal with `unique_for` dates that are required to - # be in the input data in order to validate it. - unique_constraint_names = set() - - for model_field in model_fields.values(): - # Include each of the `unique_for_*` field names. - unique_constraint_names |= set([ - model_field.unique_for_date, - model_field.unique_for_month, - model_field.unique_for_year - ]) - - unique_constraint_names -= set([None]) - - # Include each of the `unique_together` field names, - # so long as all the field names are included on the serializer. - for parent_class in [model] + list(model._meta.parents.keys()): - for unique_together_list in parent_class._meta.unique_together: - if set(field_names).issuperset(set(unique_together_list)): - unique_constraint_names |= set(unique_together_list) - - # Now we have all the field names that have uniqueness constraints - # applied, we can add the extra 'required=...' or 'default=...' - # arguments that are appropriate to these fields, or add a `HiddenField` for it. - hidden_fields = {} - uniqueness_extra_kwargs = {} - - for unique_constraint_name in unique_constraint_names: - # Get the model field that is referred too. - unique_constraint_field = model._meta.get_field(unique_constraint_name) - - if getattr(unique_constraint_field, 'auto_now_add', None): - default = CreateOnlyDefault(timezone.now) - elif getattr(unique_constraint_field, 'auto_now', None): - default = timezone.now - elif unique_constraint_field.has_default(): - default = unique_constraint_field.default - else: - default = empty - - if unique_constraint_name in model_fields: - # The corresponding field is present in the serializer - if default is empty: - uniqueness_extra_kwargs[unique_constraint_name] = {'required': True} - else: - uniqueness_extra_kwargs[unique_constraint_name] = {'default': default} - elif default is not empty: - # The corresponding field is not present in the, - # serializer. We have a default to use for it, so - # add in a hidden field that populates it. - hidden_fields[unique_constraint_name] = HiddenField(default=default) - - # Update `extra_kwargs` with any new options. - for key, value in uniqueness_extra_kwargs.items(): - if key in extra_kwargs: - extra_kwargs[key].update(value) - else: - extra_kwargs[key] = value - - return extra_kwargs, hidden_fields - - def get_extra_kwargs(self): - """ - Return a dictionary mapping field names to a dictionary of - additional keyword arguments. - """ - extra_kwargs = getattr(self.Meta, 'extra_kwargs', {}) - - read_only_fields = getattr(self.Meta, 'read_only_fields', None) - if read_only_fields is not None: - for field_name in read_only_fields: - kwargs = extra_kwargs.get(field_name, {}) - kwargs['read_only'] = True - extra_kwargs[field_name] = kwargs - - # These are all pending deprecation. - write_only_fields = getattr(self.Meta, 'write_only_fields', None) - if write_only_fields is not None: - warnings.warn( - "The `Meta.write_only_fields` option is pending deprecation. " - "Use `Meta.extra_kwargs={: {'write_only': True}}` instead.", - PendingDeprecationWarning, - stacklevel=3 - ) - for field_name in write_only_fields: - kwargs = extra_kwargs.get(field_name, {}) - kwargs['write_only'] = True - extra_kwargs[field_name] = kwargs - - view_name = getattr(self.Meta, 'view_name', None) - if view_name is not None: - warnings.warn( - "The `Meta.view_name` option is pending deprecation. " - "Use `Meta.extra_kwargs={'url': {'view_name': ...}}` instead.", - PendingDeprecationWarning, - stacklevel=3 - ) - kwargs = extra_kwargs.get(api_settings.URL_FIELD_NAME, {}) - kwargs['view_name'] = view_name - extra_kwargs[api_settings.URL_FIELD_NAME] = kwargs - - lookup_field = getattr(self.Meta, 'lookup_field', None) - if lookup_field is not None: - warnings.warn( - "The `Meta.lookup_field` option is pending deprecation. " - "Use `Meta.extra_kwargs={'url': {'lookup_field': ...}}` instead.", - PendingDeprecationWarning, - stacklevel=3 - ) - kwargs = extra_kwargs.get(api_settings.URL_FIELD_NAME, {}) - kwargs['lookup_field'] = lookup_field - extra_kwargs[api_settings.URL_FIELD_NAME] = kwargs - - return extra_kwargs - - def get_field_names(self, declared_fields, info): - """ - Returns the list of all field names that should be created when - instantiating this serializer class. This is based on the default - set of fields, but also takes into account the `Meta.fields` or - `Meta.exclude` options if they have been specified. - """ - fields = getattr(self.Meta, 'fields', None) - exclude = getattr(self.Meta, 'exclude', None) - - if fields and not isinstance(fields, (list, tuple)): - raise TypeError( - 'The `fields` option must be a list or tuple. Got %s.' % - type(fields).__name__ - ) - - if exclude and not isinstance(exclude, (list, tuple)): - raise TypeError( - 'The `exclude` option must be a list or tuple. Got %s.' % - type(exclude).__name__ - ) - - assert not (fields and exclude), ( - "Cannot set both 'fields' and 'exclude' options on " - "serializer {serializer_class}.".format( - serializer_class=self.__class__.__name__ - ) - ) - - if fields is not None: - # Ensure that all declared fields have also been included in the - # `Meta.fields` option. - for field_name in declared_fields: - assert field_name in fields, ( - "The field '{field_name}' was declared on serializer " - "{serializer_class}, but has not been included in the " - "'fields' option.".format( - field_name=field_name, - serializer_class=self.__class__.__name__ - ) - ) - return fields - - # Use the default set of field names if `Meta.fields` is not specified. - fields = self.get_default_field_names(declared_fields, info) - - if exclude is not None: - # If `Meta.exclude` is included, then remove those fields. - for field_name in exclude: - assert field_name in fields, ( - "The field '{field_name}' was include on serializer " - "{serializer_class} in the 'exclude' option, but does " - "not match any model field.".format( - field_name=field_name, - serializer_class=self.__class__.__name__ - ) - ) - fields.remove(field_name) - - return fields - - def get_default_field_names(self, declared_fields, model_info): - """ - Return the default list of field names that will be used if the - `Meta.fields` option is not specified. - """ - return ( - [model_info.pk.name] + - list(declared_fields.keys()) + - list(model_info.fields.keys()) + - list(model_info.forward_relations.keys()) - ) - - def _get_nested_class(self, nested_depth, relation_info): - class NestedSerializer(ModelSerializer): - class Meta: - model = relation_info.related - depth = nested_depth - 1 - - return NestedSerializer - class HyperlinkedModelSerializer(ModelSerializer): """ @@ -1246,7 +1302,7 @@ class HyperlinkedModelSerializer(ModelSerializer): * A 'url' field is included instead of the 'id' field. * Relationships to other instances are hyperlinks, instead of primary keys. """ - _related_class = HyperlinkedRelatedField + serializer_related_class = HyperlinkedRelatedField def get_default_field_names(self, declared_fields, model_info): """ @@ -1260,10 +1316,17 @@ class HyperlinkedModelSerializer(ModelSerializer): list(model_info.forward_relations.keys()) ) - def _get_nested_class(self, nested_depth, relation_info): + def build_nested_field(self, field_name, info, model, nested_depth): + """ + Create nested fields for forward and reverse relationships. + """ + relation_info = info.relations[field_name] + class NestedSerializer(HyperlinkedModelSerializer): class Meta: model = relation_info.related depth = nested_depth - 1 - return NestedSerializer + field_cls = NestedSerializer + kwargs = get_nested_relation_kwargs(relation_info) + return field_cls, kwargs From 62f78dfbf1b1dfa2d6406a4be5b83bc69267e851 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 15:50:29 +0000 Subject: [PATCH 28/29] Copy validators lists on instantiation. --- rest_framework/serializers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index a983d3fc7..8adbafe45 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -327,7 +327,9 @@ class Serializer(BaseSerializer): Returns a list of validator callables. """ # Used by the lazily-evaluated `validators` property. - return getattr(getattr(self, 'Meta', None), 'validators', []) + meta = getattr(self, 'Meta', None) + validators = getattr(meta, 'validators', None) + return validators[:] if validators else [] def get_initial(self): if hasattr(self, 'initial_data'): @@ -1213,7 +1215,7 @@ class ModelSerializer(Serializer): # If the validators have been declared explicitly then use that. validators = getattr(getattr(self, 'Meta', None), 'validators', None) if validators is not None: - return validators + return validators[:] # Otherwise use the default set of validators. return ( From c06a82d0531f4cb290baacee196829c770913eaa Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Dec 2014 16:02:27 +0000 Subject: [PATCH 29/29] Model serializer caching. --- rest_framework/serializers.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 8adbafe45..7b7170da2 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -682,6 +682,10 @@ def raise_errors_on_nested_writes(method_name, serializer, validated_data): ) +MODEL_SERIALIZER_FIELDS_CACHE = {} +MODEL_SERIALIZER_VALIDATORS_CACHE = {} + + class ModelSerializer(Serializer): """ A `ModelSerializer` is just a regular `Serializer`, except that: @@ -802,6 +806,11 @@ class ModelSerializer(Serializer): Return the dict of field names -> field instances that should be used for `self.fields` when instantiating the serializer. """ + cls = self.__class__ + + if cls in MODEL_SERIALIZER_FIELDS_CACHE: + return copy.deepcopy(MODEL_SERIALIZER_FIELDS_CACHE[cls]) + declared_fields = copy.deepcopy(self._declared_fields) model = getattr(self.Meta, 'model') depth = getattr(self.Meta, 'depth', 0) @@ -837,6 +846,8 @@ class ModelSerializer(Serializer): # Add in any hidden fields. ret.update(hidden_fields) + MODEL_SERIALIZER_FIELDS_CACHE[cls] = ret + return ret # Methods for determining the set of field names to include... @@ -1217,12 +1228,21 @@ class ModelSerializer(Serializer): if validators is not None: return validators[:] + cls = self.__class__ + + if cls in MODEL_SERIALIZER_VALIDATORS_CACHE: + return MODEL_SERIALIZER_VALIDATORS_CACHE[cls][:] + # Otherwise use the default set of validators. - return ( + validators = ( self.get_unique_together_validators() + self.get_unique_for_date_validators() ) + MODEL_SERIALIZER_VALIDATORS_CACHE[cls] = validators + + return validators + def get_unique_together_validators(self): """ Determine a default set of validators for any unique_together contraints.