Merge with a0fec27963ebb52d39cdad42d5a7fd428f989784

.hgignore

@@ -6,12 +6,16 @@ syntax: glob
 env
 docs/build
 html
+htmlcov
 examples/media/pygments/*
 examples/media/objectstore/*
 build/*
 dist/*
+djangorestframework.egg-info/*
 MANIFEST
 .project
 .pydevproject
 .settings
 .cache
+.coverage
+.tox

README

@@ -8,9 +8,14 @@ pip install -r requirements.txt # django
 
 # To run the tests...
 
-cd testproject
+export PYTHONPATH=.    # Ensure djangorestframework is on the PYTHONPATH
-export PYTHONPATH=..
+python djangorestframework/runtests/runtests.py
-python manage.py test djangorestframework
+
+
+# To run the test coverage report...
+
+export PYTHONPATH=.    # Ensure djangorestframework is on the PYTHONPATH
+python djangorestframework/runtests/runcoverage.py
 
 # To run the examples...
 
@@ -24,3 +29,8 @@ python manage.py runserver
 
 pip install -r docs/requirements.txt   # sphinx
 sphinx-build -c docs -b html -d docs/build docs html
+
+# To run the tests against the full set of supported configurations
+
+deactivate  # Ensure we are not currently running in a virtualenv
+tox

djangorestframework/content.py

@@ -27,29 +27,31 @@ class StandardContentMixin(ContentMixin):
 
 class OverloadedContentMixin(ContentMixin):
     """HTTP request content behaviour that also allows arbitrary content to be tunneled in form data."""
-    
+
-    """The name to use for the content override field in the POST form."""
+    """The name to use for the content override field in the POST form.
+    Set this to *None* to desactivate content overloading."""
     CONTENT_PARAM = '_content'
 
-    """The name to use for the content-type override field in the POST form."""
+    """The name to use for the content-type override field in the POST form.
+    Taken into account only if content overloading is activated."""
     CONTENTTYPE_PARAM = '_contenttype'
 
     def determine_content(self, request):
-        """If the request contains content return a tuple of (content_type, content) otherwise return None.
+        """If the request contains content, returns a tuple of (content_type, content) otherwise returns None.
         Note that content_type may be None if it is unset."""
         if not request.META.get('CONTENT_LENGTH', None) and not request.META.get('TRANSFER_ENCODING', None):
             return None
-        
         content_type = request.META.get('CONTENT_TYPE', None)
 
         if (request.method == 'POST' and self.CONTENT_PARAM and
             request.POST.get(self.CONTENT_PARAM, None) is not None):
 
-            # Set content type if form contains a none empty FORM_PARAM_CONTENTTYPE field
+            # Set content type if form contains a non-empty CONTENTTYPE_PARAM field
             content_type = None
             if self.CONTENTTYPE_PARAM and request.POST.get(self.CONTENTTYPE_PARAM, None):
                 content_type = request.POST.get(self.CONTENTTYPE_PARAM, None)
+                request.META['CONTENT_TYPE'] = content_type # TODO : VERY BAD, avoid modifying original request.
 
             return (content_type, request.POST[self.CONTENT_PARAM])
-
+        else:
-        return (content_type, request.raw_post_data)
+            return (content_type, request.raw_post_data)

djangorestframework/parsers.py

@@ -1,3 +1,7 @@
+from StringIO import StringIO
+
+from django.http.multipartparser import MultiPartParser as DjangoMPParser
+
 from djangorestframework.response import ResponseException
 from djangorestframework import status
 
@@ -6,6 +10,10 @@ try:
 except ImportError:
     import simplejson as json
 
+try:
+    from urlparse import parse_qs
+except ImportError:
+    from cgi import parse_qs
 
 class ParserMixin(object):
     parsers = ()
@@ -70,55 +78,90 @@ class JSONParser(BaseParser):
 class XMLParser(BaseParser):
     media_type = 'application/xml'
 
+class DataFlatener(object):
+    """Utility object for flatening dictionaries of lists. Useful for "urlencoded" decoded data."""
 
-class FormParser(BaseParser):
+    def flatten_data(self, data):
+        """Given a data dictionary {<key>: <value_list>}, returns a flattened dictionary
+        with information provided by the method "is_a_list"."""
+        flatdata = dict()
+        for key, val_list in data.items():
+            if self.is_a_list(key, val_list):
+                flatdata[key] = val_list
+            else:
+                if val_list:
+                    flatdata[key] = val_list[0]
+                else:
+                    # If the list is empty, but the parameter is not a list,
+                    # we strip this parameter.
+                    data.pop(key)
+        return flatdata
+
+    def is_a_list(self, key, val_list):
+        """Returns True if the parameter with name *key* is expected to be a list, or False otherwise.
+        *val_list* which is the received value for parameter *key* can be used to guess the answer."""
+        return False
+
+class FormParser(BaseParser, DataFlatener):
     """The default parser for form data.
     Return a dict containing a single value for each non-reserved parameter.
-    """
+
-    
+    In order to handle select multiple (and having possibly more than a single value for each parameter),
+    you can customize the output by subclassing the method 'is_a_list'."""
+
     media_type = 'application/x-www-form-urlencoded'
 
-    def parse(self, input):
+    """The value of the parameter when the select multiple is empty.
-        # The FormParser doesn't parse the input as other parsers would, since Django's already done the
+    Browsers are usually stripping the select multiple that have no option selected from the parameters sent.
-        # form parsing for us.  We build the content object from the request directly.
+    A common hack to avoid this is to send the parameter with a value specifying that the list is empty.
-        request = self.resource.request
+    This value will always be stripped before the data is returned."""
+    EMPTY_VALUE = '_empty'
 
-        if request.method == 'PUT':
+    def parse(self, input):
-            # Fix from piston to force Django to give PUT requests the same
+        data = parse_qs(input, keep_blank_values=True)
-            # form processing that POST requests get...
+
-            #
+        # removing EMPTY_VALUEs from the lists and flatening the data 
-            # Bug fix: if _load_post_and_files has already been called, for
+        for key, val_list in data.items():
-            # example by middleware accessing request.POST, the below code to
+            self.remove_empty_val(val_list)
-            # pretend the request is a POST instead of a PUT will be too late
+        data = self.flatten_data(data)
-            # to make a difference. Also calling _load_post_and_files will result 
-            # in the following exception:
-            #   AttributeError: You cannot set the upload handlers after the upload has been processed.
-            # The fix is to check for the presence of the _post field which is set 
-            # the first time _load_post_and_files is called (both by wsgi.py and 
-            # modpython.py). If it's set, the request has to be 'reset' to redo
-            # the query value parsing in POST mode.
-            if hasattr(request, '_post'):
-                del request._post
-                del request._files
-            
-            try:
-                request.method = "POST"
-                request._load_post_and_files()
-                request.method = "PUT"
-            except AttributeError:
-                request.META['REQUEST_METHOD'] = 'POST'
-                request._load_post_and_files()
-                request.META['REQUEST_METHOD'] = 'PUT'
 
         # Strip any parameters that we are treating as reserved
-        data = {}
+        for key in data.keys():
-        for (key, val) in request.POST.items():
+            if key in self.resource.RESERVED_FORM_PARAMS:
-            if key not in self.resource.RESERVED_FORM_PARAMS:
+                data.pop(key)
-                data[key] = val
-        
         return data
 
+    def remove_empty_val(self, val_list):
+        """ """
+        while(1): # Because there might be several times EMPTY_VALUE in the list
+            try: 
+                ind = val_list.index(self.EMPTY_VALUE)
+            except ValueError:
+                break
+            else:
+                val_list.pop(ind) 
+
 # TODO: Allow parsers to specify multiple media_types
-class MultipartParser(FormParser):
+class MultipartParser(BaseParser, DataFlatener):
     media_type = 'multipart/form-data'
 
+    def parse(self, input):
+
+        request = self.resource.request
+        #TODO : that's pretty dumb : files are loaded with
+        #upload_handlers, but as we read the request body completely (input),
+        #then it kind of misses the point. Why not input as a stream ?
+        upload_handlers = request._get_upload_handlers()
+        django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers)
+        data, files = django_mpp.parse()
+
+        # Flatening data, files and combining them
+        data = self.flatten_data(dict(data.iterlists()))
+        files = self.flatten_data(dict(files.iterlists()))
+        data.update(files)
+        
+        # Strip any parameters that we are treating as reserved
+        for key in data.keys():
+            if key in self.resource.RESERVED_FORM_PARAMS:
+                data.pop(key)
+        return data

djangorestframework/runtests/runcoverage.py

@@ -0,0 +1,53 @@
+"""
+Useful tool to run the test suite for djangorestframework and generate a coverage report.
+"""
+
+# http://ericholscher.com/blog/2009/jun/29/enable-setuppy-test-your-django-apps/
+# http://www.travisswicegood.com/2010/01/17/django-virtualenv-pip-and-fabric/
+# http://code.djangoproject.com/svn/django/trunk/tests/runtests.py
+import os
+import sys
+os.environ['DJANGO_SETTINGS_MODULE'] = 'djangorestframework.runtests.settings'
+
+from django.conf import settings
+from django.test.utils import get_runner
+from coverage import coverage
+from itertools import chain
+import djangorestframework
+
+def main():
+    """Run the tests for djangorestframework and generate a coverage report."""
+    
+    # Discover the list of all modules that we should test coverage for
+    project_dir = os.path.dirname(djangorestframework.__file__)
+    cov_files = []
+    for (path, dirs, files) in os.walk(project_dir):
+        # Drop tests and runtests directories from the test coverage report
+        if os.path.basename(path) == 'tests' or os.path.basename(path) == 'runtests':
+            continue
+        cov_files.extend([os.path.join(path, file) for file in files if file.endswith('.py')])
+
+    cov = coverage()
+    cov.erase()
+    cov.start()
+    TestRunner = get_runner(settings)
+
+    if hasattr(TestRunner, 'func_name'):
+        # Pre 1.2 test runners were just functions,
+        # and did not support the 'failfast' option.
+        import warnings
+        warnings.warn(
+            'Function-based test runners are deprecated. Test runners should be classes with a run_tests() method.',
+            DeprecationWarning
+        )
+        failures = TestRunner(['djangorestframework'])
+    else:
+        test_runner = TestRunner()
+        failures = test_runner.run_tests(['djangorestframework'])
+
+    cov.stop()
+    cov.report(cov_files)
+    sys.exit(failures)
+
+if __name__ == '__main__':
+    main()

djangorestframework/runtests/runtests.py

@@ -0,0 +1,35 @@
+'''
+Created on Mar 10, 2011
+
+@author: tomchristie
+'''
+# http://ericholscher.com/blog/2009/jun/29/enable-setuppy-test-your-django-apps/
+# http://www.travisswicegood.com/2010/01/17/django-virtualenv-pip-and-fabric/
+# http://code.djangoproject.com/svn/django/trunk/tests/runtests.py
+import os
+import sys
+os.environ['DJANGO_SETTINGS_MODULE'] = 'djangorestframework.runtests.settings'
+
+from django.conf import settings
+from django.test.utils import get_runner
+
+def main():
+    TestRunner = get_runner(settings)
+
+    if hasattr(TestRunner, 'func_name'):
+        # Pre 1.2 test runners were just functions,
+        # and did not support the 'failfast' option.
+        import warnings
+        warnings.warn(
+            'Function-based test runners are deprecated. Test runners should be classes with a run_tests() method.',
+            DeprecationWarning
+        )
+        failures = TestRunner(['djangorestframework'])
+    else:
+        test_runner = TestRunner()
+        failures = test_runner.run_tests(['djangorestframework'])
+
+    sys.exit(failures)
+
+if __name__ == '__main__':
+    main()

djangorestframework/tests/__init__.py

@@ -5,5 +5,10 @@ modules = [filename.rsplit('.', 1)[0]
            for filename in os.listdir(os.path.dirname(__file__))
            if filename.endswith('.py') and not filename.startswith('_')]
 
+__test__ = dict()
+
 for module in modules:
-    exec("from djangorestframework.tests.%s import *" % module)
+    exec("from djangorestframework.tests.%s import __doc__ as module_doc" % module)
+    exec("from djangorestframework.tests.%s import *" % module)
+    __test__[module] = module_doc or ""
+

djangorestframework/tests/content.py

@@ -111,6 +111,7 @@ class TestContentMixins(TestCase):
                      OverloadedContentMixin.CONTENTTYPE_PARAM: content_type}
         request = self.req.post('/', form_data)
         self.assertEqual(OverloadedContentMixin().determine_content(request), (content_type, content))
+        self.assertEqual(request.META['CONTENT_TYPE'], content_type)
  
     def test_overloaded_behaviour_allows_content_tunnelling_content_type_not_set(self):
         """Ensure determine_content(request) returns (None, content) for overloaded POST request with content type not set"""

djangorestframework/tests/parsers.py

@@ -0,0 +1,127 @@
+"""
+..
+    >>> from djangorestframework.parsers import FormParser
+    >>> from djangorestframework.resource import Resource
+    >>> from djangorestframework.compat import RequestFactory
+    >>> from urllib import urlencode
+    >>> req = RequestFactory().get('/')
+    >>> some_resource = Resource()
+    >>> trash = some_resource.dispatch(req)# Some variables are set only when calling dispatch
+
+FormParser
+============
+
+Data flatening
+----------------
+
+Here is some example data, which would eventually be sent along with a post request :
+
+    >>> inpt = urlencode([
+    ...     ('key1', 'bla1'),
+    ...     ('key2', 'blo1'), ('key2', 'blo2'),
+    ... ])
+
+Default behaviour for :class:`parsers.FormParser`, is to return a single value for each parameter :
+
+    >>> FormParser(some_resource).parse(inpt) == {'key1': 'bla1', 'key2': 'blo1'}
+    True
+
+However, you can customize this behaviour by subclassing :class:`parsers.FormParser`, and overriding :meth:`parsers.FormParser.is_a_list` :
+
+    >>> class MyFormParser(FormParser):
+    ... 
+    ...     def is_a_list(self, key, val_list):
+    ...         return len(val_list) > 1
+
+This new parser only flattens the lists of parameters that contain a single value.
+
+    >>> MyFormParser(some_resource).parse(inpt) == {'key1': 'bla1', 'key2': ['blo1', 'blo2']}
+    True
+
+.. note:: The same functionality is available for :class:`parsers.MultipartParser`.
+
+Submitting an empty list
+--------------------------
+
+When submitting an empty select multiple, like this one ::
+
+    <select multiple="multiple" name="key2"></select>
+
+The browsers usually strip the parameter completely. A hack to avoid this, and therefore being able to submit an empty select multiple, is to submit a value that tells the server that the list is empty ::
+
+    <select multiple="multiple" name="key2"><option value="_empty"></select>
+
+:class:`parsers.FormParser` provides the server-side implementation for this hack. Considering the following posted data :
+
+    >>> inpt = urlencode([
+    ...     ('key1', 'blo1'), ('key1', '_empty'),
+    ...     ('key2', '_empty'),
+    ... ])
+
+:class:`parsers.FormParser` strips the values ``_empty`` from all the lists.
+
+    >>> MyFormParser(some_resource).parse(inpt) == {'key1': 'blo1'}
+    True
+
+Oh ... but wait a second, the parameter ``key2`` isn't even supposed to be a list, so the parser just stripped it.
+
+    >>> class MyFormParser(FormParser):
+    ... 
+    ...     def is_a_list(self, key, val_list):
+    ...         return key == 'key2'
+    ... 
+    >>> MyFormParser(some_resource).parse(inpt) == {'key1': 'blo1', 'key2': []}
+    True
+
+Better like that. Note that you can configure something else than ``_empty`` for the empty value by setting :attr:`parsers.FormParser.EMPTY_VALUE`.
+"""
+import httplib, mimetypes
+from tempfile import TemporaryFile
+from django.test import TestCase
+from djangorestframework.compat import RequestFactory
+from djangorestframework.parsers import MultipartParser
+from djangorestframework.resource import Resource
+
+def encode_multipart_formdata(fields, files):
+    """For testing multipart parser.
+    fields is a sequence of (name, value) elements for regular form fields.
+    files is a sequence of (name, filename, value) elements for data to be uploaded as files
+    Return (content_type, body)."""
+    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
+    CRLF = '\r\n'
+    L = []
+    for (key, value) in fields:
+        L.append('--' + BOUNDARY)
+        L.append('Content-Disposition: form-data; name="%s"' % key)
+        L.append('')
+        L.append(value)
+    for (key, filename, value) in files:
+        L.append('--' + BOUNDARY)
+        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
+        L.append('Content-Type: %s' % get_content_type(filename))
+        L.append('')
+        L.append(value)
+    L.append('--' + BOUNDARY + '--')
+    L.append('')
+    body = CRLF.join(L)
+    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
+    return content_type, body
+
+def get_content_type(filename):
+    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+
+class TestMultipartParser(TestCase):
+    def setUp(self):
+        self.req = RequestFactory()
+        self.content_type, self.body = encode_multipart_formdata([('key1', 'val1'), ('key1', 'val2')],
+        [('file1', 'pic.jpg', 'blablabla'), ('file1', 't.txt', 'blobloblo')])
+
+    def test_multipartparser(self):
+        """Ensure that MultipartParser can parse multipart/form-data that contains a mix of several files and parameters."""
+        post_req = RequestFactory().post('/', self.body, content_type=self.content_type)
+        some_resource = Resource()
+        some_resource.dispatch(post_req)
+        parsed = MultipartParser(some_resource).parse(self.body)
+        self.assertEqual(parsed['key1'], 'val1')
+        self.assertEqual(parsed['file1'].read(), 'blablabla')
+

djangorestframework/tests/runtests.py

@@ -1,29 +0,0 @@
-'''
-Created on Mar 10, 2011
-
-@author: tomchristie
-'''
-# http://ericholscher.com/blog/2009/jun/29/enable-setuppy-test-your-django-apps/
-# http://www.travisswicegood.com/2010/01/17/django-virtualenv-pip-and-fabric/
-from django.conf import settings
-from django.core.management import call_command
-
-def runtests():
-    settings.configure(
-        INSTALLED_APPS=(
-            'django.contrib.auth',
-            'django.contrib.contenttypes',
-            'django.contrib.sessions',
-            'django.contrib.sites',
-            'django.contrib.messages',
-            'djangorestframework',
-        ),
-        ROOT_URLCONF='djangorestframework.tests.urls',
-        # Django replaces this, but it still wants it. *shrugs*
-        DATABASE_ENGINE='sqlite3'
-    )
-    call_command('test', 'djangorestframework')
-
-
-if __name__ == '__main__':
-    runtests()

djangorestframework/utils.py

@@ -34,44 +34,6 @@ def url_resolves(url):
         return False
     return True
 
-# From piston
-def coerce_put_post(request):
-    """
-    Django doesn't particularly understand REST.
-    In case we send data over PUT, Django won't
-    actually look at the data and load it. We need
-    to twist its arm here.
-    
-    The try/except abominiation here is due to a bug
-    in mod_python. This should fix it.
-    """
-    if request.method != 'PUT':
-        return
-
-    # Bug fix: if _load_post_and_files has already been called, for
-    # example by middleware accessing request.POST, the below code to
-    # pretend the request is a POST instead of a PUT will be too late
-    # to make a difference. Also calling _load_post_and_files will result 
-    # in the following exception:
-    #   AttributeError: You cannot set the upload handlers after the upload has been processed.
-    # The fix is to check for the presence of the _post field which is set 
-    # the first time _load_post_and_files is called (both by wsgi.py and 
-    # modpython.py). If it's set, the request has to be 'reset' to redo
-    # the query value parsing in POST mode.
-    if hasattr(request, '_post'):
-        del request._post
-        del request._files
-    
-    try:
-        request.method = "POST"
-        request._load_post_and_files()
-        request.method = "PUT"
-    except AttributeError:
-        request.META['REQUEST_METHOD'] = 'POST'
-        request._load_post_and_files()
-        request.META['REQUEST_METHOD'] = 'PUT'
-        
-    request.PUT = request.POST
 
 # From http://www.koders.com/python/fidB6E125C586A6F49EAC38992CF3AFDAAE35651975.aspx?s=mdef:xml
 #class object_dict(dict):

setup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env/python
 # -*- coding: utf-8 -*-
 
-from distutils.core import setup
+from setuptools import setup
 
 setup(
     name = "djangorestframework",
@@ -14,9 +14,11 @@ setup(
     author_email = 'tom@tomchristie.com',
     packages = ['djangorestframework',
                 'djangorestframework.templatetags',
-                'djangorestframework.tests'],
+                'djangorestframework.tests',
+                'djangorestframework.runtests'],
     package_dir={'djangorestframework': 'djangorestframework'},
     package_data = {'djangorestframework': ['templates/*', 'static/*']},
+    test_suite = 'djangorestframework.runtests.runtests.main',
     classifiers = [
         'Development Status :: 4 - Beta',
         'Environment :: Web Environment',
@@ -29,3 +31,7 @@ setup(
     ]
 )
 
+import os, shutil
+shutil.rmtree(os.path.join(os.path.dirname(__file__), 'djangorestframework.egg-info'), True)
+
+

testproject/manage.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-from django.core.management import execute_manager
-try:
-    import settings # Assumed to be in the same directory.
-except ImportError:
-    import sys
-    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
-    sys.exit(1)
-
-if __name__ == "__main__":
-    execute_manager(settings)

testproject/urls.py

@@ -1,16 +0,0 @@
-from django.conf.urls.defaults import *
-
-# Uncomment the next two lines to enable the admin:
-# from django.contrib import admin
-# admin.autodiscover()
-
-urlpatterns = patterns('',
-    # Example:
-    # (r'^testproject/', include('testproject.foo.urls')),
-
-    # Uncomment the admin/doc line below to enable admin documentation:
-    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
-
-    # Uncomment the next line to enable the admin:
-    # (r'^admin/', include(admin.site.urls)),
-)

tox.ini

@@ -0,0 +1,40 @@
+[tox]
+envlist=
+    py25-django12, py26-django12, py27-django12, py25-django13, py26-django13, py27-django13
+
+[testenv]
+commands=
+    python setup.py test
+
+[testenv:py25-django12]
+basepython=python2.5
+deps=
+    django==1.2.4
+    simplejson
+
+[testenv:py26-django12]
+basepython=python2.6
+deps=
+    django==1.2.4
+
+[testenv:py27-django12]
+basepython=python2.7
+deps=
+    django==1.2.4
+
+[testenv:py25-django13]
+basepython=python2.5
+deps=
+    http://www.djangoproject.com/download/1.3-rc-1/tarball/
+    simplejson
+
+[testenv:py26-django13]
+basepython=python2.6
+deps=
+    http://www.djangoproject.com/download/1.3-rc-1/tarball/
+
+[testenv:py27-django13]
+basepython=python2.7
+deps=
+    http://www.djangoproject.com/download/1.3-rc-1/tarball/
+