From 0c85768435e67133ff219aaddb4ea3bf122bd360 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Fri, 3 May 2013 01:37:25 +0600 Subject: [PATCH 1/5] Added FileUploadParser refs #7 --- rest_framework/parsers.py | 63 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 491acd68c..6ba05aef8 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -6,9 +6,10 @@ on the request, such as form content or json encoded data. """ from __future__ import unicode_literals from django.conf import settings +from django.core.files.uploadhandler import StopFutureHandlers from django.http import QueryDict from django.http.multipartparser import MultiPartParser as DjangoMultiPartParser -from django.http.multipartparser import MultiPartParserError +from django.http.multipartparser import MultiPartParserError, parse_header, ChunkIter from rest_framework.compat import yaml, etree from rest_framework.exceptions import ParseError from rest_framework.compat import six @@ -205,3 +206,63 @@ class XMLParser(BaseParser): pass return value + + +class FileUploadParser(BaseParser): + """ + Parser for file upload data. + """ + media_type = '*/*' + + def parse(self, stream, media_type=None, parser_context=None): + parser_context = parser_context or {} + request = parser_context['request'] + encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) + meta = request.META + + try: + disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION']) + filename = disposition[1]['filename'] + except KeyError: + filename = None + + content_type = meta.get('HTTP_CONTENT_TYPE', meta.get('CONTENT_TYPE', '')) + try: + content_length = int(meta.get('HTTP_CONTENT_LENGTH', meta.get('CONTENT_LENGTH', 0))) + except (ValueError, TypeError): + content_length = None + + # See if the handler will want to take care of the parsing. + for handler in request.upload_handlers: + result = handler.handle_raw_input(None, + meta, + content_length, + None, + encoding) + if result is not None: + return DataAndFiles(result[0], {'file': result[1]}) + + possible_sizes = [x.chunk_size for x in request.upload_handlers if x.chunk_size] + chunk_size = min([2**31-4] + possible_sizes) + chunks = ChunkIter(stream, chunk_size) + counters = [0] * len(request.upload_handlers) + + for handler in request.upload_handlers: + try: + handler.new_file(None, filename, content_type, content_length, encoding) + except StopFutureHandlers: + break + + for chunk in chunks: + for i, handler in enumerate(request.upload_handlers): + chunk_length = len(chunk) + chunk = handler.receive_data_chunk(chunk, counters[i]) + counters[i] += chunk_length + if chunk is None: + # If the chunk received by the handler is None, then don't continue. + break + + for i, handler in enumerate(request.upload_handlers): + file_obj = handler.file_complete(counters[i]) + if file_obj: + return DataAndFiles(None, {'file': file_obj}) From 318fdaabe560c99de4983e0a3cdcb79756baaf01 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Fri, 3 May 2013 01:39:08 +0600 Subject: [PATCH 2/5] Tests for FileUploadParser --- rest_framework/tests/parsers.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/rest_framework/tests/parsers.py b/rest_framework/tests/parsers.py index 539c5b44f..b18ecbf24 100644 --- a/rest_framework/tests/parsers.py +++ b/rest_framework/tests/parsers.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals from rest_framework.compat import StringIO from django import forms +from django.core.files.uploadhandler import MemoryFileUploadHandler from django.test import TestCase from django.utils import unittest from rest_framework.compat import etree -from rest_framework.parsers import FormParser +from rest_framework.parsers import FormParser, FileUploadParser from rest_framework.parsers import XMLParser import datetime @@ -82,3 +83,27 @@ class TestXMLParser(TestCase): parser = XMLParser() data = parser.parse(self._complex_data_input) self.assertEqual(data, self._complex_data) + + +class TestFileUploadParser(TestCase): + def setUp(self): + class MockRequest(object): + pass + from io import BytesIO + self.stream = BytesIO( + "Test text file".encode('utf-8') + ) + request = MockRequest() + request.upload_handlers = (MemoryFileUploadHandler(),) + request.META = { + 'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'), + 'HTTP_CONTENT_LENGTH': 14, + } + self.parser_context = {'request': request} + + def test_parse(self): + """ Make sure the `QueryDict` works OK """ + parser = FileUploadParser() + data_and_files = parser.parse(self.stream, parser_context=self.parser_context) + file_obj = data_and_files.files['file'] + self.assertEqual(file_obj._size, 14) From e36e4f48ad481b4303e68ed524677add07b224f7 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Sat, 4 May 2013 14:58:21 +0600 Subject: [PATCH 3/5] Codebase improvements on FileUploadParser * Added docstrings. * Added `FileUploadParser.get_filename` to make it easier to override. * Added url kwargs filename detection step. * Updated tests corresponding to these changes. --- rest_framework/parsers.py | 45 +++++++++++++++++++++++---------- rest_framework/tests/parsers.py | 10 ++++++-- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 6ba05aef8..7eb92184d 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -215,16 +215,19 @@ class FileUploadParser(BaseParser): media_type = '*/*' def parse(self, stream, media_type=None, parser_context=None): + """ + Returns a DataAndFiles object. + + `.data` will be None (we expect request body to be a file content). + `.files` will be a `QueryDict` containing one 'file' elemnt - a parsed file. + """ + parser_context = parser_context or {} request = parser_context['request'] encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) meta = request.META - - try: - disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION']) - filename = disposition[1]['filename'] - except KeyError: - filename = None + upload_handlers = request.upload_handlers + filename = self.get_filename(stream, media_type, parser_context) content_type = meta.get('HTTP_CONTENT_TYPE', meta.get('CONTENT_TYPE', '')) try: @@ -233,28 +236,28 @@ class FileUploadParser(BaseParser): content_length = None # See if the handler will want to take care of the parsing. - for handler in request.upload_handlers: + for handler in upload_handlers: result = handler.handle_raw_input(None, meta, content_length, None, encoding) if result is not None: - return DataAndFiles(result[0], {'file': result[1]}) + return DataAndFiles(None, {'file': result[1]}) - possible_sizes = [x.chunk_size for x in request.upload_handlers if x.chunk_size] + possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size] chunk_size = min([2**31-4] + possible_sizes) chunks = ChunkIter(stream, chunk_size) - counters = [0] * len(request.upload_handlers) + counters = [0] * len(upload_handlers) - for handler in request.upload_handlers: + for handler in upload_handlers: try: handler.new_file(None, filename, content_type, content_length, encoding) except StopFutureHandlers: break for chunk in chunks: - for i, handler in enumerate(request.upload_handlers): + for i, handler in enumerate(upload_handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length @@ -262,7 +265,23 @@ class FileUploadParser(BaseParser): # If the chunk received by the handler is None, then don't continue. break - for i, handler in enumerate(request.upload_handlers): + for i, handler in enumerate(upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: return DataAndFiles(None, {'file': file_obj}) + + def get_filename(self, stream, media_type, parser_context): + """ + Detects the uploaded file name. First searches a 'filename' url kwarg. + Then tries to parse Content-Disposition header. + """ + try: + return parser_context['kwargs']['filename'] + except KeyError: + pass + try: + meta = parser_context['request'].META + disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION']) + return disposition[1]['filename'] + except (AttributeError, KeyError): + pass diff --git a/rest_framework/tests/parsers.py b/rest_framework/tests/parsers.py index b18ecbf24..7699e10c9 100644 --- a/rest_framework/tests/parsers.py +++ b/rest_framework/tests/parsers.py @@ -99,11 +99,17 @@ class TestFileUploadParser(TestCase): 'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'), 'HTTP_CONTENT_LENGTH': 14, } - self.parser_context = {'request': request} + self.parser_context = {'request': request, 'kwargs': {}} def test_parse(self): """ Make sure the `QueryDict` works OK """ parser = FileUploadParser() - data_and_files = parser.parse(self.stream, parser_context=self.parser_context) + self.stream.seek(0) + data_and_files = parser.parse(self.stream, None, self.parser_context) file_obj = data_and_files.files['file'] self.assertEqual(file_obj._size, 14) + + def test_get_filename(self): + parser = FileUploadParser() + filename = parser.get_filename(self.stream, None, self.parser_context) + self.assertEqual(filename, 'file.txt'.encode('utf-8')) From a514232815a82ad8a4dc1819afa0d62f9bab1323 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Sat, 4 May 2013 17:18:10 +0600 Subject: [PATCH 4/5] Raise ParseError if can't handle the uploaded file --- rest_framework/parsers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 7eb92184d..27a0db65e 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -269,6 +269,7 @@ class FileUploadParser(BaseParser): file_obj = handler.file_complete(counters[i]) if file_obj: return DataAndFiles(None, {'file': file_obj}) + raise ParseError("FileUpload parse error - none of upload handlers can handle the stream") def get_filename(self, stream, media_type, parser_context): """ From 5faaba9c691851ec68e385cc87d6bce82e4d4853 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Sat, 4 May 2013 18:04:48 +0600 Subject: [PATCH 5/5] Docs for FileUploadParser --- docs/api-guide/parsers.md | 51 +++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/docs/api-guide/parsers.md b/docs/api-guide/parsers.md index a28304922..370f406d3 100644 --- a/docs/api-guide/parsers.md +++ b/docs/api-guide/parsers.md @@ -101,6 +101,28 @@ You will typically want to use both `FormParser` and `MultiPartParser` together **.media_type**: `multipart/form-data` +## FileUploadParser + +Parses raw file upload content. Returns a `DataAndFiles` object. Since we expect the whole request body to be a file content `request.DATA` will be None, and `request.FILES` will contain the only one key `'file'` matching the uploaded file. + +The `filename` property of uploaded file would be set to the result of `.get_filename()` method. By default it tries first to take it's value from the `filename` URL kwarg, and then from `Content-Disposition` HTTP header. You can implement other behaviour be overriding this method. + +Note that since this parser's `media_type` matches every HTTP request it imposes restrictions on usage in combination with other parsers for the same API view. + +Basic usage expamle: + + class FileUploadView(views.APIView): + parser_classes = (FileUploadParser,) + + def put(self, request, filename, format=None): + file_obj = request.FILES['file'] + # ... + # do some staff with uploaded file + # ... + return Response(status=204) + +**.media_type**: `*/*` + --- # Custom parsers @@ -144,35 +166,6 @@ The following is an example plaintext parser that will populate the `request.DAT """ return stream.read() -## Uploading file content - -If your custom parser needs to support file uploads, you may return a `DataAndFiles` object from the `.parse()` method. `DataAndFiles` should be instantiated with two arguments. The first argument will be used to populate the `request.DATA` property, and the second argument will be used to populate the `request.FILES` property. - -For example: - - class SimpleFileUploadParser(BaseParser): - """ - A naive raw file upload parser. - """ - media_type = '*/*' # Accept anything - - def parse(self, stream, media_type=None, parser_context=None): - content = stream.read() - name = 'example.dat' - content_type = 'application/octet-stream' - size = len(content) - charset = 'utf-8' - - # Write a temporary file based on the request content - temp = tempfile.NamedTemporaryFile(delete=False) - temp.write(content) - uploaded = UploadedFile(temp, name, content_type, size, charset) - - # Return the uploaded file - data = {} - files = {name: uploaded} - return DataAndFiles(data, files) - --- # Third party packages