From 3a2ad8e68c669eebcc9847e6c2a664e323e8da1d Mon Sep 17 00:00:00 2001 From: imdark Date: Wed, 17 May 2017 11:49:30 -0700 Subject: [PATCH 1/3] in order to solve the memory leak at #5146 Large encoded string take a very long time to to release from memory, but if we just pass the stream directly into json.load we get much better memory performance. --- rest_framework/parsers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 238382364..1a4c24387 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -22,6 +22,7 @@ from django.utils.six.moves.urllib import parse as urlparse from rest_framework import renderers from rest_framework.exceptions import ParseError +import codecs class DataAndFiles(object): @@ -61,8 +62,8 @@ class JSONParser(BaseParser): encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) try: - data = stream.read().decode(encoding) - return json.loads(data) + decoded_stream = codecs.decode(stream, encoding) + return json.load(decoded_stream) except ValueError as exc: raise ParseError('JSON parse error - %s' % six.text_type(exc)) From 9a2281167178999ad0472d3c9862af91c077322c Mon Sep 17 00:00:00 2001 From: imdark Date: Wed, 24 May 2017 17:56:49 -0700 Subject: [PATCH 2/3] modified to use a reader modified to use a reader since direct decoding is not supported --- rest_framework/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 1a4c24387..817efd2da 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -7,6 +7,7 @@ on the request, such as form content or json encoded data. from __future__ import unicode_literals import json +import codecs from django.conf import settings from django.core.files.uploadhandler import StopFutureHandlers @@ -22,7 +23,6 @@ from django.utils.six.moves.urllib import parse as urlparse from rest_framework import renderers from rest_framework.exceptions import ParseError -import codecs class DataAndFiles(object): @@ -62,7 +62,7 @@ class JSONParser(BaseParser): encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) try: - decoded_stream = codecs.decode(stream, encoding) + decoded_stream = codecs.getreader(encoding)(stream) return json.load(decoded_stream) except ValueError as exc: raise ParseError('JSON parse error - %s' % six.text_type(exc)) From cdeab1c490fa1073ce985b227e85bbdc56cb08c2 Mon Sep 17 00:00:00 2001 From: imdark Date: Wed, 24 May 2017 18:12:38 -0700 Subject: [PATCH 3/3] fixed to pass isort linting --- rest_framework/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 817efd2da..0e40e1a7a 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -6,8 +6,8 @@ on the request, such as form content or json encoded data. """ from __future__ import unicode_literals -import json import codecs +import json from django.conf import settings from django.core.files.uploadhandler import StopFutureHandlers