From 15351afa3a742d9e8cb4db36357941dc0abba27a Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Tue, 21 Feb 2017 18:18:04 -0500 Subject: [PATCH 1/7] Fix json violation according to RFC7159 for infinity/NaN --- rest_framework/utils/encoders.py | 72 +++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index 8896e4f2c..d1dabd368 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -6,6 +6,9 @@ from __future__ import unicode_literals import datetime import decimal import json +from json.encoder import (FLOAT_REPR, INFINITY, c_make_encoder, + encode_basestring_ascii, encode_basestring, + _make_iterencode) import uuid from django.db.models.query import QuerySet @@ -21,6 +24,72 @@ class JSONEncoder(json.JSONEncoder): JSONEncoder subclass that knows how to encode date/time/timedelta, decimal types, generators and other basic python objects. """ + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + For example:: + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if six.PY2: + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, + _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, + _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + + if o != o: + text = 'null' + elif o == _inf: + text = 'null' + elif o == _neginf: + text = 'null' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + if (_one_shot and c_make_encoder is not None + and self.indent is None): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot) + return _iterencode(o, 0) + def default(self, obj): # For Date Time string spec, see ECMA 262 # http://ecma-international.org/ecma-262/5.1/#sec-15.9.1.15 @@ -55,7 +124,8 @@ class JSONEncoder(json.JSONEncoder): elif hasattr(obj, 'tolist'): # Numpy arrays and array scalars. return obj.tolist() - elif (coreapi is not None) and isinstance(obj, (coreapi.Document, coreapi.Error)): + elif (coreapi is not None) and isinstance(obj, (coreapi.Document, + coreapi.Error)): raise RuntimeError( 'Cannot return a coreapi object from a JSON view. ' 'You should be using a schema renderer instead for this view.' From da63f5e82d44d81ef3c93510208e98b4696f4260 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Tue, 21 Feb 2017 18:29:16 -0500 Subject: [PATCH 2/7] Adding flake8 corrections --- rest_framework/utils/encoders.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index d1dabd368..4ef3a55a9 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -6,10 +6,10 @@ from __future__ import unicode_literals import datetime import decimal import json -from json.encoder import (FLOAT_REPR, INFINITY, c_make_encoder, - encode_basestring_ascii, encode_basestring, - _make_iterencode) import uuid +from json.encoder import (FLOAT_REPR, INFINITY, _make_iterencode, + c_make_encoder, encode_basestring, + encode_basestring_ascii) from django.db.models.query import QuerySet from django.utils import six, timezone @@ -55,7 +55,7 @@ class JSONEncoder(json.JSONEncoder): return _orig_encoder(o) def floatstr(o, allow_nan=self.allow_nan, - _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): + _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): # Check for specials. Note that this type of test is processor # and/or platform-specific, so do tests which don't depend on the # internals. @@ -76,7 +76,6 @@ class JSONEncoder(json.JSONEncoder): return text - if (_one_shot and c_make_encoder is not None and self.indent is None): _iterencode = c_make_encoder( @@ -124,7 +123,7 @@ class JSONEncoder(json.JSONEncoder): elif hasattr(obj, 'tolist'): # Numpy arrays and array scalars. return obj.tolist() - elif (coreapi is not None) and isinstance(obj, (coreapi.Document, + elif (coreapi is not None) and isinstance(obj, (coreapi.Document, coreapi.Error)): raise RuntimeError( 'Cannot return a coreapi object from a JSON view. ' From fb0f9150b3713c013d91e9ec3851f63683ccac98 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Tue, 21 Feb 2017 19:25:28 -0500 Subject: [PATCH 3/7] Forgot to remove _one_shot C path --- rest_framework/utils/encoders.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index 4ef3a55a9..18532ec81 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -8,8 +8,7 @@ import decimal import json import uuid from json.encoder import (FLOAT_REPR, INFINITY, _make_iterencode, - c_make_encoder, encode_basestring, - encode_basestring_ascii) + encode_basestring, encode_basestring_ascii) from django.db.models.query import QuerySet from django.utils import six, timezone @@ -76,18 +75,10 @@ class JSONEncoder(json.JSONEncoder): return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot) - return _iterencode(o, 0) + return _make_iterencode(markers, self.default, _encoder, self.indent, + floatstr, self.key_separator, + self.item_separator, self.sort_keys, + self.skipkeys, _one_shot)(o, 0) def default(self, obj): # For Date Time string spec, see ECMA 262 From 8383e4021674b5c81e25db5602af9f0327d8c762 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Tue, 21 Feb 2017 19:38:12 -0500 Subject: [PATCH 4/7] Fix for Python 3 --- rest_framework/utils/encoders.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index 18532ec81..17cbd3cd5 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -7,7 +7,7 @@ import datetime import decimal import json import uuid -from json.encoder import (FLOAT_REPR, INFINITY, _make_iterencode, +from json.encoder import (INFINITY, _make_iterencode, encode_basestring, encode_basestring_ascii) from django.db.models.query import QuerySet @@ -17,6 +17,11 @@ from django.utils.functional import Promise from rest_framework.compat import coreapi, total_seconds +try: + from json.encoder import FLOAT_REPR +except: + FLOAT_REPR = float.__repr__ + class JSONEncoder(json.JSONEncoder): """ @@ -47,12 +52,6 @@ class JSONEncoder(json.JSONEncoder): o = o.decode(_encoding) return _orig_encoder(o) - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): # Check for specials. Note that this type of test is processor From 3f5c1425bcfd22fa20cc3cf82100b881b170b3e7 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Wed, 22 Feb 2017 10:13:44 -0500 Subject: [PATCH 5/7] Return strings instead of null --- rest_framework/utils/encoders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index 17cbd3cd5..b48b8ec0f 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -59,11 +59,11 @@ class JSONEncoder(json.JSONEncoder): # internals. if o != o: - text = 'null' + text = '"NaN"' elif o == _inf: - text = 'null' + text = '"Infinity"' elif o == _neginf: - text = 'null' + text = '"-Infinity"' else: return _repr(o) From 5777d25030f366559c92014791c6206c70841fc7 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Wed, 22 Feb 2017 10:31:10 -0500 Subject: [PATCH 6/7] Add unit test for JSONEncoder for floats --- tests/test_encoders.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_encoders.py b/tests/test_encoders.py index 8f8694c47..31b8391d6 100644 --- a/tests/test_encoders.py +++ b/tests/test_encoders.py @@ -1,3 +1,4 @@ +import json from datetime import date, datetime, timedelta from decimal import Decimal from uuid import uuid4 @@ -92,3 +93,11 @@ class JSONEncoderTests(TestCase): """ foo = MockList() assert self.encoder.default(foo) == [1, 2, 3] + + def test_encode_float(self): + """ + Tests encoding floats with special values + """ + + f = [3.141592653, float('inf'), float('-inf'), float('nan')] + assert json.dumps(f, cls=JSONEncoder) == '[3.141592653, "Infinity", "-Infinity", "NaN"]' From 80cf8f4d32ac3504e7c844554f3524951e511741 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Wed, 22 Feb 2017 11:28:34 -0500 Subject: [PATCH 7/7] Increase coverage -Use encoder.encode instead of json.dumps. encoder.default can't be used to test floats --- tests/test_encoders.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_encoders.py b/tests/test_encoders.py index 31b8391d6..61e049ae1 100644 --- a/tests/test_encoders.py +++ b/tests/test_encoders.py @@ -1,10 +1,12 @@ -import json +# -*- coding: utf-8 -*- + from datetime import date, datetime, timedelta from decimal import Decimal from uuid import uuid4 import pytest from django.test import TestCase +from django.utils import six from django.utils.timezone import utc from rest_framework.compat import coreapi @@ -100,4 +102,21 @@ class JSONEncoderTests(TestCase): """ f = [3.141592653, float('inf'), float('-inf'), float('nan')] - assert json.dumps(f, cls=JSONEncoder) == '[3.141592653, "Infinity", "-Infinity", "NaN"]' + assert self.encoder.encode(f) == '[3.141592653, "Infinity", "-Infinity", "NaN"]' + + encoder = JSONEncoder(allow_nan=False) + try: + encoder.encode(f) + except ValueError: + pass + else: + assert False + + def test_encode_string(self): + """ + Tests encoding string + """ + + if six.PY2: + encoder2 = JSONEncoder(encoding='latin_1', check_circular=False) + assert encoder2.encode(['foo☺']) == '["foo\\u00e2\\u0098\\u00ba"]'