From 513a49d63b6332e373c89fb0737a0745c1f0a734 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Tue, 30 Apr 2019 22:49:17 -0700 Subject: [PATCH] Drop default 'utf-8' to .encode()/.decode() (#6633) A Python 3 cleanup that allows for less noise in the code. https://docs.python.org/3/library/stdtypes.html#bytes.decode https://docs.python.org/3/library/stdtypes.html#str.encode --- rest_framework/fields.py | 7 ++---- .../management/commands/generateschema.py | 2 +- rest_framework/parsers.py | 2 +- rest_framework/renderers.py | 23 +++++++------------ rest_framework/utils/encoders.py | 2 +- tests/authentication/test_authentication.py | 2 +- tests/browsable_api/test_browsable_api.py | 12 +++++----- .../test_browsable_nested_api.py | 2 +- tests/test_generics.py | 6 ++--- tests/test_parsers.py | 6 ++--- tests/test_renderers.py | 22 +++++++++--------- tests/test_routers.py | 10 ++++---- tests/test_templates.py | 4 ++-- 13 files changed, 44 insertions(+), 56 deletions(-) diff --git a/rest_framework/fields.py b/rest_framework/fields.py index 1cffdcc2d..a41934ac1 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -1754,7 +1754,7 @@ class JSONField(Field): try: if self.binary or getattr(data, 'is_json_string', False): if isinstance(data, bytes): - data = data.decode('utf-8') + data = data.decode() return json.loads(data) else: json.dumps(data) @@ -1765,10 +1765,7 @@ class JSONField(Field): def to_representation(self, value): if self.binary: value = json.dumps(value) - # On python 2.x the return type for json.dumps() is underspecified. - # On python 3.x json.dumps() returns unicode strings. - if isinstance(value, str): - value = bytes(value.encode('utf-8')) + value = value.encode() return value diff --git a/rest_framework/management/commands/generateschema.py b/rest_framework/management/commands/generateschema.py index 591073ba0..40909bd04 100644 --- a/rest_framework/management/commands/generateschema.py +++ b/rest_framework/management/commands/generateschema.py @@ -29,7 +29,7 @@ class Command(BaseCommand): renderer = self.get_renderer(options['format']) output = renderer.render(schema, renderer_context={}) - self.stdout.write(output.decode('utf-8')) + self.stdout.write(output.decode()) def get_renderer(self, format): renderer_cls = { diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py index 5b5e3f158..a48c31631 100644 --- a/rest_framework/parsers.py +++ b/rest_framework/parsers.py @@ -202,7 +202,7 @@ class FileUploadParser(BaseParser): try: meta = parser_context['request'].META - disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8')) + disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode()) filename_parm = disposition[1] if 'filename*' in filename_parm: return self.get_encoded_filename(filename_parm) diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py index eb5da008b..623702966 100644 --- a/rest_framework/renderers.py +++ b/rest_framework/renderers.py @@ -104,18 +104,11 @@ class JSONRenderer(BaseRenderer): allow_nan=not self.strict, separators=separators ) - # On python 2.x json.dumps() returns bytestrings if ensure_ascii=True, - # but if ensure_ascii=False, the return type is underspecified, - # and may (or may not) be unicode. - # On python 3.x json.dumps() returns unicode strings. - if isinstance(ret, str): - # We always fully escape \u2028 and \u2029 to ensure we output JSON - # that is a strict javascript subset. If bytes were returned - # by json.dumps() then we don't have these characters in any case. - # See: http://timelessrepo.com/json-isnt-a-javascript-subset - ret = ret.replace('\u2028', '\\u2028').replace('\u2029', '\\u2029') - return bytes(ret.encode('utf-8')) - return ret + # We always fully escape \u2028 and \u2029 to ensure we output JSON + # that is a strict javascript subset. + # See: http://timelessrepo.com/json-isnt-a-javascript-subset + ret = ret.replace('\u2028', '\\u2028').replace('\u2029', '\\u2029') + return ret.encode() class TemplateHTMLRenderer(BaseRenderer): @@ -574,7 +567,7 @@ class BrowsableAPIRenderer(BaseRenderer): data.pop(name, None) content = renderer.render(data, accepted, context) # Renders returns bytes, but CharField expects a str. - content = content.decode('utf-8') + content = content.decode() else: content = None @@ -1032,7 +1025,7 @@ class OpenAPIRenderer(_BaseOpenAPIRenderer): def render(self, data, media_type=None, renderer_context=None): structure = self.get_structure(data) - return yaml.dump(structure, default_flow_style=False).encode('utf-8') + return yaml.dump(structure, default_flow_style=False).encode() class JSONOpenAPIRenderer(_BaseOpenAPIRenderer): @@ -1045,4 +1038,4 @@ class JSONOpenAPIRenderer(_BaseOpenAPIRenderer): def render(self, data, media_type=None, renderer_context=None): structure = self.get_structure(data) - return json.dumps(structure, indent=4).encode('utf-8') + return json.dumps(structure, indent=4).encode() diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py index dee2f942e..a7875a868 100644 --- a/rest_framework/utils/encoders.py +++ b/rest_framework/utils/encoders.py @@ -47,7 +47,7 @@ class JSONEncoder(json.JSONEncoder): return tuple(obj) elif isinstance(obj, bytes): # Best-effort for binary blobs. See #4187. - return obj.decode('utf-8') + return obj.decode() elif hasattr(obj, 'tolist'): # Numpy arrays and array scalars. return obj.tolist() diff --git a/tests/authentication/test_authentication.py b/tests/authentication/test_authentication.py index f7e9fcf18..927989028 100644 --- a/tests/authentication/test_authentication.py +++ b/tests/authentication/test_authentication.py @@ -183,7 +183,7 @@ class SessionAuthTests(TestCase): cf. [#1810](https://github.com/encode/django-rest-framework/pull/1810) """ response = self.csrf_client.get('/auth/login/') - content = response.content.decode('utf8') + content = response.content.decode() assert '' in content def test_post_form_session_auth_failing_csrf(self): diff --git a/tests/browsable_api/test_browsable_api.py b/tests/browsable_api/test_browsable_api.py index 81090e223..17644c2ac 100644 --- a/tests/browsable_api/test_browsable_api.py +++ b/tests/browsable_api/test_browsable_api.py @@ -24,18 +24,18 @@ class DropdownWithAuthTests(TestCase): def test_name_shown_when_logged_in(self): self.client.login(username=self.username, password=self.password) response = self.client.get('/') - content = response.content.decode('utf8') + content = response.content.decode() assert 'john' in content def test_logout_shown_when_logged_in(self): self.client.login(username=self.username, password=self.password) response = self.client.get('/') - content = response.content.decode('utf8') + content = response.content.decode() assert '>Log out<' in content def test_login_shown_when_logged_out(self): response = self.client.get('/') - content = response.content.decode('utf8') + content = response.content.decode() assert '>Log in<' in content @@ -59,16 +59,16 @@ class NoDropdownWithoutAuthTests(TestCase): def test_name_shown_when_logged_in(self): self.client.login(username=self.username, password=self.password) response = self.client.get('/') - content = response.content.decode('utf8') + content = response.content.decode() assert 'john' in content def test_dropdown_not_shown_when_logged_in(self): self.client.login(username=self.username, password=self.password) response = self.client.get('/') - content = response.content.decode('utf8') + content = response.content.decode() assert '
  • ' not in content def test_dropdown_not_shown_when_logged_out(self): response = self.client.get('/') - content = response.content.decode('utf8') + content = response.content.decode() assert '
  • ' not in content diff --git a/tests/browsable_api/test_browsable_nested_api.py b/tests/browsable_api/test_browsable_nested_api.py index 3fef74023..f945d2a43 100644 --- a/tests/browsable_api/test_browsable_nested_api.py +++ b/tests/browsable_api/test_browsable_nested_api.py @@ -34,7 +34,7 @@ class DropdownWithAuthTests(TestCase): def test_login(self): response = self.client.get('/api/') assert 200 == response.status_code - content = response.content.decode('utf-8') + content = response.content.decode() assert 'form action="/api/"' in content assert 'input name="nested.one"' in content assert 'input name="nested.two"' in content diff --git a/tests/test_generics.py b/tests/test_generics.py index f41ebe6da..0b91e3465 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -164,7 +164,7 @@ class TestRootView(TestCase): request = factory.post('/', data, HTTP_ACCEPT='text/html') response = self.view(request).render() expected_error = 'Ensure this field has no more than 100 characters.' - assert expected_error in response.rendered_content.decode('utf-8') + assert expected_error in response.rendered_content.decode() EXPECTED_QUERIES_FOR_PUT = 2 @@ -311,7 +311,7 @@ class TestInstanceView(TestCase): request = factory.put('/', data, HTTP_ACCEPT='text/html') response = self.view(request, pk=1).render() expected_error = 'Ensure this field has no more than 100 characters.' - assert expected_error in response.rendered_content.decode('utf-8') + assert expected_error in response.rendered_content.decode() class TestFKInstanceView(TestCase): @@ -538,7 +538,7 @@ class TestFilterBackendAppliedToViews(TestCase): view = DynamicSerializerView.as_view() request = factory.get('/') response = view(request).render() - content = response.content.decode('utf8') + content = response.content.decode() assert 'field_b' in content assert 'field_a' not in content diff --git a/tests/test_parsers.py b/tests/test_parsers.py index 7cf0c938a..dcd62fac9 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -40,9 +40,7 @@ class TestFileUploadParser(TestCase): def setUp(self): class MockRequest: pass - self.stream = io.BytesIO( - "Test text file".encode('utf-8') - ) + self.stream = io.BytesIO(b"Test text file") request = MockRequest() request.upload_handlers = (MemoryFileUploadHandler(),) request.META = { @@ -128,7 +126,7 @@ class TestFileUploadParser(TestCase): class TestJSONParser(TestCase): def bytes(self, value): - return io.BytesIO(value.encode('utf-8')) + return io.BytesIO(value.encode()) def test_float_strictness(self): parser = JSONParser() diff --git a/tests/test_renderers.py b/tests/test_renderers.py index bc775547d..f374fc344 100644 --- a/tests/test_renderers.py +++ b/tests/test_renderers.py @@ -304,14 +304,14 @@ class JSONRendererTests(TestCase): o = DummyTestModel.objects.create(name='dummy') qs = DummyTestModel.objects.values('id', 'name') ret = JSONRenderer().render(qs) - data = json.loads(ret.decode('utf-8')) + data = json.loads(ret.decode()) self.assertEqual(data, [{'id': o.id, 'name': o.name}]) def test_render_queryset_values_list(self): o = DummyTestModel.objects.create(name='dummy') qs = DummyTestModel.objects.values_list('id', 'name') ret = JSONRenderer().render(qs) - data = json.loads(ret.decode('utf-8')) + data = json.loads(ret.decode()) self.assertEqual(data, [[o.id, o.name]]) def test_render_dict_abc_obj(self): @@ -341,7 +341,7 @@ class JSONRendererTests(TestCase): x['key'] = 'string value' x[2] = 3 ret = JSONRenderer().render(x) - data = json.loads(ret.decode('utf-8')) + data = json.loads(ret.decode()) self.assertEqual(data, {'key': 'string value', '2': 3}) def test_render_obj_with_getitem(self): @@ -381,7 +381,7 @@ class JSONRendererTests(TestCase): renderer = JSONRenderer() content = renderer.render(obj, 'application/json') # Fix failing test case which depends on version of JSON library. - self.assertEqual(content.decode('utf-8'), _flat_repr) + self.assertEqual(content.decode(), _flat_repr) def test_with_content_type_args(self): """ @@ -390,7 +390,7 @@ class JSONRendererTests(TestCase): obj = {'foo': ['bar', 'baz']} renderer = JSONRenderer() content = renderer.render(obj, 'application/json; indent=2') - self.assertEqual(strip_trailing_whitespace(content.decode('utf-8')), _indented_repr) + self.assertEqual(strip_trailing_whitespace(content.decode()), _indented_repr) class UnicodeJSONRendererTests(TestCase): @@ -401,7 +401,7 @@ class UnicodeJSONRendererTests(TestCase): obj = {'countries': ['United Kingdom', 'France', 'España']} renderer = JSONRenderer() content = renderer.render(obj, 'application/json') - self.assertEqual(content, '{"countries":["United Kingdom","France","España"]}'.encode('utf-8')) + self.assertEqual(content, '{"countries":["United Kingdom","France","España"]}'.encode()) def test_u2028_u2029(self): # The \u2028 and \u2029 characters should be escaped, @@ -410,7 +410,7 @@ class UnicodeJSONRendererTests(TestCase): obj = {'should_escape': '\u2028\u2029'} renderer = JSONRenderer() content = renderer.render(obj, 'application/json') - self.assertEqual(content, '{"should_escape":"\\u2028\\u2029"}'.encode('utf-8')) + self.assertEqual(content, '{"should_escape":"\\u2028\\u2029"}'.encode()) class AsciiJSONRendererTests(TestCase): @@ -423,7 +423,7 @@ class AsciiJSONRendererTests(TestCase): obj = {'countries': ['United Kingdom', 'France', 'España']} renderer = AsciiJSONRenderer() content = renderer.render(obj, 'application/json') - self.assertEqual(content, '{"countries":["United Kingdom","France","Espa\\u00f1a"]}'.encode('utf-8')) + self.assertEqual(content, '{"countries":["United Kingdom","France","Espa\\u00f1a"]}'.encode()) # Tests for caching issue, #346 @@ -654,9 +654,9 @@ class BrowsableAPIRendererTests(URLPatternsTestCase): def test_extra_actions_dropdown(self): resp = self.client.get('/api/examples/', HTTP_ACCEPT='text/html') - assert 'id="extra-actions-menu"' in resp.content.decode('utf-8') - assert '/api/examples/list_action/' in resp.content.decode('utf-8') - assert '>Extra list action<' in resp.content.decode('utf-8') + assert 'id="extra-actions-menu"' in resp.content.decode() + assert '/api/examples/list_action/' in resp.content.decode() + assert '>Extra list action<' in resp.content.decode() class AdminRendererTests(TestCase): diff --git a/tests/test_routers.py b/tests/test_routers.py index adcec8bd6..0f428e2a5 100644 --- a/tests/test_routers.py +++ b/tests/test_routers.py @@ -438,13 +438,13 @@ class TestEmptyPrefix(URLPatternsTestCase, TestCase): def test_empty_prefix_list(self): response = self.client.get('/empty-prefix/') assert response.status_code == 200 - assert json.loads(response.content.decode('utf-8')) == [{'uuid': '111', 'text': 'First'}, - {'uuid': '222', 'text': 'Second'}] + assert json.loads(response.content.decode()) == [{'uuid': '111', 'text': 'First'}, + {'uuid': '222', 'text': 'Second'}] def test_empty_prefix_detail(self): response = self.client.get('/empty-prefix/1/') assert response.status_code == 200 - assert json.loads(response.content.decode('utf-8')) == {'uuid': '111', 'text': 'First'} + assert json.loads(response.content.decode()) == {'uuid': '111', 'text': 'First'} class TestRegexUrlPath(URLPatternsTestCase, TestCase): @@ -456,14 +456,14 @@ class TestRegexUrlPath(URLPatternsTestCase, TestCase): kwarg = '1234' response = self.client.get('/regex/list/{}/'.format(kwarg)) assert response.status_code == 200 - assert json.loads(response.content.decode('utf-8')) == {'kwarg': kwarg} + assert json.loads(response.content.decode()) == {'kwarg': kwarg} def test_regex_url_path_detail(self): pk = '1' kwarg = '1234' response = self.client.get('/regex/{}/detail/{}/'.format(pk, kwarg)) assert response.status_code == 200 - assert json.loads(response.content.decode('utf-8')) == {'pk': pk, 'kwarg': kwarg} + assert json.loads(response.content.decode()) == {'pk': pk, 'kwarg': kwarg} class TestViewInitkwargs(URLPatternsTestCase, TestCase): diff --git a/tests/test_templates.py b/tests/test_templates.py index 19f511b96..0dba78ea2 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -6,7 +6,7 @@ from django.shortcuts import render def test_base_template_with_context(): context = {'request': True, 'csrf_token': 'TOKEN'} result = render({}, 'rest_framework/base.html', context=context) - assert re.search(r'\bcsrfToken: "TOKEN"', result.content.decode('utf-8')) + assert re.search(r'\bcsrfToken: "TOKEN"', result.content.decode()) def test_base_template_with_no_context(): @@ -14,4 +14,4 @@ def test_base_template_with_no_context(): # so it can be easily extended. result = render({}, 'rest_framework/base.html') # note that this response will not include a valid CSRF token - assert re.search(r'\bcsrfToken: ""', result.content.decode('utf-8')) + assert re.search(r'\bcsrfToken: ""', result.content.decode())