Drop default 'utf-8' to .encode()/.decode() (#6633)

A Python 3 cleanup that allows for less noise in the code.

https://docs.python.org/3/library/stdtypes.html#bytes.decode
https://docs.python.org/3/library/stdtypes.html#str.encode

rest_framework/fields.py

@@ -1754,7 +1754,7 @@ class JSONField(Field):
         try:
             if self.binary or getattr(data, 'is_json_string', False):
                 if isinstance(data, bytes):
-                    data = data.decode('utf-8')
+                    data = data.decode()
                 return json.loads(data)
             else:
                 json.dumps(data)
@@ -1765,10 +1765,7 @@ class JSONField(Field):
     def to_representation(self, value):
         if self.binary:
             value = json.dumps(value)
-            # On python 2.x the return type for json.dumps() is underspecified.
+            value = value.encode()
-            # On python 3.x json.dumps() returns unicode strings.
-            if isinstance(value, str):
-                value = bytes(value.encode('utf-8'))
         return value
 
 

rest_framework/management/commands/generateschema.py

@@ -29,7 +29,7 @@ class Command(BaseCommand):
 
         renderer = self.get_renderer(options['format'])
         output = renderer.render(schema, renderer_context={})
-        self.stdout.write(output.decode('utf-8'))
+        self.stdout.write(output.decode())
 
     def get_renderer(self, format):
         renderer_cls = {

rest_framework/parsers.py

@@ -202,7 +202,7 @@ class FileUploadParser(BaseParser):
 
         try:
             meta = parser_context['request'].META
-            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8'))
+            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode())
             filename_parm = disposition[1]
             if 'filename*' in filename_parm:
                 return self.get_encoded_filename(filename_parm)

rest_framework/renderers.py

@@ -104,18 +104,11 @@ class JSONRenderer(BaseRenderer):
             allow_nan=not self.strict, separators=separators
         )
 
-        # On python 2.x json.dumps() returns bytestrings if ensure_ascii=True,
+        # We always fully escape \u2028 and \u2029 to ensure we output JSON
-        # but if ensure_ascii=False, the return type is underspecified,
+        # that is a strict javascript subset.
-        # and may (or may not) be unicode.
+        # See: http://timelessrepo.com/json-isnt-a-javascript-subset
-        # On python 3.x json.dumps() returns unicode strings.
+        ret = ret.replace('\u2028', '\\u2028').replace('\u2029', '\\u2029')
-        if isinstance(ret, str):
+        return ret.encode()
-            # We always fully escape \u2028 and \u2029 to ensure we output JSON
-            # that is a strict javascript subset. If bytes were returned
-            # by json.dumps() then we don't have these characters in any case.
-            # See: http://timelessrepo.com/json-isnt-a-javascript-subset
-            ret = ret.replace('\u2028', '\\u2028').replace('\u2029', '\\u2029')
-            return bytes(ret.encode('utf-8'))
-        return ret
 
 
 class TemplateHTMLRenderer(BaseRenderer):
@@ -574,7 +567,7 @@ class BrowsableAPIRenderer(BaseRenderer):
                         data.pop(name, None)
                 content = renderer.render(data, accepted, context)
                 # Renders returns bytes, but CharField expects a str.
-                content = content.decode('utf-8')
+                content = content.decode()
             else:
                 content = None
 
@@ -1032,7 +1025,7 @@ class OpenAPIRenderer(_BaseOpenAPIRenderer):
 
     def render(self, data, media_type=None, renderer_context=None):
         structure = self.get_structure(data)
-        return yaml.dump(structure, default_flow_style=False).encode('utf-8')
+        return yaml.dump(structure, default_flow_style=False).encode()
 
 
 class JSONOpenAPIRenderer(_BaseOpenAPIRenderer):
@@ -1045,4 +1038,4 @@ class JSONOpenAPIRenderer(_BaseOpenAPIRenderer):
 
     def render(self, data, media_type=None, renderer_context=None):
         structure = self.get_structure(data)
-        return json.dumps(structure, indent=4).encode('utf-8')
+        return json.dumps(structure, indent=4).encode()

rest_framework/utils/encoders.py

@@ -47,7 +47,7 @@ class JSONEncoder(json.JSONEncoder):
             return tuple(obj)
         elif isinstance(obj, bytes):
             # Best-effort for binary blobs. See #4187.
-            return obj.decode('utf-8')
+            return obj.decode()
         elif hasattr(obj, 'tolist'):
             # Numpy arrays and array scalars.
             return obj.tolist()

tests/authentication/test_authentication.py

@@ -183,7 +183,7 @@ class SessionAuthTests(TestCase):
         cf. [#1810](https://github.com/encode/django-rest-framework/pull/1810)
         """
         response = self.csrf_client.get('/auth/login/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert '<label for="id_username">Username:</label>' in content
 
     def test_post_form_session_auth_failing_csrf(self):

tests/browsable_api/test_browsable_api.py

@@ -24,18 +24,18 @@ class DropdownWithAuthTests(TestCase):
     def test_name_shown_when_logged_in(self):
         self.client.login(username=self.username, password=self.password)
         response = self.client.get('/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert 'john' in content
 
     def test_logout_shown_when_logged_in(self):
         self.client.login(username=self.username, password=self.password)
         response = self.client.get('/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert '>Log out<' in content
 
     def test_login_shown_when_logged_out(self):
         response = self.client.get('/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert '>Log in<' in content
 
 
@@ -59,16 +59,16 @@ class NoDropdownWithoutAuthTests(TestCase):
     def test_name_shown_when_logged_in(self):
         self.client.login(username=self.username, password=self.password)
         response = self.client.get('/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert 'john' in content
 
     def test_dropdown_not_shown_when_logged_in(self):
         self.client.login(username=self.username, password=self.password)
         response = self.client.get('/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert '<li class="dropdown">' not in content
 
     def test_dropdown_not_shown_when_logged_out(self):
         response = self.client.get('/')
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert '<li class="dropdown">' not in content

tests/browsable_api/test_browsable_nested_api.py

@@ -34,7 +34,7 @@ class DropdownWithAuthTests(TestCase):
     def test_login(self):
         response = self.client.get('/api/')
         assert 200 == response.status_code
-        content = response.content.decode('utf-8')
+        content = response.content.decode()
         assert 'form action="/api/"' in content
         assert 'input name="nested.one"' in content
         assert 'input name="nested.two"' in content

tests/test_generics.py

@@ -164,7 +164,7 @@ class TestRootView(TestCase):
         request = factory.post('/', data, HTTP_ACCEPT='text/html')
         response = self.view(request).render()
         expected_error = '<span class="help-block">Ensure this field has no more than 100 characters.</span>'
-        assert expected_error in response.rendered_content.decode('utf-8')
+        assert expected_error in response.rendered_content.decode()
 
 
 EXPECTED_QUERIES_FOR_PUT = 2
@@ -311,7 +311,7 @@ class TestInstanceView(TestCase):
         request = factory.put('/', data, HTTP_ACCEPT='text/html')
         response = self.view(request, pk=1).render()
         expected_error = '<span class="help-block">Ensure this field has no more than 100 characters.</span>'
-        assert expected_error in response.rendered_content.decode('utf-8')
+        assert expected_error in response.rendered_content.decode()
 
 
 class TestFKInstanceView(TestCase):
@@ -538,7 +538,7 @@ class TestFilterBackendAppliedToViews(TestCase):
         view = DynamicSerializerView.as_view()
         request = factory.get('/')
         response = view(request).render()
-        content = response.content.decode('utf8')
+        content = response.content.decode()
         assert 'field_b' in content
         assert 'field_a' not in content
 

tests/test_parsers.py

@@ -40,9 +40,7 @@ class TestFileUploadParser(TestCase):
     def setUp(self):
         class MockRequest:
             pass
-        self.stream = io.BytesIO(
+        self.stream = io.BytesIO(b"Test text file")
-            "Test text file".encode('utf-8')
-        )
         request = MockRequest()
         request.upload_handlers = (MemoryFileUploadHandler(),)
         request.META = {
@@ -128,7 +126,7 @@ class TestFileUploadParser(TestCase):
 
 class TestJSONParser(TestCase):
     def bytes(self, value):
-        return io.BytesIO(value.encode('utf-8'))
+        return io.BytesIO(value.encode())
 
     def test_float_strictness(self):
         parser = JSONParser()

tests/test_renderers.py

@@ -304,14 +304,14 @@ class JSONRendererTests(TestCase):
         o = DummyTestModel.objects.create(name='dummy')
         qs = DummyTestModel.objects.values('id', 'name')
         ret = JSONRenderer().render(qs)
-        data = json.loads(ret.decode('utf-8'))
+        data = json.loads(ret.decode())
         self.assertEqual(data, [{'id': o.id, 'name': o.name}])
 
     def test_render_queryset_values_list(self):
         o = DummyTestModel.objects.create(name='dummy')
         qs = DummyTestModel.objects.values_list('id', 'name')
         ret = JSONRenderer().render(qs)
-        data = json.loads(ret.decode('utf-8'))
+        data = json.loads(ret.decode())
         self.assertEqual(data, [[o.id, o.name]])
 
     def test_render_dict_abc_obj(self):
@@ -341,7 +341,7 @@ class JSONRendererTests(TestCase):
         x['key'] = 'string value'
         x[2] = 3
         ret = JSONRenderer().render(x)
-        data = json.loads(ret.decode('utf-8'))
+        data = json.loads(ret.decode())
         self.assertEqual(data, {'key': 'string value', '2': 3})
 
     def test_render_obj_with_getitem(self):
@@ -381,7 +381,7 @@ class JSONRendererTests(TestCase):
         renderer = JSONRenderer()
         content = renderer.render(obj, 'application/json')
         # Fix failing test case which depends on version of JSON library.
-        self.assertEqual(content.decode('utf-8'), _flat_repr)
+        self.assertEqual(content.decode(), _flat_repr)
 
     def test_with_content_type_args(self):
         """
@@ -390,7 +390,7 @@ class JSONRendererTests(TestCase):
         obj = {'foo': ['bar', 'baz']}
         renderer = JSONRenderer()
         content = renderer.render(obj, 'application/json; indent=2')
-        self.assertEqual(strip_trailing_whitespace(content.decode('utf-8')), _indented_repr)
+        self.assertEqual(strip_trailing_whitespace(content.decode()), _indented_repr)
 
 
 class UnicodeJSONRendererTests(TestCase):
@@ -401,7 +401,7 @@ class UnicodeJSONRendererTests(TestCase):
         obj = {'countries': ['United Kingdom', 'France', 'España']}
         renderer = JSONRenderer()
         content = renderer.render(obj, 'application/json')
-        self.assertEqual(content, '{"countries":["United Kingdom","France","España"]}'.encode('utf-8'))
+        self.assertEqual(content, '{"countries":["United Kingdom","France","España"]}'.encode())
 
     def test_u2028_u2029(self):
         # The \u2028 and \u2029 characters should be escaped,
@@ -410,7 +410,7 @@ class UnicodeJSONRendererTests(TestCase):
         obj = {'should_escape': '\u2028\u2029'}
         renderer = JSONRenderer()
         content = renderer.render(obj, 'application/json')
-        self.assertEqual(content, '{"should_escape":"\\u2028\\u2029"}'.encode('utf-8'))
+        self.assertEqual(content, '{"should_escape":"\\u2028\\u2029"}'.encode())
 
 
 class AsciiJSONRendererTests(TestCase):
@@ -423,7 +423,7 @@ class AsciiJSONRendererTests(TestCase):
         obj = {'countries': ['United Kingdom', 'France', 'España']}
         renderer = AsciiJSONRenderer()
         content = renderer.render(obj, 'application/json')
-        self.assertEqual(content, '{"countries":["United Kingdom","France","Espa\\u00f1a"]}'.encode('utf-8'))
+        self.assertEqual(content, '{"countries":["United Kingdom","France","Espa\\u00f1a"]}'.encode())
 
 
 # Tests for caching issue, #346
@@ -654,9 +654,9 @@ class BrowsableAPIRendererTests(URLPatternsTestCase):
 
     def test_extra_actions_dropdown(self):
         resp = self.client.get('/api/examples/', HTTP_ACCEPT='text/html')
-        assert 'id="extra-actions-menu"' in resp.content.decode('utf-8')
+        assert 'id="extra-actions-menu"' in resp.content.decode()
-        assert '/api/examples/list_action/' in resp.content.decode('utf-8')
+        assert '/api/examples/list_action/' in resp.content.decode()
-        assert '>Extra list action<' in resp.content.decode('utf-8')
+        assert '>Extra list action<' in resp.content.decode()
 
 
 class AdminRendererTests(TestCase):

tests/test_routers.py

@@ -438,13 +438,13 @@ class TestEmptyPrefix(URLPatternsTestCase, TestCase):
     def test_empty_prefix_list(self):
         response = self.client.get('/empty-prefix/')
         assert response.status_code == 200
-        assert json.loads(response.content.decode('utf-8')) == [{'uuid': '111', 'text': 'First'},
+        assert json.loads(response.content.decode()) == [{'uuid': '111', 'text': 'First'},
-                                                                {'uuid': '222', 'text': 'Second'}]
+                                                         {'uuid': '222', 'text': 'Second'}]
 
     def test_empty_prefix_detail(self):
         response = self.client.get('/empty-prefix/1/')
         assert response.status_code == 200
-        assert json.loads(response.content.decode('utf-8')) == {'uuid': '111', 'text': 'First'}
+        assert json.loads(response.content.decode()) == {'uuid': '111', 'text': 'First'}
 
 
 class TestRegexUrlPath(URLPatternsTestCase, TestCase):
@@ -456,14 +456,14 @@ class TestRegexUrlPath(URLPatternsTestCase, TestCase):
         kwarg = '1234'
         response = self.client.get('/regex/list/{}/'.format(kwarg))
         assert response.status_code == 200
-        assert json.loads(response.content.decode('utf-8')) == {'kwarg': kwarg}
+        assert json.loads(response.content.decode()) == {'kwarg': kwarg}
 
     def test_regex_url_path_detail(self):
         pk = '1'
         kwarg = '1234'
         response = self.client.get('/regex/{}/detail/{}/'.format(pk, kwarg))
         assert response.status_code == 200
-        assert json.loads(response.content.decode('utf-8')) == {'pk': pk, 'kwarg': kwarg}
+        assert json.loads(response.content.decode()) == {'pk': pk, 'kwarg': kwarg}
 
 
 class TestViewInitkwargs(URLPatternsTestCase, TestCase):

tests/test_templates.py

@@ -6,7 +6,7 @@ from django.shortcuts import render
 def test_base_template_with_context():
     context = {'request': True, 'csrf_token': 'TOKEN'}
     result = render({}, 'rest_framework/base.html', context=context)
-    assert re.search(r'\bcsrfToken: "TOKEN"', result.content.decode('utf-8'))
+    assert re.search(r'\bcsrfToken: "TOKEN"', result.content.decode())
 
 
 def test_base_template_with_no_context():
@@ -14,4 +14,4 @@ def test_base_template_with_no_context():
     # so it can be easily extended.
     result = render({}, 'rest_framework/base.html')
     # note that this response will not include a valid CSRF token
-    assert re.search(r'\bcsrfToken: ""', result.content.decode('utf-8'))
+    assert re.search(r'\bcsrfToken: ""', result.content.decode())