From 089162e6e319a1c35f60398319cf70a13d404fa5 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 13 May 2020 03:11:26 -0700 Subject: [PATCH 1/5] Fix ModelSerializer unique_together handling for field sources (#7143) * Fix ModelSerializer unique_together field sources Updates ModelSerializer to check for serializer fields that map to the model field sources in the unique_together lists. * Ensure field name ordering consistency --- rest_framework/serializers.py | 51 ++++++++++++++++++++++++++--------- tests/test_validators.py | 43 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 12 deletions(-) diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index 8c2486bea..c1cea1e83 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -13,7 +13,7 @@ response content is handled by parsers and renderers. import copy import inspect import traceback -from collections import OrderedDict +from collections import OrderedDict, defaultdict from collections.abc import Mapping from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured @@ -1508,28 +1508,55 @@ class ModelSerializer(Serializer): # which may map onto a model field. Any dotted field name lookups # cannot map to a field, and must be a traversal, so we're not # including those. - field_names = { - field.source for field in self._writable_fields + field_sources = OrderedDict( + (field.field_name, field.source) for field in self._writable_fields if (field.source != '*') and ('.' not in field.source) - } + ) # Special Case: Add read_only fields with defaults. - field_names |= { - field.source for field in self.fields.values() + field_sources.update(OrderedDict( + (field.field_name, field.source) for field in self.fields.values() if (field.read_only) and (field.default != empty) and (field.source != '*') and ('.' not in field.source) - } + )) + + # Invert so we can find the serializer field names that correspond to + # the model field names in the unique_together sets. This also allows + # us to check that multiple fields don't map to the same source. + source_map = defaultdict(list) + for name, source in field_sources.items(): + source_map[source].append(name) # Note that we make sure to check `unique_together` both on the # base model class, but also on any parent classes. validators = [] for parent_class in model_class_inheritance_tree: for unique_together in parent_class._meta.unique_together: - if field_names.issuperset(set(unique_together)): - validator = UniqueTogetherValidator( - queryset=parent_class._default_manager, - fields=unique_together + # Skip if serializer does not map to all unique together sources + if not set(source_map).issuperset(set(unique_together)): + continue + + for source in unique_together: + assert len(source_map[source]) == 1, ( + "Unable to create `UniqueTogetherValidator` for " + "`{model}.{field}` as `{serializer}` has multiple " + "fields ({fields}) that map to this model field. " + "Either remove the extra fields, or override " + "`Meta.validators` with a `UniqueTogetherValidator` " + "using the desired field names." + .format( + model=self.Meta.model.__name__, + serializer=self.__class__.__name__, + field=source, + fields=', '.join(source_map[source]), + ) ) - validators.append(validator) + + field_names = tuple(source_map[f][0] for f in unique_together) + validator = UniqueTogetherValidator( + queryset=parent_class._default_manager, + fields=field_names + ) + validators.append(validator) return validators def get_unique_for_date_validators(self): diff --git a/tests/test_validators.py b/tests/test_validators.py index 21c00073d..4962cf581 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -344,6 +344,49 @@ class TestUniquenessTogetherValidation(TestCase): ] } + def test_default_validator_with_fields_with_source(self): + class TestSerializer(serializers.ModelSerializer): + name = serializers.CharField(source='race_name') + + class Meta: + model = UniquenessTogetherModel + fields = ['name', 'position'] + + serializer = TestSerializer() + expected = dedent(""" + TestSerializer(): + name = CharField(source='race_name') + position = IntegerField() + class Meta: + validators = [] + """) + assert repr(serializer) == expected + + def test_default_validator_with_multiple_fields_with_same_source(self): + class TestSerializer(serializers.ModelSerializer): + name = serializers.CharField(source='race_name') + other_name = serializers.CharField(source='race_name') + + class Meta: + model = UniquenessTogetherModel + fields = ['name', 'other_name', 'position'] + + serializer = TestSerializer(data={ + 'name': 'foo', + 'other_name': 'foo', + 'position': 1, + }) + with pytest.raises(AssertionError) as excinfo: + serializer.is_valid() + + expected = ( + "Unable to create `UniqueTogetherValidator` for " + "`UniquenessTogetherModel.race_name` as `TestSerializer` has " + "multiple fields (name, other_name) that map to this model field. " + "Either remove the extra fields, or override `Meta.validators` " + "with a `UniqueTogetherValidator` using the desired field names.") + assert str(excinfo.value) == expected + def test_allow_explict_override(self): """ Ensure validators can be explicitly removed.. From e888fc11c76d1a671f6f7d3c080915ffe722f3f6 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 13 May 2020 09:59:04 -0400 Subject: [PATCH 2/5] Merge NullBooleanField with BooleanField(allow_null=True) (#7122) * Make `NullBooleanField` subclass `BooleanField` This removes a lot of the redundancy that was in place becuase we were not doing this. This maintains the `None` initial value that was previously present, as well as disallowing `allow_null` to be passed in. * Remove special case for mapping `NullBooleanField` In newer versions of Django, the `NullBooleanField` is handled the same way as a `BooleanField(null=True)`. Given that we also support that combination, and that our own `NullBooleanField` behaves in the same manner, it makes sense to remove the special casing that exists for it. * Add test for BooleanField(null=True, choices) * Remove special case for NullBooleanField * Adjust mapping tests for NullBooleanField * Fixed linting error * Raise deprecation warning when NullBooleanField is used * Fix linting issue in imports --- rest_framework/__init__.py | 4 ++ rest_framework/fields.py | 55 ++++++--------------------- rest_framework/serializers.py | 2 +- rest_framework/utils/field_mapping.py | 2 +- tests/test_model_serializer.py | 23 ++++++++++- 5 files changed, 40 insertions(+), 46 deletions(-) diff --git a/rest_framework/__init__.py b/rest_framework/__init__.py index 8f2bc4466..84e9a8b86 100644 --- a/rest_framework/__init__.py +++ b/rest_framework/__init__.py @@ -31,3 +31,7 @@ class RemovedInDRF313Warning(DeprecationWarning): class RemovedInDRF314Warning(PendingDeprecationWarning): pass + + +class RemovedInDRF314Warning(PendingDeprecationWarning): + pass diff --git a/rest_framework/fields.py b/rest_framework/fields.py index 958bebeef..0447eb74d 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -30,7 +30,9 @@ from django.utils.timezone import utc from django.utils.translation import gettext_lazy as _ from pytz.exceptions import InvalidTimeError -from rest_framework import ISO_8601, RemovedInDRF313Warning +from rest_framework import ( + ISO_8601, RemovedInDRF313Warning, RemovedInDRF314Warning +) from rest_framework.compat import ProhibitNullCharactersValidator from rest_framework.exceptions import ErrorDetail, ValidationError from rest_framework.settings import api_settings @@ -740,55 +742,22 @@ class BooleanField(Field): return bool(value) -class NullBooleanField(Field): - default_error_messages = { - 'invalid': _('Must be a valid boolean.') - } +class NullBooleanField(BooleanField): initial = None - TRUE_VALUES = { - 't', 'T', - 'y', 'Y', 'yes', 'YES', - 'true', 'True', 'TRUE', - 'on', 'On', 'ON', - '1', 1, - True - } - FALSE_VALUES = { - 'f', 'F', - 'n', 'N', 'no', 'NO', - 'false', 'False', 'FALSE', - 'off', 'Off', 'OFF', - '0', 0, 0.0, - False - } - NULL_VALUES = {'null', 'Null', 'NULL', '', None} def __init__(self, **kwargs): + warnings.warn( + "The `NullBooleanField` is deprecated and will be removed starting " + "with 3.14. Instead use the `BooleanField` field and set " + "`null=True` which does the same thing.", + RemovedInDRF314Warning, stacklevel=2 + ) + assert 'allow_null' not in kwargs, '`allow_null` is not a valid option.' kwargs['allow_null'] = True + super().__init__(**kwargs) - def to_internal_value(self, data): - try: - if data in self.TRUE_VALUES: - return True - elif data in self.FALSE_VALUES: - return False - elif data in self.NULL_VALUES: - return None - except TypeError: # Input is an unhashable type - pass - self.fail('invalid', input=data) - - def to_representation(self, value): - if value in self.NULL_VALUES: - return None - if value in self.TRUE_VALUES: - return True - elif value in self.FALSE_VALUES: - return False - return bool(value) - # String types... diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py index c1cea1e83..cfb54de13 100644 --- a/rest_framework/serializers.py +++ b/rest_framework/serializers.py @@ -868,7 +868,7 @@ class ModelSerializer(Serializer): models.FloatField: FloatField, models.ImageField: ImageField, models.IntegerField: IntegerField, - models.NullBooleanField: NullBooleanField, + models.NullBooleanField: BooleanField, models.PositiveIntegerField: IntegerField, models.PositiveSmallIntegerField: IntegerField, models.SlugField: SlugField, diff --git a/rest_framework/utils/field_mapping.py b/rest_framework/utils/field_mapping.py index a25880d0f..ed270be5e 100644 --- a/rest_framework/utils/field_mapping.py +++ b/rest_framework/utils/field_mapping.py @@ -104,7 +104,7 @@ def get_field_kwargs(field_name, model_field): if model_field.has_default() or model_field.blank or model_field.null: kwargs['required'] = False - if model_field.null and not isinstance(model_field, models.NullBooleanField): + if model_field.null: kwargs['allow_null'] = True if model_field.blank and (isinstance(model_field, (models.CharField, models.TextField))): diff --git a/tests/test_model_serializer.py b/tests/test_model_serializer.py index 0de628dc8..51b8f2e22 100644 --- a/tests/test_model_serializer.py +++ b/tests/test_model_serializer.py @@ -182,7 +182,7 @@ class TestRegularFieldMappings(TestCase): email_field = EmailField(max_length=100) float_field = FloatField() integer_field = IntegerField() - null_boolean_field = NullBooleanField(required=False) + null_boolean_field = BooleanField(allow_null=True, required=False) positive_integer_field = IntegerField() positive_small_integer_field = IntegerField() slug_field = SlugField(allow_unicode=False, max_length=100) @@ -236,6 +236,27 @@ class TestRegularFieldMappings(TestCase): self.assertEqual(repr(NullableBooleanSerializer()), expected) + def test_nullable_boolean_field_choices(self): + class NullableBooleanChoicesModel(models.Model): + CHECKLIST_OPTIONS = ( + (None, 'Unknown'), + (True, 'Yes'), + (False, 'No'), + ) + + field = models.BooleanField(null=True, choices=CHECKLIST_OPTIONS) + + class NullableBooleanChoicesSerializer(serializers.ModelSerializer): + class Meta: + model = NullableBooleanChoicesModel + fields = ['field'] + + serializer = NullableBooleanChoicesSerializer(data=dict( + field=None, + )) + self.assertTrue(serializer.is_valid()) + self.assertEqual(serializer.errors, {}) + def test_method_field(self): """ Properties and methods on the model should be allowed as `Meta.fields` From 8bb9a37f4b680a9efb43996705df885dc5b3802e Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 13 May 2020 20:41:53 +0200 Subject: [PATCH 3/5] Removed duplicated class RemovedInDRF314Warning. Added accidently in e888fc11c76d1a671f6f7d3c080915ffe722f3f6 Co-authored-by: Jair Henrique --- rest_framework/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rest_framework/__init__.py b/rest_framework/__init__.py index 84e9a8b86..8f2bc4466 100644 --- a/rest_framework/__init__.py +++ b/rest_framework/__init__.py @@ -31,7 +31,3 @@ class RemovedInDRF313Warning(DeprecationWarning): class RemovedInDRF314Warning(PendingDeprecationWarning): pass - - -class RemovedInDRF314Warning(PendingDeprecationWarning): - pass From fccfdd21c078579bb0029db7289d5e19d58201e8 Mon Sep 17 00:00:00 2001 From: johnthagen Date: Wed, 13 May 2020 20:54:46 -0400 Subject: [PATCH 4/5] Remove object inheritance in docs (#7332) --- docs/api-guide/fields.md | 2 +- docs/api-guide/generic-views.md | 2 +- docs/api-guide/serializers.md | 2 +- docs/api-guide/validators.md | 2 +- rest_framework/schemas/generators.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/api-guide/fields.md b/docs/api-guide/fields.md index 65c83b78e..b2bdd50c8 100644 --- a/docs/api-guide/fields.md +++ b/docs/api-guide/fields.md @@ -603,7 +603,7 @@ The `to_internal_value()` method is called to restore a primitive datatype into Let's look at an example of serializing a class that represents an RGB color value: - class Color(object): + class Color: """ A color represented in the RGB colorspace. """ diff --git a/docs/api-guide/generic-views.md b/docs/api-guide/generic-views.md index a256eb2d9..4ff549f07 100644 --- a/docs/api-guide/generic-views.md +++ b/docs/api-guide/generic-views.md @@ -319,7 +319,7 @@ Often you'll want to use the existing generic views, but use some slightly custo For example, if you need to lookup objects based on multiple fields in the URL conf, you could create a mixin class like the following: - class MultipleFieldLookupMixin(object): + class MultipleFieldLookupMixin: """ Apply this mixin to any view or viewset to get multiple field filtering based on a `lookup_fields` attribute, instead of the default single field filtering. diff --git a/docs/api-guide/serializers.md b/docs/api-guide/serializers.md index 87d3d4056..4f566ff59 100644 --- a/docs/api-guide/serializers.md +++ b/docs/api-guide/serializers.md @@ -21,7 +21,7 @@ Let's start by creating a simple object we can use for example purposes: from datetime import datetime - class Comment(object): + class Comment: def __init__(self, email, content, created=None): self.email = email self.content = content diff --git a/docs/api-guide/validators.md b/docs/api-guide/validators.md index 009cd2468..4451489d4 100644 --- a/docs/api-guide/validators.md +++ b/docs/api-guide/validators.md @@ -282,7 +282,7 @@ to your `Serializer` subclass. This is documented in the To write a class-based validator, use the `__call__` method. Class-based validators are useful as they allow you to parameterize and reuse behavior. - class MultipleOf(object): + class MultipleOf: def __init__(self, base): self.base = base diff --git a/rest_framework/schemas/generators.py b/rest_framework/schemas/generators.py index 4b6d82a14..77502d028 100644 --- a/rest_framework/schemas/generators.py +++ b/rest_framework/schemas/generators.py @@ -143,7 +143,7 @@ class EndpointEnumerator: return [method for method in methods if method not in ('OPTIONS', 'HEAD')] -class BaseSchemaGenerator(object): +class BaseSchemaGenerator: endpoint_inspector_cls = EndpointEnumerator # 'pk' isn't great as an externally exposed name for an identifier, From aed74961ba03e3e6f53c468353f4e255eb788555 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Thu, 14 May 2020 04:24:09 -0300 Subject: [PATCH 5/5] Remove compat for ProhibitNullCharactersValidator (#7333) --- rest_framework/compat.py | 5 ----- rest_framework/fields.py | 8 +++----- tests/test_fields.py | 6 ++---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/rest_framework/compat.py b/rest_framework/compat.py index df100966b..1de23bfaa 100644 --- a/rest_framework/compat.py +++ b/rest_framework/compat.py @@ -19,11 +19,6 @@ except ImportError: RegexURLResolver as URLResolver, ) -try: - from django.core.validators import ProhibitNullCharactersValidator # noqa -except ImportError: - ProhibitNullCharactersValidator = None - def get_original_route(urlpattern): """ diff --git a/rest_framework/fields.py b/rest_framework/fields.py index 0447eb74d..da2dd54be 100644 --- a/rest_framework/fields.py +++ b/rest_framework/fields.py @@ -14,7 +14,8 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError as DjangoValidationError from django.core.validators import ( EmailValidator, MaxLengthValidator, MaxValueValidator, MinLengthValidator, - MinValueValidator, RegexValidator, URLValidator, ip_address_validators + MinValueValidator, ProhibitNullCharactersValidator, RegexValidator, + URLValidator, ip_address_validators ) from django.forms import FilePathField as DjangoFilePathField from django.forms import ImageField as DjangoImageField @@ -33,7 +34,6 @@ from pytz.exceptions import InvalidTimeError from rest_framework import ( ISO_8601, RemovedInDRF313Warning, RemovedInDRF314Warning ) -from rest_framework.compat import ProhibitNullCharactersValidator from rest_framework.exceptions import ErrorDetail, ValidationError from rest_framework.settings import api_settings from rest_framework.utils import html, humanize_datetime, json, representation @@ -785,9 +785,7 @@ class CharField(Field): self.validators.append( MinLengthValidator(self.min_length, message=message)) - # ProhibitNullCharactersValidator is None on Django < 2.0 - if ProhibitNullCharactersValidator is not None: - self.validators.append(ProhibitNullCharactersValidator()) + self.validators.append(ProhibitNullCharactersValidator()) self.validators.append(ProhibitSurrogateCharactersValidator()) def run_validation(self, data=empty): diff --git a/tests/test_fields.py b/tests/test_fields.py index a4b78fd51..b1ad1dc66 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -13,7 +13,6 @@ from django.utils.timezone import activate, deactivate, override, utc import rest_framework from rest_framework import exceptions, serializers -from rest_framework.compat import ProhibitNullCharactersValidator from rest_framework.fields import ( BuiltinSignatureError, DjangoImageField, is_simple_callable ) @@ -747,7 +746,6 @@ class TestCharField(FieldValues): field.run_validation(' ') assert exc_info.value.detail == ['This field may not be blank.'] - @pytest.mark.skipif(ProhibitNullCharactersValidator is None, reason="Skipped on Django < 2.0") def test_null_bytes(self): field = serializers.CharField() @@ -762,8 +760,8 @@ class TestCharField(FieldValues): field = serializers.CharField() for code_point, expected_message in ( - (0xD800, 'Surrogate characters are not allowed: U+D800.'), - (0xDFFF, 'Surrogate characters are not allowed: U+DFFF.'), + (0xD800, 'Surrogate characters are not allowed: U+D800.'), + (0xDFFF, 'Surrogate characters are not allowed: U+DFFF.'), ): with pytest.raises(serializers.ValidationError) as exc_info: field.run_validation(chr(code_point))