Merge branch 'encode:master' into fix-ordering-field-rename

docs/api-guide/filtering.md

@@ -218,14 +218,18 @@ For [JSONField][JSONField] and [HStoreField][HStoreField] fields you can filter
 
     search_fields = ['data__breed', 'data__owner__other_pets__0__name']
 
-By default, searches will use case-insensitive partial matches.  The search parameter may contain multiple search terms, which should be whitespace and/or comma separated.  If multiple search terms are used then objects will be returned in the list only if all the provided terms are matched.
+By default, searches will use case-insensitive partial matches. The search parameter may contain multiple search terms, which should be whitespace and/or comma separated. If multiple search terms are used then objects will be returned in the list only if all the provided terms are matched. Searches may contain _quoted phrases_ with spaces, each phrase is considered as a single search term.
 
-The search behavior may be restricted by prepending various characters to the `search_fields`.
 
-* '^' Starts-with search.
-* '=' Exact matches.
-* '@' Full-text search.  (Currently only supported Django's [PostgreSQL backend][postgres-search].)
-* '$' Regex search.
+The search behavior may be specified by prefixing field names in `search_fields` with one of the following characters (which is equivalent to adding `__<lookup>` to the field):
+
+| Prefix | Lookup        |                    |
+| ------ | --------------| ------------------ |
+| `^`    | `istartswith` | Starts-with search.|
+| `=`    | `iexact`      | Exact matches.     |
+| `$`    | `iregex`      | Regex search.      |
+| `@`    | `search`      | Full-text search (Currently only supported Django's [PostgreSQL backend][postgres-search]). |
+| None   | `icontains`   | Contains search (Default).  |
 
 For example:
 

rest_framework/compat.py

@@ -3,7 +3,6 @@ The `compat` module provides support for backwards compatibility with older
 versions of Django/Python, and compatibility wrappers around optional packages.
 """
 import django
-from django.conf import settings
 from django.views.generic import View
 
 
@@ -14,13 +13,6 @@ def unicode_http_header(value):
     return value
 
 
-def distinct(queryset, base):
-    if settings.DATABASES[queryset.db]["ENGINE"] == "django.db.backends.oracle":
-        # distinct analogue for Oracle users
-        return base.filter(pk__in=set(queryset.values_list('pk', flat=True)))
-    return queryset.distinct()
-
-
 # django.contrib.postgres requires psycopg2
 try:
     from django.contrib.postgres import fields as postgres_fields

rest_framework/fields.py

@@ -8,6 +8,7 @@ import logging
 import re
 import uuid
 from collections.abc import Mapping
+from enum import Enum
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
@@ -17,7 +18,6 @@ from django.core.validators import (
     MinValueValidator, ProhibitNullCharactersValidator, RegexValidator,
     URLValidator, ip_address_validators
 )
-from django.db.models import IntegerChoices, TextChoices
 from django.forms import FilePathField as DjangoFilePathField
 from django.forms import ImageField as DjangoImageField
 from django.utils import timezone
@@ -1401,11 +1401,8 @@ class ChoiceField(Field):
     def to_internal_value(self, data):
         if data == '' and self.allow_blank:
             return ''
-
-        if isinstance(data, (IntegerChoices, TextChoices)) and str(data) != \
-                str(data.value):
+        if isinstance(data, Enum) and str(data) != str(data.value):
             data = data.value
-
         try:
             return self.choice_strings_to_values[str(data)]
         except KeyError:
@@ -1414,11 +1411,8 @@ class ChoiceField(Field):
     def to_representation(self, value):
         if value in ('', None):
             return value
-
-        if isinstance(value, (IntegerChoices, TextChoices)) and str(value) != \
-                str(value.value):
+        if isinstance(value, Enum) and str(value) != str(value.value):
             value = value.value
-
         return self.choice_strings_to_values.get(str(value), value)
 
     def iter_options(self):
@@ -1442,8 +1436,7 @@ class ChoiceField(Field):
         # Allows us to deal with eg. integer choices while supporting either
         # integer or string input, but still get the correct datatype out.
         self.choice_strings_to_values = {
-            str(key.value) if isinstance(key, (IntegerChoices, TextChoices))
-            and str(key) != str(key.value) else str(key): key for key in self.choices
+            str(key.value) if isinstance(key, Enum) and str(key) != str(key.value) else str(key): key for key in self.choices
         }
 
     choices = property(_get_choices, _set_choices)
@@ -1829,6 +1822,7 @@ class HiddenField(Field):
     constraint on a pair of fields, as we need some way to include the date in
     the validated data.
     """
+
     def __init__(self, **kwargs):
         assert 'default' in kwargs, 'default is a required argument.'
         kwargs['write_only'] = True
@@ -1858,6 +1852,7 @@ class SerializerMethodField(Field):
         def get_extra_info(self, obj):
             return ...  # Calculate some data to return.
     """
+
     def __init__(self, method_name=None, **kwargs):
         self.method_name = method_name
         kwargs['source'] = '*'

rest_framework/filters.py

@@ -6,18 +6,35 @@ import operator
 import warnings
 from functools import reduce
 
-from django.core.exceptions import ImproperlyConfigured
+from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured
 from django.db import models
 from django.db.models.constants import LOOKUP_SEP
 from django.template import loader
 from django.utils.encoding import force_str
+from django.utils.text import smart_split, unescape_string_literal
 from django.utils.translation import gettext_lazy as _
 
 from rest_framework import RemovedInDRF317Warning
-from rest_framework.compat import coreapi, coreschema, distinct
+from rest_framework.compat import coreapi, coreschema
+from rest_framework.fields import CharField
 from rest_framework.settings import api_settings
 
 
+def search_smart_split(search_terms):
+    """generator that first splits string by spaces, leaving quoted phrases togheter,
+    then it splits non-quoted phrases by commas.
+    """
+    for term in smart_split(search_terms):
+        # trim commas to avoid bad matching for quoted phrases
+        term = term.strip(',')
+        if term.startswith(('"', "'")) and term[0] == term[-1]:
+            # quoted phrases are kept togheter without any other split
+            yield unescape_string_literal(term)
+        else:
+            # non-quoted tokens are split by comma, keeping only non-empty ones
+            yield from (sub_term.strip() for sub_term in term.split(',') if sub_term)
+
+
 class BaseFilterBackend:
     """
     A base class from which all filter backend classes should inherit.
@@ -64,18 +81,41 @@ class SearchFilter(BaseFilterBackend):
     def get_search_terms(self, request):
         """
         Search terms are set by a ?search=... query parameter,
-        and may be comma and/or whitespace delimited.
+        and may be whitespace delimited.
         """
-        params = request.query_params.get(self.search_param, '')
-        params = params.replace('\x00', '')  # strip null characters
-        params = params.replace(',', ' ')
-        return params.split()
+        value = request.query_params.get(self.search_param, '')
+        field = CharField(trim_whitespace=False, allow_blank=True)
+        return field.run_validation(value)
 
-    def construct_search(self, field_name):
+    def construct_search(self, field_name, queryset):
         lookup = self.lookup_prefixes.get(field_name[0])
         if lookup:
             field_name = field_name[1:]
         else:
+            # Use field_name if it includes a lookup.
+            opts = queryset.model._meta
+            lookup_fields = field_name.split(LOOKUP_SEP)
+            # Go through the fields, following all relations.
+            prev_field = None
+            for path_part in lookup_fields:
+                if path_part == "pk":
+                    path_part = opts.pk.name
+                try:
+                    field = opts.get_field(path_part)
+                except FieldDoesNotExist:
+                    # Use valid query lookups.
+                    if prev_field and prev_field.get_lookup(path_part):
+                        return field_name
+                else:
+                    prev_field = field
+                    if hasattr(field, "path_infos"):
+                        # Update opts to follow the relation.
+                        opts = field.path_infos[-1].to_opts
+                    # django < 4.1
+                    elif hasattr(field, 'get_path_info'):
+                        # Update opts to follow the relation.
+                        opts = field.get_path_info()[-1].to_opts
+            # Otherwise, use the field with icontains.
             lookup = 'icontains'
         return LOOKUP_SEP.join([field_name, lookup])
 
@@ -113,37 +153,36 @@ class SearchFilter(BaseFilterBackend):
             return queryset
 
         orm_lookups = [
-            self.construct_search(str(search_field))
+            self.construct_search(str(search_field), queryset)
             for search_field in search_fields
         ]
 
         base = queryset
-        conditions = []
-        for search_term in search_terms:
-            queries = [
-                models.Q(**{orm_lookup: search_term})
-                for orm_lookup in orm_lookups
-            ]
-            conditions.append(reduce(operator.or_, queries))
+        # generator which for each term builds the corresponding search
+        conditions = (
+            reduce(
+                operator.or_,
+                (models.Q(**{orm_lookup: term}) for orm_lookup in orm_lookups)
+            ) for term in search_smart_split(search_terms)
+        )
         queryset = queryset.filter(reduce(operator.and_, conditions))
 
+        # Remove duplicates from results, if necessary
         if self.must_call_distinct(queryset, search_fields):
-            # Filtering against a many-to-many field requires us to
-            # call queryset.distinct() in order to avoid duplicate items
-            # in the resulting queryset.
-            # We try to avoid this if possible, for performance reasons.
-            queryset = distinct(queryset, base)
+            # inspired by django.contrib.admin
+            # this is more accurate than .distinct form M2M relationship
+            # also is cross-database
+            queryset = queryset.filter(pk=models.OuterRef('pk'))
+            queryset = base.filter(models.Exists(queryset))
         return queryset
 
     def to_html(self, request, queryset, view):
         if not getattr(view, 'search_fields', None):
             return ''
 
-        term = self.get_search_terms(request)
-        term = term[0] if term else ''
         context = {
             'param': self.search_param,
-            'term': term
+            'term': request.query_params.get(self.search_param, ''),
         }
         template = loader.get_template(self.template)
         return template.render(context)

rest_framework/pagination.py

@@ -239,6 +239,7 @@ class PageNumberPagination(BasePagination):
     def get_paginated_response_schema(self, schema):
         return {
             'type': 'object',
+            'required': ['count', 'results'],
             'properties': {
                 'count': {
                     'type': 'integer',
@@ -411,6 +412,7 @@ class LimitOffsetPagination(BasePagination):
     def get_paginated_response_schema(self, schema):
         return {
             'type': 'object',
+            'required': ['count', 'results'],
             'properties': {
                 'count': {
                     'type': 'integer',
@@ -906,6 +908,7 @@ class CursorPagination(BasePagination):
     def get_paginated_response_schema(self, schema):
         return {
             'type': 'object',
+            'required': ['results'],
             'properties': {
                 'next': {
                     'type': 'string',

rest_framework/serializers.py

@@ -653,6 +653,17 @@ class ListSerializer(BaseSerializer):
 
         return value
 
+    def run_child_validation(self, data):
+        """
+        Run validation on child serializer.
+        You may need to override this method to support multiple updates. For example:
+
+        self.child.instance = self.instance.get(pk=data['id'])
+        self.child.initial_data = data
+        return super().run_child_validation(data)
+        """
+        return self.child.run_validation(data)
+
     def to_internal_value(self, data):
         """
         List of dicts of native values <- List of dicts of primitive datatypes.
@@ -697,7 +708,7 @@ class ListSerializer(BaseSerializer):
             ):
                 self.child.instance = self.instance[idx]
             try:
-                validated = self.child.run_validation(item)
+                validated = self.run_child_validation(item)
             except ValidationError as exc:
                 errors.append(exc.detail)
             else:
@@ -1372,8 +1383,8 @@ class ModelSerializer(Serializer):
         Raise an error on any unknown fields.
         """
         raise ImproperlyConfigured(
-            'Field name `%s` is not valid for model `%s`.' %
-            (field_name, model_class.__name__)
+            'Field name `%s` is not valid for model `%s` in `%s.%s`.' %
+            (field_name, model_class.__name__, self.__class__.__module__, self.__class__.__name__)
         )
 
     def include_extra_kwargs(self, kwargs, extra_kwargs):

tests/test_fields.py

@@ -1875,6 +1875,31 @@ class TestChoiceField(FieldValues):
             field.run_validation(2)
         assert exc_info.value.detail == ['"2" is not a valid choice.']
 
+    def test_enum_integer_choices(self):
+        from enum import IntEnum
+
+        class ChoiceCase(IntEnum):
+            first = auto()
+            second = auto()
+        # Enum validate
+        choices = [
+            (ChoiceCase.first, "1"),
+            (ChoiceCase.second, "2")
+        ]
+        field = serializers.ChoiceField(choices=choices)
+        assert field.run_validation(1) == 1
+        assert field.run_validation(ChoiceCase.first) == 1
+        assert field.run_validation("1") == 1
+        # Enum.value validate
+        choices = [
+            (ChoiceCase.first.value, "1"),
+            (ChoiceCase.second.value, "2")
+        ]
+        field = serializers.ChoiceField(choices=choices)
+        assert field.run_validation(1) == 1
+        assert field.run_validation(ChoiceCase.first) == 1
+        assert field.run_validation("1") == 1
+
     def test_integer_choices(self):
         class ChoiceCase(IntegerChoices):
             first = auto()

tests/test_filters.py

@@ -6,16 +6,36 @@ from django.core.exceptions import ImproperlyConfigured
 from django.db import models
 from django.db.models import CharField, Transform
 from django.db.models.functions import Concat, Upper
-from django.test import TestCase
+from django.test import SimpleTestCase, TestCase
 from django.test.utils import override_settings
 
 from rest_framework import filters, generics, serializers
 from rest_framework.compat import coreschema
+from rest_framework.exceptions import ValidationError
 from rest_framework.test import APIRequestFactory
 
 factory = APIRequestFactory()
 
 
+class SearchSplitTests(SimpleTestCase):
+
+    def test_keep_quoted_togheter_regardless_of_commas(self):
+        assert ['hello, world'] == list(filters.search_smart_split('"hello, world"'))
+
+    def test_strips_commas_around_quoted(self):
+        assert ['hello, world'] == list(filters.search_smart_split(',,"hello, world"'))
+        assert ['hello, world'] == list(filters.search_smart_split(',,"hello, world",,'))
+        assert ['hello, world'] == list(filters.search_smart_split('"hello, world",,'))
+
+    def test_splits_by_comma(self):
+        assert ['hello', 'world'] == list(filters.search_smart_split(',,hello, world'))
+        assert ['hello', 'world'] == list(filters.search_smart_split(',,hello, world,,'))
+        assert ['hello', 'world'] == list(filters.search_smart_split('hello, world,,'))
+
+    def test_splits_quotes_followed_by_comma_and_sentence(self):
+        assert ['"hello', 'world"', 'found'] == list(filters.search_smart_split('"hello, world",found'))
+
+
 class BaseFilterTests(TestCase):
     def setUp(self):
         self.original_coreapi = filters.coreapi
@@ -50,7 +70,8 @@ class SearchFilterSerializer(serializers.ModelSerializer):
 
 
 class SearchFilterTests(TestCase):
-    def setUp(self):
+    @classmethod
+    def setUpTestData(cls):
         # Sequence of title/text is:
         #
         # z   abc
@@ -66,6 +87,9 @@ class SearchFilterTests(TestCase):
             )
             SearchFilterModel(title=title, text=text).save()
 
+        SearchFilterModel(title='A title', text='The long text').save()
+        SearchFilterModel(title='The title', text='The "text').save()
+
     def test_search(self):
         class SearchListView(generics.ListAPIView):
             queryset = SearchFilterModel.objects.all()
@@ -186,9 +210,21 @@ class SearchFilterTests(TestCase):
         request = factory.get('/?search=\0as%00d\x00f')
         request = view.initialize_request(request)
 
-        terms = filters.SearchFilter().get_search_terms(request)
+        with self.assertRaises(ValidationError):
+            filters.SearchFilter().get_search_terms(request)
 
-        assert terms == ['asdf']
+    def test_search_field_with_custom_lookup(self):
+        class SearchListView(generics.ListAPIView):
+            queryset = SearchFilterModel.objects.all()
+            serializer_class = SearchFilterSerializer
+            filter_backends = (filters.SearchFilter,)
+            search_fields = ('text__iendswith',)
+        view = SearchListView.as_view()
+        request = factory.get('/', {'search': 'c'})
+        response = view(request)
+        assert response.data == [
+            {'id': 1, 'title': 'z', 'text': 'abc'},
+        ]
 
     def test_search_field_with_additional_transforms(self):
         from django.test.utils import register_lookup
@@ -225,6 +261,49 @@ class SearchFilterTests(TestCase):
                 {'id': 2, 'title': 'zz', 'text': 'bcd'},
             ]
 
+    def test_search_field_with_multiple_words(self):
+        class SearchListView(generics.ListAPIView):
+            queryset = SearchFilterModel.objects.all()
+            serializer_class = SearchFilterSerializer
+            filter_backends = (filters.SearchFilter,)
+            search_fields = ('title', 'text')
+
+        search_query = 'foo bar,baz'
+        view = SearchListView()
+        request = factory.get('/', {'search': search_query})
+        request = view.initialize_request(request)
+
+        rendered_search_field = filters.SearchFilter().to_html(
+            request=request, queryset=view.queryset, view=view
+        )
+        assert search_query in rendered_search_field
+
+    def test_search_field_with_escapes(self):
+        class SearchListView(generics.ListAPIView):
+            queryset = SearchFilterModel.objects.all()
+            serializer_class = SearchFilterSerializer
+            filter_backends = (filters.SearchFilter,)
+            search_fields = ('title', 'text',)
+        view = SearchListView.as_view()
+        request = factory.get('/', {'search': '"\\\"text"'})
+        response = view(request)
+        assert response.data == [
+            {'id': 12, 'title': 'The title', 'text': 'The "text'},
+        ]
+
+    def test_search_field_with_quotes(self):
+        class SearchListView(generics.ListAPIView):
+            queryset = SearchFilterModel.objects.all()
+            serializer_class = SearchFilterSerializer
+            filter_backends = (filters.SearchFilter,)
+            search_fields = ('title', 'text',)
+        view = SearchListView.as_view()
+        request = factory.get('/', {'search': '"long text"'})
+        response = view(request)
+        assert response.data == [
+            {'id': 11, 'title': 'A title', 'text': 'The long text'},
+        ]
+
 
 class AttributeModel(models.Model):
     label = models.CharField(max_length=32)
@@ -267,6 +346,13 @@ class SearchFilterFkTests(TestCase):
                 ["%sattribute__label" % prefix, "%stitle" % prefix]
             )
 
+    def test_custom_lookup_to_related_model(self):
+        # In this test case the attribute of the fk model comes first in the
+        # list of search fields.
+        filter_ = filters.SearchFilter()
+        assert 'attribute__label__icontains' == filter_.construct_search('attribute__label', SearchFilterModelFk._meta)
+        assert 'attribute__label__iendswith' == filter_.construct_search('attribute__label__iendswith', SearchFilterModelFk._meta)
+
 
 class SearchFilterModelM2M(models.Model):
     title = models.CharField(max_length=20)

tests/test_model_serializer.py

@@ -315,7 +315,8 @@ class TestRegularFieldMappings(TestCase):
                 model = RegularFieldsModel
                 fields = ('auto_field', 'invalid')
 
-        expected = 'Field name `invalid` is not valid for model `RegularFieldsModel`.'
+        expected = 'Field name `invalid` is not valid for model `RegularFieldsModel` ' \
+                   'in `tests.test_model_serializer.TestSerializer`.'
         with self.assertRaisesMessage(ImproperlyConfigured, expected):
             TestSerializer().fields
 

tests/test_pagination.py

@@ -274,6 +274,7 @@ class TestPageNumberPagination:
 
         assert self.pagination.get_paginated_response_schema(unpaginated_schema) == {
             'type': 'object',
+            'required': ['count', 'results'],
             'properties': {
                 'count': {
                     'type': 'integer',
@@ -585,6 +586,7 @@ class TestLimitOffset:
 
         assert self.pagination.get_paginated_response_schema(unpaginated_schema) == {
             'type': 'object',
+            'required': ['count', 'results'],
             'properties': {
                 'count': {
                     'type': 'integer',
@@ -937,6 +939,7 @@ class CursorPaginationTestsMixin:
 
         assert self.pagination.get_paginated_response_schema(unpaginated_schema) == {
             'type': 'object',
+            'required': ['results'],
             'properties': {
                 'next': {
                     'type': 'string',

tests/test_serializer_lists.py

@@ -153,6 +153,61 @@ class TestListSerializerContainingNestedSerializer:
         assert serializer.is_valid()
         assert serializer.validated_data == expected_output
 
+    def test_update_allow_custom_child_validation(self):
+        """
+        Update a list of objects thanks custom run_child_validation implementation.
+        """
+
+        class TestUpdateSerializer(serializers.Serializer):
+            integer = serializers.IntegerField()
+            boolean = serializers.BooleanField()
+
+            def update(self, instance, validated_data):
+                instance._data.update(validated_data)
+                return instance
+
+            def validate(self, data):
+                # self.instance is set to current BasicObject instance
+                assert isinstance(self.instance, BasicObject)
+                # self.initial_data is current dictionary
+                assert isinstance(self.initial_data, dict)
+                assert self.initial_data["pk"] == self.instance.pk
+                return super().validate(data)
+
+        class ListUpdateSerializer(serializers.ListSerializer):
+            child = TestUpdateSerializer()
+
+            def run_child_validation(self, data):
+                # find related instance in self.instance list
+                child_instance = next(o for o in self.instance if o.pk == data["pk"])
+                # set instance and initial_data for child serializer
+                self.child.instance = child_instance
+                self.child.initial_data = data
+                return super().run_child_validation(data)
+
+            def update(self, instance, validated_data):
+                return [
+                    self.child.update(instance, attrs)
+                    for instance, attrs in zip(self.instance, validated_data)
+                ]
+
+        instance = [
+            BasicObject(pk=1, integer=11, private_field="a"),
+            BasicObject(pk=2, integer=22, private_field="b"),
+        ]
+        input_data = [
+            {"pk": 1, "integer": "123", "boolean": "true"},
+            {"pk": 2, "integer": "456", "boolean": "false"},
+        ]
+        expected_output = [
+            BasicObject(pk=1, integer=123, boolean=True, private_field="a"),
+            BasicObject(pk=2, integer=456, boolean=False, private_field="b"),
+        ]
+        serializer = ListUpdateSerializer(instance, data=input_data)
+        assert serializer.is_valid()
+        updated_instances = serializer.save()
+        assert updated_instances == expected_output
+
 
 class TestNestedListSerializer:
     """
@@ -481,7 +536,7 @@ class TestSerializerPartialUsage:
         assert serializer.validated_data == {}
         assert serializer.errors == {}
 
-    def test_udate_as_field_allow_empty_true(self):
+    def test_update_as_field_allow_empty_true(self):
         class ListSerializer(serializers.Serializer):
             update_field = serializers.IntegerField()
             store_field = serializers.IntegerField()