From 1817656adcc803b10d585baf15c7e58a6bb95af2 Mon Sep 17 00:00:00 2001 From: Alexandr Karpov Date: Sun, 23 Oct 2022 18:07:06 +0300 Subject: [PATCH] optimised search result --- app/search/api/serializers.py | 37 +++++++++++++++++++++++++++ app/search/api/views.py | 8 +++--- app/search/models.py | 8 ++++++ app/search/services/search/main.py | 33 +++++++++++++++++++++--- app/search/services/search/methods.py | 32 +++++++++++------------ app/search/services/search/prepare.py | 4 +-- app/search/services/spell_check.py | 2 +- 7 files changed, 96 insertions(+), 28 deletions(-) diff --git a/app/search/api/serializers.py b/app/search/api/serializers.py index e408e3a..291e571 100644 --- a/app/search/api/serializers.py +++ b/app/search/api/serializers.py @@ -1,6 +1,8 @@ from rest_framework import serializers from django.core.validators import MinLengthValidator, MinValueValidator +from search.models import Product, UnitCharacteristic, Characteristic + class QueryFilterSerializer(serializers.Serializer): value = serializers.CharField(max_length=100) @@ -86,3 +88,38 @@ class AutoCompleteResponseSerializer(serializers.Serializer): def update(self, instance, validated_data): raise NotImplementedError + + +class CharacteristicSerializer(serializers.ModelSerializer): + class Meta: + fields = ["name", "value"] + model = Characteristic + + +class UnitCharacteristicSerializer(serializers.ModelSerializer): + value = serializers.SerializerMethodField("get_value_n") + + def get_value_n(self, obj): + return obj.num_value + + class Meta: + fields = ["name", "value", "unit"] + model = UnitCharacteristic + + +class ProductSerializer(serializers.ModelSerializer): + characteristic = serializers.SerializerMethodField("get_characteristic_n") + + def get_characteristic_n(self, obj: Product): + return ( + CharacteristicSerializer( + Characteristic.objects.filter(products__product=obj), many=True + ).data + + UnitCharacteristicSerializer( + UnitCharacteristic.objects.filter(products__product=obj), many=True + ).data + ) + + class Meta: + fields = ["id", "name", "score", "characteristic"] + model = Product diff --git a/app/search/api/views.py b/app/search/api/views.py index bf8481c..41c294d 100644 --- a/app/search/api/views.py +++ b/app/search/api/views.py @@ -4,7 +4,7 @@ from rest_framework import status from rest_framework.generics import get_object_or_404 from rest_framework.response import Response from rest_framework.views import APIView -from search.api.serializers import HintRequestSerializer +from search.api.serializers import HintRequestSerializer, ProductSerializer from search.api.serializers import ( SearchSerializer, @@ -33,14 +33,14 @@ class SearchApi(APIView): serializer = SearchSerializer(data=request.data) serializer.is_valid(raise_exception=True) return Response( - group( + ProductSerializer( process_search( serializer.data["body"], serializer.data["limit"], serializer.data["offset"], ), - serializer.data["body"], - ), + many=True, + ).data, status=status.HTTP_200_OK, ) diff --git a/app/search/models.py b/app/search/models.py index b7107e6..13b90ad 100644 --- a/app/search/models.py +++ b/app/search/models.py @@ -34,6 +34,14 @@ class UnitCharacteristic(models.Model): "unit": self.unit, } + @property + def num_value(self): + return ( + self.numeric_value_min + if self.numeric_value_min == self.numeric_value_max + else f"{self.numeric_value_min}:{self.numeric_value_max}" + ) + class Meta: db_table = "unit_characteristic" diff --git a/app/search/services/search/main.py b/app/search/services/search/main.py index 5e61583..2a38035 100644 --- a/app/search/services/search/main.py +++ b/app/search/services/search/main.py @@ -10,9 +10,29 @@ from search.services.search.prepare import apply_union from search.models import Product -def process_search(data: List[dict], limit=5, offset=0) -> List[dict]: - prep_data = apply_union(data) - # ----------------------------------- apply filters on QuerySet -------------------------------------------------- # +def call(prep_data): + if len(prep_data) == 1: + typ = prep_data[0]["type"] + val = prep_data[0]["value"] + if typ == "Name": + return apply_qs_search(val).order_by("-score") + elif typ == "All": + return apply_all_qs_search(val).order_by("-score") + elif typ == "Category": + return Product.objects.filter(category__name__icontains=val).order_by( + "-score" + ) + elif typ == "Characteristic": + return appy_qs_characteristic(Product.objects.filter(), val).order_by( + "-score" + ) + elif typ == "Unknown": + return [] + else: + if typ.startswith("*"): + return Product.objects.filter(unit_characteristics__in=val) + else: + return Product.objects.filter(characteristics__in=val) qs = Product.objects.filter() for x in prep_data: typ = x["type"] @@ -35,4 +55,9 @@ def process_search(data: List[dict], limit=5, offset=0) -> List[dict]: qs = qs.filter(unit_characteristics__in=val) else: qs = qs.filter(characteristics__in=val) - return [x.serialize_self() for x in qs.distinct()[offset : offset + limit]] + return [] + + +def process_search(body: List[dict], limit=5, offset=0) -> List[dict]: + prep_data = apply_union(body) + return call(prep_data)[offset : offset + limit] diff --git a/app/search/services/search/methods.py b/app/search/services/search/methods.py index e9368cd..309aa46 100644 --- a/app/search/services/search/methods.py +++ b/app/search/services/search/methods.py @@ -1,8 +1,5 @@ -from functools import cache from typing import List -from django.utils.text import slugify - from search.models import ( Product, ProductCharacteristic, @@ -12,15 +9,20 @@ from search.services.spell_check import pos, spell_check def _clean_text(text: str) -> List[str]: + for st in [".", ",", "!", "?"]: text = text.replace(st, " ") + text = text.split() functors_pos = {"INTJ", "PRCL", "CONJ", "PREP"} # function words + text = [word for word in text if pos(word) not in functors_pos] - return [spell_check(x) for x in text] + + text = [spell_check(x) for x in text] + + return text -@cache def process_unit_operation(unit: ProductUnitCharacteristic.objects, operation: str): if operation.startswith("<=") or operation.startswith("=<"): return unit.filter( @@ -46,7 +48,6 @@ def process_unit_operation(unit: ProductUnitCharacteristic.objects, operation: s return unit -@cache def apply_qs_search(text: str): text = _clean_text(text) qs = Product.objects.filter() @@ -58,7 +59,6 @@ def apply_qs_search(text: str): return products -@cache def apply_all_qs_search(text: str): # words text = _clean_text(text) @@ -103,16 +103,22 @@ def apply_all_qs_search(text: str): del text[i] break - prod = Product.objects.filter() + if u_qs: + prod = Product.objects.filter(unit_characteristics__in=u_qs) + else: + prod = Product.objects.filter() + for word in text: car = ProductCharacteristic.objects.filter( characteristic__value__icontains=word, + ) | ProductCharacteristic.objects.filter( + characteristic__value__trigram_similar=word, ) qs = ( Product.objects.filter(name__icontains=word) - | Product.objects.filter(name__trigram_similar=word) - | Product.objects.filter(category__name__icontains=word) | Product.objects.filter(characteristics__in=car) + | Product.objects.filter(category__name__icontains=word) + | Product.objects.filter(name__trigram_similar=word) ) if any( x in word @@ -126,22 +132,16 @@ def apply_all_qs_search(text: str): ) ) ) - print(qs) prod = prod & qs - if u_qs: - prod = prod & Product.objects.filter(unit_characteristics__in=u_qs) - return prod -@cache def apply_qs_category(qs, name: str): qs = qs.filter(category__name__icontains=name) return qs -@cache def appy_qs_characteristic(qs, name: str): char = ProductCharacteristic.objects.filter(product__in=qs) char = char.filter(characteristic__value__icontains=name) | char.filter( diff --git a/app/search/services/search/prepare.py b/app/search/services/search/prepare.py index 22f601d..826810d 100644 --- a/app/search/services/search/prepare.py +++ b/app/search/services/search/prepare.py @@ -27,9 +27,7 @@ def apply_union(data: List[Dict]) -> List[Dict]: prep_data.append( { "type": dat["type"], - "value": spell_check( - dat["value"], - ), + "value": dat["value"], } ) elif x["type"] == "Unknown": diff --git a/app/search/services/spell_check.py b/app/search/services/spell_check.py index 00697b3..172559d 100644 --- a/app/search/services/spell_check.py +++ b/app/search/services/spell_check.py @@ -5,7 +5,7 @@ speller_ru = SpellChecker(language="ru") speller_eng = SpellChecker(language="en") -def spell_check_ru(word: str) -> str: +def spell_check(word: str) -> str: res = speller_ru.correction(word) if not res or not len(res): return word