Merge 8c97fceef7 into aff1cc0591

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.envs/.local/.django

@@ -7,6 +7,7 @@ DJANGO_READ_DOT_ENV_FILE=no
 # ------------------------------------------------------------------------------
 REDIS_URL=redis://redis:6379/1
 REDIS_CACHE=rediscache://redis:6379/1
+REDIS_CACHE_URL=redis://redis:6379/1
 CELERY_BROKER_URL=redis://redis:6379/0
 
 # Celery
@@ -15,3 +16,5 @@ CELERY_BROKER_URL=redis://redis:6379/0
 # Flower
 CELERY_FLOWER_USER=debug
 CELERY_FLOWER_PASSWORD=debug
+
+ELASTIC_SEARCH=http://elasticsearch:9200/

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Install poetry
         run: pipx install poetry
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
           cache: 'poetry'

.gitignore

@@ -1,4 +1,6 @@
 .idea
+django_setup.txt
+django_setup.prof
 
 ### Python template
 # Byte-compiled / optimized / DLL files

README.md

@@ -12,7 +12,6 @@ https://git.akarpov.ru/sanspie/akarpov
 ### installation
 ```shell
 $ poetry install & poetry shell
-$ ./spacy_setup.sh
 $ python3 manage.py migrate
 ```
 

akarpov/about/api/serializers.py

@@ -0,0 +1,5 @@
+from rest_framework import serializers
+
+
+class StatusSerializer(serializers.Serializer):
+    status = serializers.CharField(default="pong")

akarpov/about/api/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from akarpov.about.api.views import PingAPIView
+
+app_name = "about"
+
+urlpatterns = [path("ping", PingAPIView.as_view(), name="ping")]

akarpov/about/api/views.py

@@ -0,0 +1,11 @@
+from rest_framework import generics, permissions, response
+
+from akarpov.about.api.serializers import StatusSerializer
+
+
+class PingAPIView(generics.GenericAPIView):
+    serializer_class = StatusSerializer
+    permission_classes = [permissions.AllowAny]
+
+    def get(self, request):
+        return response.Response(data={"status": "pong"})

akarpov/blog/api/serializers.py

@@ -2,7 +2,7 @@
 from rest_framework import serializers
 
 from akarpov.blog.models import Comment, Post, Tag
-from akarpov.common.api import RecursiveField
+from akarpov.common.api.serializers import RecursiveField
 from akarpov.users.api.serializers import UserPublicInfoSerializer
 
 

akarpov/blog/api/views.py

@@ -9,7 +9,7 @@
 )
 from akarpov.blog.models import Post
 from akarpov.blog.services import get_main_rating_posts
-from akarpov.common.api import StandardResultsSetPagination
+from akarpov.common.api.pagination import StandardResultsSetPagination
 
 
 class ListMainPostsView(generics.ListAPIView):

akarpov/common/api.py

@@ -1,45 +0,0 @@
-from rest_framework import serializers
-from rest_framework.pagination import PageNumberPagination
-from rest_framework.permissions import SAFE_METHODS, BasePermission
-
-from akarpov.utils.models import get_object_user
-
-
-class SmallResultsSetPagination(PageNumberPagination):
-    page_size = 10
-    page_size_query_param = "page_size"
-    max_page_size = 100
-
-
-class StandardResultsSetPagination(PageNumberPagination):
-    page_size = 50
-    page_size_query_param = "page_size"
-    max_page_size = 200
-
-
-class BigResultsSetPagination(PageNumberPagination):
-    page_size = 100
-    page_size_query_param = "page_size"
-    max_page_size = 1000
-
-
-class RecursiveField(serializers.Serializer):
-    def to_representation(self, value):
-        serializer = self.parent.parent.__class__(value, context=self.context)
-        return serializer.data
-
-
-class IsCreatorOrReadOnly(BasePermission):
-    def has_permission(self, request, view):
-        return bool(
-            request.method in SAFE_METHODS
-            or request.user
-            and get_object_user(view.get_object()) == request.user
-        )
-
-
-class SetUserModelSerializer(serializers.ModelSerializer):
-    def create(self, validated_data):
-        creator = self.context["request"].user
-        obj = self.Meta.model.objects.create(creator=creator, **validated_data)
-        return obj

akarpov/common/api/pagination.py

@@ -0,0 +1,19 @@
+from rest_framework.pagination import PageNumberPagination
+
+
+class SmallResultsSetPagination(PageNumberPagination):
+    page_size = 10
+    page_size_query_param = "page_size"
+    max_page_size = 100
+
+
+class StandardResultsSetPagination(PageNumberPagination):
+    page_size = 50
+    page_size_query_param = "page_size"
+    max_page_size = 200
+
+
+class BigResultsSetPagination(PageNumberPagination):
+    page_size = 100
+    page_size_query_param = "page_size"
+    max_page_size = 1000

akarpov/common/api/permissions.py

@@ -0,0 +1,19 @@
+from rest_framework.permissions import SAFE_METHODS, BasePermission
+
+from akarpov.utils.models import get_object_user
+
+
+class IsCreatorOrReadOnly(BasePermission):
+    def has_permission(self, request, view):
+        return bool(
+            request.method in SAFE_METHODS
+            or request.user
+            and get_object_user(view.get_object()) == request.user
+        )
+
+
+class IsAdminOrReadOnly(BasePermission):
+    def has_permission(self, request, view):
+        return bool(
+            request.method in SAFE_METHODS or request.user and request.user.is_staff
+        )

akarpov/common/api/serializers.py

@@ -0,0 +1,14 @@
+from rest_framework import serializers
+
+
+class RecursiveField(serializers.Serializer):
+    def to_representation(self, value):
+        serializer = self.parent.parent.__class__(value, context=self.context)
+        return serializer.data
+
+
+class SetUserModelSerializer(serializers.ModelSerializer):
+    def create(self, validated_data):
+        creator = self.context["request"].user
+        obj = self.Meta.model.objects.create(creator=creator, **validated_data)
+        return obj

akarpov/common/views.py

@@ -11,7 +11,7 @@ class HasPermissions(SingleObjectMixin):
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         has_perm = False
-        if self.request.user.is_authentificated:
+        if self.request.user.is_authenticated:
             has_perm = self.object.user == self.request.user
         context["has_permissions"] = has_perm
         return context

akarpov/files/documents.py

@@ -0,0 +1,31 @@
+from django_elasticsearch_dsl import Document
+from django_elasticsearch_dsl.registries import registry
+
+from akarpov.files.models import File
+
+
+@registry.register_document
+class FileDocument(Document):
+    class Index:
+        name = "files"
+        settings = {"number_of_shards": 1, "number_of_replicas": 0}
+
+    class Django:
+        model = File
+        fields = [
+            "name",
+            "description",
+            "content",
+        ]
+
+    def prepare_description(self, instance):
+        # This method is called for every instance before indexing
+        return instance.description or ""
+
+    def prepare_content(self, instance):
+        # This method is called for every instance before indexing
+        return (
+            instance.content.decode("utf-8")
+            if isinstance(instance.content, bytes)
+            else instance.content
+        )

akarpov/files/migrations/0029_remove_file_embeddings_remove_file_lang_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.6 on 2023-11-06 21:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0028_file_content_file_lang"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="file",
+            name="embeddings",
+        ),
+        migrations.RemoveField(
+            model_name="file",
+            name="lang",
+        ),
+        migrations.AlterField(
+            model_name="file",
+            name="content",
+            field=models.TextField(),
+        ),
+    ]

akarpov/files/migrations/0030_auto_20231107_0023.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.6 on 2023-11-06 21:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0029_remove_file_embeddings_remove_file_lang_and_more"),
+    ]
+    operations = [
+        migrations.RunSQL(
+            sql="CREATE EXTENSION IF NOT EXISTS unaccent;",
+            reverse_sql="DROP EXTENSION unaccent;",
+        ),
+        migrations.RunSQL(
+            sql="CREATE EXTENSION IF NOT EXISTS pg_trgm;",
+            reverse_sql="DROP EXTENSION pg_trgm;",
+        ),
+    ]

akarpov/files/models.py

@@ -17,7 +17,6 @@
 from django.urls import reverse
 from model_utils.fields import AutoCreatedField, AutoLastModifiedField
 from model_utils.models import TimeStampedModel
-from pgvector.django import VectorField
 from polymorphic.models import PolymorphicModel
 
 from akarpov.files.services.files import trash_file_upload, user_unique_file_upload
@@ -70,9 +69,8 @@ class File(BaseFileItem, TimeStampedModel, ShortLinkModel, UserHistoryModel):
 
     preview = FileField(blank=True, upload_to="file/previews/")
     file_obj = FileField(blank=False, upload_to=user_unique_file_upload)
-    embeddings = VectorField(dimensions=768, null=True)
+    content = TextField()
-    content = TextField(max_length=10000)
+    # lang = CharField(max_length=2, choices=[("ru", "ru"), ("en", "en")])
-    lang = CharField(max_length=2, choices=[("ru", "ru"), ("en", "en")])
 
     # meta
     name = CharField(max_length=255, null=True, blank=True)
@@ -128,7 +126,7 @@ class Folder(BaseFileItem, ShortLinkModel, UserHistoryModel):
     amount = IntegerField(default=0)
 
     def get_last_preview_files(self, cut=4):
-        return self.children.filter(~Q(File___preview=""))[:cut]
+        return self.children.cache().filter(~Q(File___preview=""))[:cut]
 
     def get_absolute_url(self):
         return reverse("files:folder", kwargs={"slug": self.slug})

akarpov/files/services/lema.py

@@ -0,0 +1,42 @@
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from nltk.tokenize import word_tokenize
+from pymorphy3 import MorphAnalyzer
+
+# Set up stop words
+english_stopwords = set(stopwords.words("english"))
+russian_stopwords = set(stopwords.words("russian"))
+
+# Set up lemmatizers
+english_lemmatizer = None
+russian_lemmatizer = None
+
+
+def lemmatize_and_remove_stopwords(text, language="english"):
+    # Tokenize the text
+    global english_lemmatizer, russian_lemmatizer
+    tokens = word_tokenize(text)
+
+    # Lemmatize each token based on the specified language
+    lemmatized_tokens = []
+    for token in tokens:
+        if language == "russian":
+            if not russian_lemmatizer:
+                russian_lemmatizer = MorphAnalyzer()
+            lemmatized_token = russian_lemmatizer.parse(token)[0].normal_form
+        else:  # Default to English
+            if not english_lemmatizer:
+                english_lemmatizer = WordNetLemmatizer()
+            lemmatized_token = english_lemmatizer.lemmatize(token)
+        lemmatized_tokens.append(lemmatized_token)
+
+    # Remove stop words
+    filtered_tokens = [
+        token
+        for token in lemmatized_tokens
+        if token not in english_stopwords and token not in russian_stopwords
+    ]
+
+    # Reconstruct the text
+    filtered_text = " ".join(filtered_tokens)
+    return filtered_text

akarpov/files/services/preview.py

@@ -16,7 +16,7 @@
     "Consola.ttf",
 ]
 
-manager = PreviewManager(cache_path, create_folder=True)
+manager = None
 
 
 def textfile_to_image(textfile_path) -> Image:
@@ -79,7 +79,10 @@ def _font_points_to_pixels(pt):
 
 
 def create_preview(file_path: str) -> str:
+    global manager
     # TODO: add text image generation/code image
+    if not manager:
+        manager = PreviewManager(cache_path, create_folder=True)
     if manager.has_jpeg_preview(file_path):
         return manager.get_jpeg_preview(file_path, height=500)
     return ""
@@ -91,6 +94,10 @@ def get_file_mimetype(file_path: str) -> str:
 
 
 def get_description(file_path: str) -> str:
+    global manager
+    if not manager:
+        manager = PreviewManager(cache_path, create_folder=True)
+
     if manager.has_text_preview(file_path):
         return manager.get_text_preview(file_path)
     return ""

akarpov/files/services/search.py

@@ -0,0 +1,167 @@
+import os
+import re
+from typing import BinaryIO
+
+from django.conf import settings
+from django.contrib.postgres.search import TrigramSimilarity
+from django.db.models import Case, F, FloatField, Func, Q, QuerySet, Value, When
+from django.db.models.functions import Coalesce
+from elasticsearch_dsl import Q as ES_Q
+
+from akarpov.files.models import File
+
+from ..documents import FileDocument
+from .lema import lemmatize_and_remove_stopwords
+
+
+class BaseSearch:
+    def __init__(self, queryset: QuerySet | None = None):
+        self.queryset: QuerySet | None = queryset
+
+    def search(self, query: str) -> QuerySet | list[File]:
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+class NeuroSearch(BaseSearch):
+    def search(self, query: str):
+        if not self.queryset:
+            raise ValueError("Queryset cannot be None for search")
+
+        # Perform the Elasticsearch query using a combination of match, match_phrase_prefix, and wildcard queries
+        search = FileDocument.search()
+        search_query = ES_Q(
+            "bool",
+            should=[
+                ES_Q(
+                    "multi_match",
+                    query=query,
+                    fields=["name", "description", "content"],
+                    type="best_fields",
+                ),
+                ES_Q("match_phrase_prefix", name=query),
+                ES_Q("wildcard", name=f"*{query}*"),
+                ES_Q("wildcard", description=f"*{query}*"),
+                ES_Q("wildcard", content=f"*{query}*"),
+            ],
+            minimum_should_match=1,
+        )
+
+        search = search.query(search_query)
+
+        # Execute the search to get the results
+        response = search.execute()
+
+        # Check if there are hits, if not return an empty queryset
+        if not response.hits:
+            return self.queryset.none()
+
+        # Collect the IDs of the hits
+        hit_ids = [hit.meta.id for hit in response.hits]
+
+        # Use the hit IDs to filter the queryset and preserve the order
+        preserved_order = Case(
+            *[When(pk=pk, then=pos) for pos, pk in enumerate(hit_ids)]
+        )
+        relevant_queryset = self.queryset.filter(pk__in=hit_ids).order_by(
+            preserved_order
+        )
+
+        return relevant_queryset
+
+
+class CaseSensitiveSearch(BaseSearch):
+    def search(self, query: str) -> QuerySet[File]:
+        if self.queryset is None:
+            raise ValueError("Queryset cannot be None for text search")
+
+        # Escape any regex special characters in the query string
+        query_escaped = re.escape(query)
+
+        # Use a case-sensitive regex to filter
+        return self.queryset.filter(
+            Q(name__regex=query_escaped)
+            | Q(description__regex=query_escaped)
+            | Q(content__regex=query_escaped)
+        )
+
+
+class ByteSearch(BaseSearch):
+    def search(self, hex_query: str) -> list[File]:
+        # Convert the hex query to bytes
+        try:
+            byte_query: bytes = bytes.fromhex(hex_query)
+        except ValueError:
+            # If hex_query is not a valid hex, return an empty list
+            return []
+
+        matching_files: list[File] = []
+        if self.queryset is not None:
+            for file_item in self.queryset:
+                file_path: str = file_item.file.path
+                full_path: str = os.path.join(settings.MEDIA_ROOT, file_path)
+                if os.path.exists(full_path):
+                    with open(full_path, "rb") as file:
+                        if self._byte_search_in_file(file, byte_query):
+                            matching_files.append(file_item)
+        return matching_files
+
+    @staticmethod
+    def _byte_search_in_file(file: BinaryIO, byte_sequence: bytes) -> bool:
+        # Read the file in chunks to avoid loading large files into memory
+        chunk_size: int = 4096  # or another size depending on the expected file sizes
+        while True:
+            chunk: bytes = file.read(chunk_size)
+            if byte_sequence in chunk:
+                return True
+            if not chunk:  # End of file reached
+                return False
+
+
+class UnaccentLower(Func):
+    function = "UNACCENT"
+
+    def as_sql(self, compiler, connection):
+        unaccented_sql, unaccented_params = compiler.compile(
+            self.get_source_expressions()[0]
+        )
+        lower_unaccented_sql = f"LOWER({unaccented_sql})"
+        return lower_unaccented_sql, unaccented_params
+
+
+class SimilaritySearch(BaseSearch):
+    def search(self, query: str) -> QuerySet[File]:
+        if self.queryset is None:
+            raise ValueError("Queryset cannot be None for similarity search")
+
+        language = "russian" if re.search("[а-яА-Я]", query) else "english"
+        filtered_query = lemmatize_and_remove_stopwords(query, language=language)
+        queryset = (
+            self.queryset.annotate(
+                name_similarity=Coalesce(
+                    TrigramSimilarity(UnaccentLower("name"), filtered_query),
+                    Value(0),
+                    output_field=FloatField(),
+                ),
+                description_similarity=Coalesce(
+                    TrigramSimilarity(UnaccentLower("description"), filtered_query),
+                    Value(0),
+                    output_field=FloatField(),
+                ),
+                content_similarity=Coalesce(
+                    TrigramSimilarity(UnaccentLower("content"), filtered_query),
+                    Value(0),
+                    output_field=FloatField(),
+                ),
+            )
+            .annotate(
+                combined_similarity=(
+                    F("name_similarity")
+                    + F("description_similarity")
+                    + F("content_similarity")
+                )
+            )
+            .filter(combined_similarity__gt=0.1)
+            .order_by("-combined_similarity")
+        )
+
+        return queryset

akarpov/files/services/text.py

@@ -1,7 +1,18 @@
+import chardet
 import textract
+from textract.exceptions import ExtensionNotSupported
 
 
 def extract_file_text(file: str) -> str:
+    try:
         text = textract.process(file)
+    except ExtensionNotSupported:
+        try:
+            rawdata = open(file, "rb").read()
+            enc = chardet.detect(rawdata)
+            with open(file, encoding=enc["encoding"]) as f:
+                text = f.read()
+        except Exception:
+            return ""
 
     return text

akarpov/files/tasks.py

@@ -1,11 +1,14 @@
 import os
+import time
 
 import structlog
 from celery import shared_task
+from django.core import management
 from django.core.files import File
 
 from akarpov.files.models import File as FileModel
 from akarpov.files.services.preview import create_preview, get_file_mimetype
+from akarpov.files.services.text import extract_file_text
 
 logger = structlog.get_logger(__name__)
 
@@ -28,7 +31,17 @@ def process_file(pk: int):
     except Exception as e:
         logger.error(e)
     file.file_type = get_file_mimetype(file.file.path)
-    file.save(update_fields=["preview", "name", "file_type"])
+    file.content = extract_file_text(file.file.path)
+    file.save(update_fields=["preview", "name", "file_type", "content"])
     if pth and os.path.isfile(pth):
         os.remove(pth)
     return pk
+
+
+@shared_task
+def update_index_task():
+    start_time = time.time()
+    management.call_command("search_index", "--rebuild", "-f")
+    end_time = time.time()
+    duration = end_time - start_time
+    logger.info("update_index_completed", duration=duration)

akarpov/files/views.py

@@ -33,15 +33,44 @@
 from akarpov.files.previews import extensions, meta, meta_extensions, previews
 from akarpov.files.services.folders import delete_folder
 from akarpov.files.services.preview import get_base_meta
+from akarpov.files.services.search import (
+    ByteSearch,
+    CaseSensitiveSearch,
+    NeuroSearch,
+    SimilaritySearch,
+)
 from akarpov.files.tables import FileTable
 from akarpov.notifications.services import send_notification
 
 logger = structlog.get_logger(__name__)
 
+search_classes = {
+    "neuro": NeuroSearch,
+    "case_sensitive": CaseSensitiveSearch,
+    "byte_search": ByteSearch,
+    "similarity": SimilaritySearch,
+}
 
-class TopFolderView(LoginRequiredMixin, ListView):
+
+class FileFilterView(View):
+    def filter(self, queryset):
+        if "query" in self.request.GET and "search_type" in self.request.GET:
+            query = self.request.GET["query"]
+            search_type = self.request.GET["search_type"]
+            if not query or not self.request.user.is_authenticated:
+                return queryset
+
+            if search_type in search_classes:
+                search_instance = search_classes[search_type](
+                    queryset=File.objects.filter(user=self.request.user)
+                )
+                queryset = search_instance.search(query)
+        return queryset
+
+
+class TopFolderView(LoginRequiredMixin, ListView, FileFilterView):
     template_name = "files/list.html"
-    paginate_by = 18
+    paginate_by = 38
     model = BaseFileItem
 
     def get_context_data(self, **kwargs):
@@ -55,10 +84,18 @@ def get_context_data(self, **kwargs):
         return context
 
     def get_queryset(self):
-        return BaseFileItem.objects.filter(user=self.request.user, parent__isnull=True)
+        if (
+            "query" in self.request.GET
+            and "search_type" in self.request.GET
+            and self.request.GET["query"]
+        ):
+            return self.filter(BaseFileItem.objects.none())
+        return self.filter(
+            BaseFileItem.objects.filter(user=self.request.user, parent__isnull=True)
+        )
 
 
-class FileFolderView(ListView):
+class FileFolderView(ListView, FileFilterView):
     template_name = "files/folder.html"
     model = BaseFileItem
     paginate_by = 38
@@ -94,6 +131,13 @@ def get_object(self, *args):
 
     def get_queryset(self):
         folder = self.get_object()
+
+        if (
+            "query" in self.request.GET
+            and "search_type" in self.request.GET
+            and self.request.GET["query"]
+        ):
+            return self.filter(BaseFileItem.objects.none())
         return BaseFileItem.objects.filter(parent=folder)
 
 

akarpov/gallery/api/serializers.py

@@ -0,0 +1,24 @@
+from rest_framework import serializers
+
+from akarpov.gallery.models import Image
+
+
+class ImageSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Image
+        fields = (
+            "slug",
+            "image",
+            "collection",
+            "public",
+            "image_cropped",
+            "created",
+            "modified",
+        )
+        extra_kwargs = {
+            "slug": {"read_only": True},
+            "collection": {"write_only": True},
+            "image_cropped": {"read_only": True},
+            "created": {"read_only": True},
+            "modified": {"read_only": True},
+        }

akarpov/gallery/api/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+
+from . import views
+
+app_name = "gallery"
+
+urlpatterns = [
+    path("", views.ListCreateImageAPIView.as_view(), name="list-create-all"),
+]

akarpov/gallery/api/views.py

@@ -0,0 +1,21 @@
+from rest_framework import generics, parsers
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
+
+from akarpov.common.api.pagination import StandardResultsSetPagination
+from akarpov.gallery.api.serializers import ImageSerializer
+from akarpov.gallery.models import Image
+
+
+class ListCreateImageAPIView(generics.ListCreateAPIView):
+    serializer_class = ImageSerializer
+    permission_classes = [IsAuthenticatedOrReadOnly]
+    parser_classes = [parsers.MultiPartParser, parsers.FormParser]
+    pagination_class = StandardResultsSetPagination
+
+    def get_queryset(self):
+        if self.request.user.is_authenticated:
+            return self.request.user.images.all()
+        return Image.objects.filter(public=True)
+
+    def perform_create(self, serializer):
+        serializer.save(user=self.request.user)

akarpov/gallery/forms.py

@@ -0,0 +1,25 @@
+from django import forms
+
+from akarpov.gallery.models import Collection, Image
+
+
+class ImageUploadForm(forms.ModelForm):
+    class Meta:
+        model = Image
+        fields = (
+            "collection",
+            "public",
+        )
+
+    def __init__(self, *args, **kwargs):
+        user = kwargs.pop("user", None)
+        super().__init__(*args, **kwargs)
+        if user is not None:
+            self.fields["collection"].queryset = Collection.objects.filter(user=user)
+
+
+ImageFormSet = forms.modelformset_factory(
+    Image,
+    form=ImageUploadForm,
+    extra=3,  # Number of images to upload at once; adjust as needed
+)

akarpov/gallery/migrations/0004_image_public.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.6 on 2023-11-17 20:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("gallery", "0003_alter_collection_options_alter_image_options"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="image",
+            name="public",
+            field=models.BooleanField(default=False),
+        ),
+    ]

akarpov/gallery/models.py

@@ -20,6 +20,9 @@ class Collection(TimeStampedModel, ShortLinkModel, UserHistoryModel):
     def get_absolute_url(self):
         return reverse("gallery:collection", kwargs={"slug": self.slug})
 
+    def get_preview_images(self):
+        return self.images.cache().all()[:6]
+
     def __str__(self):
         return self.name
 
@@ -31,6 +34,7 @@ class Image(TimeStampedModel, ShortLinkModel, BaseImageModel, UserHistoryModel):
     collection = models.ForeignKey(
         "Collection", related_name="images", on_delete=models.CASCADE
     )
+    public = models.BooleanField(default=False)
     user = models.ForeignKey(
         "users.User", related_name="images", on_delete=models.CASCADE
     )

akarpov/gallery/urls.py

@@ -2,6 +2,7 @@
 
 from akarpov.gallery.views import (
     collection_view,
+    image_upload_view,
     image_view,
     list_collections_view,
     list_tag_images_view,
@@ -10,6 +11,7 @@
 app_name = "gallery"
 urlpatterns = [
     path("", list_collections_view, name="list"),
+    path("upload/", image_upload_view, name="upload"),
     path("<str:slug>", collection_view, name="collection"),
     path("tag/<str:slug>", list_tag_images_view, name="tag"),
     path("image/<str:slug>", image_view, name="view"),

akarpov/gallery/views.py

@@ -1,7 +1,9 @@
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.shortcuts import get_object_or_404
 from django.views import generic
 
 from akarpov.common.views import HasPermissions
+from akarpov.gallery.forms import ImageUploadForm
 from akarpov.gallery.models import Collection, Image, Tag
 
 
@@ -14,6 +16,17 @@ def get_queryset(self):
             return self.request.user.collections.all()
         return Collection.objects.filter(public=True)
 
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context["collection_previews"] = [
+            {
+                "collection": collection,
+                "preview_images": collection.get_preview_images(),
+            }
+            for collection in context["collection_list"]
+        ]
+        return context
+
 
 list_collections_view = ListCollectionsView.as_view()
 
@@ -46,3 +59,25 @@ class ImageView(generic.DetailView, HasPermissions):
 
 
 image_view = ImageView.as_view()
+
+
+class ImageUploadView(LoginRequiredMixin, generic.CreateView):
+    model = Image
+    form_class = ImageUploadForm
+    success_url = ""  # Replace with your success URL
+
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
+        kwargs["user"] = self.request.user
+        return kwargs
+
+    def post(self, request, *args, **kwargs):
+        form = self.get_form()
+        if form.is_valid():
+            form.save()
+            return self.form_valid(form)
+        else:
+            return self.form_invalid(form)
+
+
+image_upload_view = ImageUploadView.as_view()

akarpov/music/api/serializers.py

@@ -1,7 +1,7 @@
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
-from akarpov.common.api import SetUserModelSerializer
+from akarpov.common.api.serializers import SetUserModelSerializer
 from akarpov.music.models import Album, Author, Playlist, Song
 from akarpov.users.api.serializers import UserPublicInfoSerializer
 
@@ -56,6 +56,10 @@ class Meta:
 
 class ListSongSerializer(SetUserModelSerializer):
     album = serializers.CharField(source="album.name", read_only=True)
+    liked = serializers.SerializerMethodField(method_name="get_liked")
+
+    def get_liked(self, obj):
+        return obj.id in self.context["likes_ids"]
 
     class Meta:
         model = Song

akarpov/music/api/views.py

@@ -1,13 +1,28 @@
 from rest_framework import generics, permissions
 
-from akarpov.common.api import IsCreatorOrReadOnly
+from akarpov.common.api.pagination import StandardResultsSetPagination
+from akarpov.common.api.permissions import IsAdminOrReadOnly, IsCreatorOrReadOnly
 from akarpov.music.api.serializers import (
     FullPlaylistSerializer,
     ListSongSerializer,
     PlaylistSerializer,
     SongSerializer,
 )
-from akarpov.music.models import Playlist, Song
+from akarpov.music.models import Playlist, Song, SongUserRating
+
+
+class LikedSongsContextMixin(generics.GenericAPIView):
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.user.is_authenticated:
+            context["likes_ids"] = (
+                SongUserRating.objects.cache()
+                .filter(user=self.request.user, like=True)
+                .values_list("song_id", flat=True)
+            )
+        else:
+            context["likes_ids"] = []
+        return context
 
 
 class ListCreatePlaylistAPIView(generics.ListCreateAPIView):
@@ -18,7 +33,9 @@ def get_queryset(self):
         return Playlist.objects.filter(creator=self.request.user)
 
 
-class RetrieveUpdateDestroyPlaylistAPIView(generics.RetrieveUpdateDestroyAPIView):
+class RetrieveUpdateDestroyPlaylistAPIView(
+    LikedSongsContextMixin, generics.RetrieveUpdateDestroyAPIView
+):
     lookup_field = "slug"
     lookup_url_kwarg = "slug"
     permission_classes = [IsCreatorOrReadOnly]
@@ -34,12 +51,23 @@ def get_object(self):
         return self.object
 
 
-class ListCreateSongAPIView(generics.ListCreateAPIView):
+class ListCreateSongAPIView(LikedSongsContextMixin, generics.ListCreateAPIView):
     serializer_class = ListSongSerializer
-    permission_classes = [IsCreatorOrReadOnly]
+    permission_classes = [IsAdminOrReadOnly]
+    pagination_class = StandardResultsSetPagination
 
     def get_queryset(self):
-        return Song.objects.all()
+        if self.request.user.is_authenticated:
+            return (
+                Song.objects.exclude(
+                    id__in=SongUserRating.objects.filter(
+                        user=self.request.user
+                    ).values_list("song_id", flat=True)
+                )
+                .prefetch_related("authors")
+                .select_related("album")
+            )
+        return Song.objects.all().prefetch_related("authors").select_related("album")
 
 
 class RetrieveUpdateDestroySongAPIView(generics.RetrieveUpdateDestroyAPIView):
@@ -56,3 +84,21 @@ def get_object(self):
         if not self.object:
             self.object = super().get_object()
         return self.object
+
+
+class ListLikedSongsAPIView(generics.ListAPIView):
+    serializer_class = ListSongSerializer
+    pagination_class = StandardResultsSetPagination
+    authentication_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        return (
+            Song.objects.cache()
+            .filter(
+                id__in=self.request.user.song_likes.objects.cache()
+                .all()
+                .values_list("song_id", flat=True)
+            )
+            .prefetch_related("authors")
+            .select_related("album")
+        )

akarpov/music/migrations/0010_song_likes_songuserrating.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.7 on 2023-12-07 22:36
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("music", "0009_alter_songinque_name_alter_songinque_status"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="song",
+            name="likes",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.CreateModel(
+            name="SongUserRating",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("like", models.BooleanField(default=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                (
+                    "song",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="user_likes",
+                        to="music.song",
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="song_likes",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["-created"],
+                "unique_together": {("song", "user")},
+            },
+        ),
+    ]

akarpov/music/models.py

@@ -43,6 +43,7 @@ class Song(BaseImageModel, ShortLinkModel):
         "users.User", related_name="songs", on_delete=models.SET_NULL, null=True
     )
     meta = models.JSONField(blank=True, null=True)
+    likes = models.IntegerField(default=0)
 
     def get_absolute_url(self):
         return reverse("music:song", kwargs={"slug": self.slug})
@@ -128,3 +129,21 @@ class RadioSong(models.Model):
     start = models.DateTimeField(auto_now=True)
     slug = models.SlugField(unique=True)
     song = models.ForeignKey("Song", related_name="radio", on_delete=models.CASCADE)
+
+
+class SongUserRating(models.Model):
+    song = models.ForeignKey(
+        "Song", related_name="user_likes", on_delete=models.PROTECT
+    )
+    user = models.ForeignKey(
+        "users.User", related_name="song_likes", on_delete=models.CASCADE
+    )
+    like = models.BooleanField(default=True)
+    created = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return f"{self.user} {self.song} {'like' if self.like else 'dislike'}"
+
+    class Meta:
+        unique_together = ["song", "user"]
+        ordering = ["-created"]

akarpov/music/services/youtube.py

@@ -9,7 +9,6 @@
 from PIL import Image
 from pydub import AudioSegment
 from pytube import Search, YouTube
-from yt_dlp import YoutubeDL
 
 from akarpov.music.models import Song
 from akarpov.music.services.db import load_track
@@ -67,7 +66,7 @@ def parse_description(description: str) -> list:
 def download_from_youtube_link(link: str, user_id: int) -> Song:
     song = None
 
-    with YoutubeDL(ydl_opts) as ydl:
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
         info_dict = ydl.extract_info(link, download=False)
         title = info_dict.get("title", None)
         description = info_dict.get("description", None)

akarpov/music/signals.py

@@ -1,9 +1,9 @@
 import os
 
-from django.db.models.signals import post_delete, post_save
+from django.db.models.signals import post_delete, post_save, pre_save
 from django.dispatch import receiver
 
-from akarpov.music.models import Song
+from akarpov.music.models import Song, SongUserRating
 
 
 @receiver(post_delete, sender=Song)
@@ -16,3 +16,21 @@ def auto_delete_file_on_delete(sender, instance, **kwargs):
 @receiver(post_save)
 def send_que_status(sender, instance, created, **kwargs):
     ...
+
+
+@receiver(pre_save, sender=SongUserRating)
+def create_or_update_rating(sender, instance: SongUserRating, **kwargs):
+    song = instance.song
+    if instance.pk:
+        previous = SongUserRating.objects.get(pk=instance.pk)
+        if previous.like != instance.like:
+            if instance.like:
+                song.likes += 2
+            else:
+                song.likes -= 2
+    else:
+        if instance.like:
+            song.likes += 1
+        else:
+            song.likes -= 1
+    song.save(update_fields=["likes"])

akarpov/notifications/providers/site/api/views.py

@@ -1,6 +1,6 @@
 from rest_framework import generics, permissions
 
-from akarpov.common.api import StandardResultsSetPagination
+from akarpov.common.api.pagination import StandardResultsSetPagination
 from akarpov.notifications.models import Notification
 from akarpov.notifications.providers.site.api.serializers import (
     SiteNotificationSerializer,

akarpov/static/css/project.css

@@ -531,3 +531,37 @@ p {
 .nav-active {
   color: white;
 }
+
+.username {
+    max-width: 120px; /* Adjust as needed */
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+}
+
+.username:hover {
+    text-decoration: underline;
+    cursor: pointer;
+}
+
+/* Tooltip CSS */
+.username:hover::after {
+    content: attr(title);
+    position: absolute;
+    bottom: -20px;
+    left: 50%;
+    transform: translateX(-50%);
+    background-color: black;
+    color: white;
+    padding: 5px;
+    border-radius: 5px;
+    font-size: smaller;
+    white-space: nowrap;
+}
+
+/* Responsive font size */
+@media (max-width: 600px) {
+    .username {
+        font-size: smaller;
+    }
+}

akarpov/static/js/project.js

@@ -1 +1,29 @@
-/* Project specific Javascript goes here. */
+
+      function sleep(ms) {
+          return new Promise(resolve => setTimeout(resolve, ms));
+      }
+
+      function timeSince(date) {
+            let seconds = Math.floor((new Date() - date) / 1000);
+            let interval = seconds / 31536000;
+            if (interval > 1) {
+              return Math.floor(interval) + " years";
+            }
+            interval = seconds / 2592000;
+            if (interval > 1) {
+              return Math.floor(interval) + " months";
+            }
+            interval = seconds / 86400;
+            if (interval > 1) {
+              return Math.floor(interval) + " days";
+            }
+            interval = seconds / 3600;
+            if (interval > 1) {
+              return Math.floor(interval) + " hours";
+            }
+            interval = seconds / 60;
+            if (interval > 1) {
+              return Math.floor(interval) + " minutes";
+            }
+            return Math.floor(seconds) + " seconds";
+      }

akarpov/static/js/ws_script.js

@@ -0,0 +1,177 @@
+// Determine the correct WebSocket protocol to use
+const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+const host = window.location.host; // Assuming host includes the port if needed
+const socketPath = `${protocol}//${host}/ws/radio/`;
+
+let socket = new WebSocket(socketPath);
+
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+// Your existing timeSince function can remain unchanged
+
+// Update the onmessage handler here as needed for the music player
+socket.onmessage = function(event) {
+    let data = JSON.parse(event.data);
+    // Process the data
+};
+
+// This function will attempt to reconnect the WebSocket after a 5-second delay
+async function reconnectSocket() {
+    console.log("Radio socket disconnected, reconnecting...");
+    let socketClosed = true;
+    await sleep(5000); // Wait 5 seconds before attempting to reconnect
+    while (socketClosed) {
+        try {
+            socket = new WebSocket(socketPath);
+            socket.onmessage = onMessageHandler; // Reassign your onmessage handler
+            socket.onclose = onSocketCloseHandler; // Reassign the onclose handler to this function
+            socketClosed = false; // Exit the loop if the connection is successful
+        } catch (e) {
+            console.log("Can't connect to socket, retrying in 1 second...");
+            await sleep(1000); // Wait 1 second before retrying
+        }
+    }
+}
+
+// Define a separate function for the onclose event
+const onSocketCloseHandler = async function(event) {
+    if (!event.wasClean) {
+        // Only attempt to reconnect if the socket did not close cleanly
+        await reconnectSocket();
+    }
+};
+
+// Assign the onclose handler to the socket
+socket.onclose = onSocketCloseHandler;
+let isAdmin = false; // This will be set based on socket connect data for admin TODO: retrieve from server
+
+
+
+function displaySong(data) {
+  // Display song details using jQuery to manipulate the DOM
+  $('#song-info').html(/* HTML structure for song info */);
+  // Update admin controls if isAdmin is true
+  if (isAdmin) {
+    $('#admin-controls').show();
+    // Add slider and other controls
+  }
+  // Add song to history
+  addToHistory(data);
+}
+
+function displayAdminControls(data) {
+  // Display admin controls if user is admin
+  if (isAdmin) {
+    $('#admin-controls').html(/* HTML structure for admin controls */);
+  }
+}
+
+function updateSong(data) {
+  // Update the song information on the page
+  displaySong(data);
+}
+
+function addToHistory(songData) {
+  var authors = songData.authors.map(author => author.name).join(", ");
+  var historyItemHtml = `
+    <div class="card mb-3" style="max-width: 540px;">
+        <div class="row g-0">
+            <div class="col-md-4">
+                <img src="${songData.image}" class="img-fluid rounded-start" alt="${songData.name}">
+            </div>
+            <div class="col-md-8">
+                <div class="card-body">
+                    <h5 class="card-title">${songData.name}</h5>
+                    <p class="card-text"><small class="text-muted">by ${authors}</small></p>
+                    <p class="card-text">${songData.album.name}</p>
+                    <!-- Admin controls would go here if needed -->
+                </div>
+            </div>
+        </div>
+    </div>
+  `;
+  $('#history').prepend(historyItemHtml); // prepend to add it to the top of the list
+}
+
+// Functionality to handle sliding track length and selecting track to play
+// ...
+
+// ws_script.js
+
+$(document).ready(function() {
+    var audioContext;
+    var analyser;
+    var started = false; // Flag to indicate if the audio context has started
+    var audio = document.getElementById('audio-player');
+    var canvas = document.getElementById('audio-visualization');
+    var ctx = canvas.getContext('2d');
+
+    // Function to initialize the AudioContext and analyser
+    function initAudioContext() {
+        audioContext = new (window.AudioContext || window.webkitAudioContext)();
+        analyser = audioContext.createAnalyser();
+        var audioSrc = audioContext.createMediaElementSource(audio);
+        audioSrc.connect(analyser);
+        analyser.connect(audioContext.destination);
+    }
+
+    // Function to start or resume the audio context on user interaction
+    function startOrResumeContext() {
+        if (audioContext.state === 'suspended') {
+            audioContext.resume().then(() => {
+                console.log('Playback resumed successfully');
+            });
+        }
+    }
+
+    // Volume control listener
+    $('#volume-control').on('input change', function() {
+        if (audio) {
+            audio.volume = $(this).val() / 100;
+        }
+    });
+
+    // Listener for play/pause button
+    $('#play-pause-button').on('click', function() {
+        if (!started) {
+            initAudioContext();
+            started = true;
+        }
+
+        startOrResumeContext(); // Resume the audio context if needed
+
+        if (audio.paused) {
+            audio.play().then(() => {
+                console.log('Audio playing');
+            }).catch((e) => {
+                console.error('Error playing audio:', e);
+            });
+        } else {
+            audio.pause();
+            console.log('Audio paused');
+        }
+    });
+
+    function renderFrame() {
+        if (!analyser) return;
+
+        requestAnimationFrame(renderFrame);
+        var fbc_array = new Uint8Array(analyser.frequencyBinCount);
+        analyser.getByteFrequencyData(fbc_array);
+        ctx.clearRect(0, 0, canvas.width, canvas.height); // Clear the canvas
+        // Visualization code...
+    }
+
+    // Error event listener for the audio element
+    audio.addEventListener('error', function(e) {
+        console.error('Error with audio playback:', e);
+    });
+
+    // Attempt to play the audio and start visualization when the audio can play through
+    audio.addEventListener('canplaythrough', function() {
+        startOrResumeContext();
+        renderFrame(); // Start the visualizer
+    });
+});

akarpov/templates/base.html

@@ -22,9 +22,7 @@
     <!-- Latest compiled and minified Bootstrap CSS -->
     <link href="{% static 'css/bootstrap.min.css' %}" rel="stylesheet">
     {% if request.user.is_authenticated %}
-      {% if request.user.theme %}
+      <link href="{{ request.user.get_theme_url }}" rel="stylesheet">
-        <link href="{{ request.user.theme.file.url }}" rel="stylesheet">
-      {% endif %}
     {% endif %}
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.2/font/bootstrap-icons.css">
     <script src="https://kit.fontawesome.com/32fd82c823.js" crossorigin="anonymous"></script>
@@ -70,12 +68,13 @@
                                 <i class="fs-5 bi-folder-fill"></i><span class="ms-1 d-none d-sm-inline">Files</span></a>
                         </li>
                         {% endif %}
-                        <li class="dropdown">
+                        <li class="nav-item dropdown">
-                            <a href="#" class="text-muted nav-link dropdown-toggle px-sm-0 px-1" id="dropdown" data-bs-toggle="dropdown" aria-expanded="false">
+                            <a class="nav-link dropdown-toggle text-muted px-sm-0 px-1" href="#" id="navbarDropdownMenuLink" role="button" data-bs-toggle="dropdown" aria-expanded="false">
                                 <i class="fs-5 bi-terminal-fill"></i><span class="ms-1 d-none d-sm-inline">Apps</span>
                             </a>
-                            <ul class="dropdown-menu dropdown-menu-dark text-small shadow" aria-labelledby="dropdown">
+                            <ul class="dropdown-menu dropdown-menu-dark text-small shadow" aria-labelledby="navbarDropdownMenuLink">
                                 <li><a class="dropdown-item {% active_link 'tools:qr:create' %}" href="{% url 'tools:qr:create' %}">QR generator</a></li>
+                                <li><a class="dropdown-item {% active_link 'tools:uuid:main' %}" href="{% url 'tools:uuid:main' %}">UUID tools</a></li>
                                 <li><a class="dropdown-item {% active_link 'tools:shortener:create' %}" href="{% url 'tools:shortener:create' %}">URL shortcuter</a></li>
                             </ul>
                         </li>
@@ -89,7 +88,7 @@
                   {% if request.user.is_authenticated %}
                         <a href="#" class="d-flex align-items-center text-white text-decoration-none dropdown-toggle" id="dropdownUser1" data-bs-toggle="dropdown" aria-expanded="false">
                             {% if request.user.image_cropped %}<img src="{{ request.user.image_cropped.url }}" alt="hugenerd" width="28" height="28" class="rounded-circle">{% endif %}
-                            <span class="d-none d-sm-inline mx-1">{{ request.user.username }}</span>
+                            <span class="d-none d-sm-inline mx-1 username" title="{{ request.user.username }}">{{ request.user.username }}</span>
                         </a>
                         <ul class="dropdown-menu dropdown-menu-dark text-small shadow" aria-labelledby="dropdownUser1">
                             <li><a class="dropdown-item {% active_link 'users:update' %}" href="{% url 'users:update'  %}">Settings</a></li>
@@ -151,40 +150,11 @@
     {% endblock inline_javascript %}
     {% if request.user.is_authenticated %}
     <script>
-      {% if request.is_secure %}
+      const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
-      let socket = new WebSocket(`wss://{{ request.get_host }}/ws/notifications/`);
+      const host = window.location.host; // Assuming host includes the port if needed
-      {% else %}
+      const socketPath = `${protocol}//${host}/ws/notifications/`;
-      let socket = new WebSocket(`ws://{{ request.get_host }}/ws/notifications/`);
-      {% endif %}
 
-      function sleep(ms) {
+      let notification_socket = new WebSocket(socketPath);
-          return new Promise(resolve => setTimeout(resolve, ms));
-      }
-
-      function timeSince(date) {
-            let seconds = Math.floor((new Date() - date) / 1000);
-            let interval = seconds / 31536000;
-            if (interval > 1) {
-              return Math.floor(interval) + " years";
-            }
-            interval = seconds / 2592000;
-            if (interval > 1) {
-              return Math.floor(interval) + " months";
-            }
-            interval = seconds / 86400;
-            if (interval > 1) {
-              return Math.floor(interval) + " days";
-            }
-            interval = seconds / 3600;
-            if (interval > 1) {
-              return Math.floor(interval) + " hours";
-            }
-            interval = seconds / 60;
-            if (interval > 1) {
-              return Math.floor(interval) + " minutes";
-            }
-            return Math.floor(seconds) + " seconds";
-      }
 
       const toastContainer = document.getElementById('toastContainer')
 
@@ -210,18 +180,17 @@
         toastBootstrap.show()
       }
 
-      socket.onmessage = fn
+      notification_socket.onmessage = fn
-      socket.onclose = async function(event) {
+      notification_socket.onclose = async function(event) {
         console.log("Notifications socket disconnected, reconnecting...")
         let socketClosed = true;
         await sleep(5000)
         while (socketClosed) {
-          {# TODO: reconnect socket here #}
           try {
-            let cl = socket.onclose
+            let cl = notification_socket.onclose
-            socket = new WebSocket(`ws://127.0.0.1:8000/ws/notifications/`);
+            notification_socket = new WebSocket(socketPath);
-            socket.onmessage = fn
+            notification_socket.onmessage = fn
-            socket.onclose = cl
+            notification_socket.onclose = cl
             socketClosed = false
           } catch (e) {
             console.log("Can't connect to socket, reconnecting...")

akarpov/templates/files/list.html

@@ -15,6 +15,7 @@
 {% endblock %}
 
 {% block css %}
+  <link href=" https://cdn.jsdelivr.net/npm/bootstrap-select@1.13.18/dist/css/bootstrap-select.min.css" rel="stylesheet">
 <style>
 .row {
   display: -webkit-box;
@@ -62,6 +63,29 @@
   </div>
   {% endif %}
   </div>
+  <form id="searchForm" class="row" method="get">
+    <div class="col-lg-9 col-md-8 col-sm-7">
+      <input type="text" class="form-control" placeholder="Search..." name="query" aria-label="Search" value="{{ request.GET.query|default_if_none:'' }}">
+    </div>
+    <div class="col-lg-2 col-md-3 col-sm-4">
+{#      <select class="selectpicker form-select" name="search_type" title="Choose...">#}
+{#        <option data-icon="bi bi-brain" value="neuro" {% if request.GET.search_type == "neuro" %}selected{% endif %}>Neuro Search</option>#}
+{#        <option data-icon="bi bi-textarea-t" value="case_sensitive" {% if request.GET.search_type == "case_sensitive" %}selected{% endif %}>Case Sensitive</option>#}
+{#        <option data-icon="bi bi-file-earmark-binary" value="byte_search" {% if request.GET.search_type == "byte_search" %}selected{% endif %}>Byte Search</option>#}
+{#        <option data-icon="bi bi-stars" value="similarity" {% if request.GET.search_type == "similarity" %}selected{% endif %}>Similarity Search</option>#}
+{#      </select>#}
+    <select name="search_type" class="form-select" id="inlineFormSelectPref">
+        <option data-icon="bi bi-brain" value="neuro" {% if request.GET.search_type == "neuro" %}selected{% endif %}>Neuro Search</option>
+        <option data-icon="bi bi-textarea-t" value="case_sensitive" {% if request.GET.search_type == "case_sensitive" %}selected{% endif %}>Case Sensitive</option>
+        <option data-icon="bi bi-file-earmark-binary" value="byte_search" {% if request.GET.search_type == "byte_search" %}selected{% endif %}>Byte Search</option>
+        <option data-icon="bi bi-stars" value="similarity" {% if request.GET.search_type == "similarity" %}selected{% endif %}>Similarity Search</option>
+    </select>
+
+    </div>
+    <div class="col-lg-1 col-md-1 col-sm-2">
+      <button type="submit" class="btn btn-primary w-100"><i class="bi bi-search"></i> Search</button>
+    </div>
+  </form>
   <div class="row">
   {% if request.user.is_authenticated and is_folder_owner %}
   <div class="col-lg-2 col-xxl-2 col-md-4 col-sm-6 col-xs-12 mb-3 m-3 d-flex align-items-stretch card">
@@ -108,7 +132,7 @@
     </div>
   </div>
   {% endif %}
-  {% for file in basefileitem_list %}
+  {% for file in object_list %}
     <div class="col-lg-2 col-xxl-2 col-md-4 col-sm-6 col-xs-12 mb-3 m-3 d-flex align-items-stretch card justify-content-center">
         {% if file.is_file %}
         <div class="card-body d-flex flex-column">
@@ -172,7 +196,12 @@
 {% endblock %}
 
 {% block inline_javascript %}
+  <script src="https://cdn.jsdelivr.net/npm/bootstrap-select@1.13.18/js/bootstrap-select.min.js"></script>
   <script type="text/javascript">
+  $(function () {
+      $('selectpicker').selectpicker();
+  });
+
     $.notify.defaults(
       {
         // whether to hide the notification on click
@@ -225,7 +254,7 @@
         } else {
           md5 = spark.end();
         }
-      };
+      }
 
       function read_next_chunk() {
         var reader = new FileReader();

akarpov/templates/gallery/collection.html

@@ -1 +1,36 @@
 {% extends 'base.html' %}
+
+{% block css %}
+<style>
+    .gallery {
+        display: flex;
+        flex-wrap: wrap;
+        justify-content: start; /* Align items to the start of the container */
+        align-items: stretch; /* Stretch items to fill the container */
+    }
+
+    .gallery-item {
+        /* Adjust the margin as needed */
+        margin: 5px;
+        flex-grow: 1;
+        flex-shrink: 1;
+        flex-basis: auto; /* Automatically adjust the basis based on the content size */
+    }
+
+    .gallery-item img {
+        width: 100%; /* Make image responsive */
+        height: auto; /* Maintain aspect ratio */
+        display: block; /* Remove bottom space under the image */
+    }
+</style>
+{% endblock %}
+
+{% block content %}
+<div class="gallery">
+    {% for image in object_list %}
+    <div class="gallery-item">
+        <img src="{{ image.url }}" alt="Image">
+    </div>
+    {% endfor %}
+</div>
+{% endblock %}

akarpov/templates/gallery/image_form.html

@@ -0,0 +1,150 @@
+{% extends 'base.html' %}
+{% load static %}
+{% load crispy_forms_tags %}
+
+{% block content %}
+<form method="post" action="{% url 'api:gallery:list-create-all' %}" enctype="multipart/form-data">
+    {% csrf_token %}
+    {{ form.management_form }}
+    {% for field in form %}
+      {{ field | as_crispy_field }}
+    {% endfor %}
+      <input type="file" id="imageInput" multiple accept="image/*" class="form-control" />
+      <div class="preview mt-3" id="preview"></div>
+      <button type="submit">Upload</button>
+</form>
+
+<div id="progress-bar" style="width:0; height:20px; background:green;"></div>
+{% endblock %}
+
+{% block javascript %}
+<script src="{% static 'js/jquery.min.js' %}"></script>
+<script>
+document.addEventListener('DOMContentLoaded', function () {
+    let form = document.querySelector('form');
+    let imageInput = document.getElementById('imageInput');
+    const preview = document.getElementById('preview');
+    let currentFiles = [];
+
+    imageInput.addEventListener('change', function () {
+        updateFiles(this.files);
+        updatePreview();
+    });
+
+    function updateFiles(newFiles) {
+        currentFiles = Array.from(newFiles);
+    }
+
+    function updatePreview() {
+        preview.innerHTML = '';
+        currentFiles.forEach((file, index) => {
+            let imgContainer = document.createElement('div');
+            imgContainer.classList.add('position-relative', 'd-inline-block', 'm-2');
+            imgContainer.setAttribute('data-index', index); // Set the data-index attribute
+
+            let img = document.createElement('img');
+            img.classList.add('img-thumbnail');
+            img.style.width = '150px';
+            img.style.height = '150px';
+            imgContainer.appendChild(img);
+
+            let deleteButton = document.createElement('span');
+            deleteButton.classList.add('material-icons', 'position-absolute', 'top-0', 'end-0', 'btn', 'btn-danger');
+            deleteButton.innerText = 'delete';
+            deleteButton.style.cursor = 'pointer';
+            deleteButton.onclick = function () {
+                currentFiles.splice(index, 1);
+                updatePreview();
+                imageInput.value = "";
+            };
+            imgContainer.appendChild(deleteButton);
+
+            let reader = new FileReader();
+            reader.onload = (function (aImg) {
+                return function (e) {
+                    aImg.src = e.target.result;
+                };
+            })(img);
+            reader.readAsDataURL(file);
+
+            preview.appendChild(imgContainer);
+        });
+    }
+
+    function createProgressBar(index) {
+        const progressBarContainer = document.createElement('div');
+        progressBarContainer.classList.add('progress', 'mb-2');
+        progressBarContainer.setAttribute('id', 'progress-container-' + index);
+
+        const progressBar = document.createElement('div');
+        progressBar.id = 'progress-bar-' + index;
+        progressBar.classList.add('progress-bar');
+        progressBar.setAttribute('role', 'progressbar');
+        progressBar.setAttribute('aria-valuenow', '0');
+        progressBar.setAttribute('aria-valuemin', '0');
+        progressBar.setAttribute('aria-valuemax', '100');
+        progressBar.style.width = '0%';
+        progressBar.style.height = '20px'; // Set the height of the progress bar
+
+        progressBarContainer.appendChild(progressBar);
+        form.appendChild(progressBarContainer); // Append the progress bar container to the form
+    }
+
+    form.addEventListener('submit', function (e) {
+        e.preventDefault();
+
+        // Get the CSRF token from the hidden input
+        const csrfToken = document.querySelector('input[name="csrfmiddlewaretoken"]').value;
+
+        // Clear previous progress bars if any
+        document.querySelectorAll('.progress').forEach(function(progressBar) {
+            progressBar.remove();
+        });
+
+        // Create a new progress bar for each file
+        currentFiles.forEach((file, index) => createProgressBar(index));
+
+        // Perform the upload for each file
+        currentFiles.forEach((file, index) => {
+            const formData = new FormData();
+            formData.append('image', file);
+            formData.append('collection', document.querySelector('#id_collection').value);
+            formData.append('public', document.querySelector('#id_public').checked);
+
+            const xhr = new XMLHttpRequest();
+            xhr.open('POST', form.action, true);
+            xhr.setRequestHeader('X-CSRFToken', csrfToken);
+
+            xhr.upload.onprogress = function (e) {
+                if (e.lengthComputable) {
+                    const progressBar = document.getElementById('progress-bar-' + index);
+                    const percentage = (e.loaded / e.total) * 100;
+                    progressBar.style.width = percentage + '%';
+                }
+            };
+
+            xhr.onload = function () {
+                if (this.status === 200 || this.status === 201) {
+                    console.log('Upload complete for file', index);
+                    // Remove the image preview and progress bar for the uploaded file
+                    const imgPreview = preview.querySelector(`div[data-index="${index}"]`);
+                    if (imgPreview) {
+                        preview.removeChild(imgPreview);
+                    }
+                    const progressBarContainer = document.getElementById('progress-container-' + index);
+                    if (progressBarContainer) {
+                        progressBarContainer.remove();
+                    }
+                    // Remove the file from the currentFiles array
+                    currentFiles = currentFiles.filter((_, i) => i !== index);
+                } else {
+                    console.error('Upload failed for file', index, ':', this.responseText);
+                }
+            };
+
+            xhr.send(formData);
+        });
+    });
+});
+</script>
+{% endblock %}

akarpov/templates/gallery/list.html

@@ -1 +1,49 @@
 {% extends 'base.html' %}
+
+{% block content %}
+<div class="container">
+  <div class="row">
+    {% for collection_preview in collection_previews %}
+      <div class="col-md-4 col-sm-6 mb-4">
+        <div class="card">
+          <div class="card-body">
+            <!-- Folder Icon -->
+            <div class="folder-icon">
+              <i class="fas fa-folder" style="font-size: 6em;"></i>
+            </div>
+            <h5 class="card-title text-center">{{ collection_preview.collection.name }}</h5>
+            <!-- Image Thumbnails -->
+            <div class="folder-images d-flex flex-wrap justify-content-center">
+              {% for image in collection_preview.preview_images %}
+                <img src="{{ image.image_cropped.url }}" class="img-thumbnail m-1" alt="{{ image }}" style="width: 50px; height: 50px; object-fit: cover;">
+              {% endfor %}
+            </div>
+          </div>
+          <div class="card-footer">
+            <a href="{{ collection_preview.collection.get_absolute_url }}" class="btn btn-primary btn-block">View Collection</a>
+          </div>
+        </div>
+      </div>
+    {% empty %}
+      <p>No collections found.</p>
+    {% endfor %}
+  </div>
+</div>
+{% endblock %}
+
+{% block css %}
+<style>
+  .folder-icon {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    min-height: 120px; /* Adjust size as needed */
+  }
+  .folder-images img {
+    transition: transform 0.2s; /* Smooth transition for image hover */
+  }
+  .folder-images img:hover {
+    transform: scale(1.1); /* Slightly enlarge images on hover */
+  }
+</style>
+{% endblock %}

akarpov/templates/gallery/upload.html

@@ -0,0 +1 @@
+{% extends 'base.html' %}

akarpov/templates/music/playlist_create.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% load crispy_forms_filters %}
+{% load static %}
+
+{% block title %}creating playlist on akarpov{% endblock %}
+
+{% block content %}
+<form class="pt-2" enctype="multipart/form-data" method="POST" id="designer-form">
+    {% csrf_token %}
+    {{ form.media }}
+    {% for field in form %}
+        {{ field|as_crispy_field }}
+    {% endfor %}
+    <div class="mt-4 flex justify-end space-x-4">
+        <button class="btn btn-success" type="submit" id="submit">
+            <span class="spinner-border spinner-border-sm" id="spinner" role="status" aria-hidden="true" style="display: none"></span>
+            Create
+        </button>
+    </div>
+    <!-- TODO: add song select via API here -->
+</form>
+{% endblock %}

akarpov/templates/music/playlist_list.html

@@ -0,0 +1,59 @@
+{% extends 'base.html' %}
+
+{% block css %}
+<style>
+  .music-container {
+  width: 300px;
+  height: 300px;
+  display: flex;
+  flex-wrap: wrap;
+}
+.music-container > div {
+  flex: 1 0 50%;
+  height: 50%;
+}
+.music-container > div:nth-child(1):last-child, .music-container > div:nth-child(2):last-child, .music-container > div:nth-child(3):last-child {
+  flex-basis: 100%;
+}
+.music-container > div:nth-child(1):last-child:nth-child(1):last-child, .music-container > div:nth-child(2):last-child:nth-child(1):last-child, .music-container > div:nth-child(3):last-child:nth-child(1):last-child {
+  height: 100%;
+}
+.music-container > div img {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+  object-position: center;
+}
+</style>
+{% endblock %}
+
+{% block content %}
+  <div class="ms-3 row">
+    <div class="col-lg-2 col-xxl-2 col-md-4 col-sm-6 col-xs-12 mb-3 m-3 d-flex align-items-stretch card">
+      <div class="card-body d-flex flex-column justify-content-center align-items-center">
+        <h5 class="card-title">Create Playlist</h5>
+        <p class="card-text">Create your own playlist</p>
+        <a href="{% url 'music:create_playlist' %}" class="btn btn-primary"><i style="bi bi-plus"></i></a>
+      </div>
+  </div>
+{% for playlist in playlist_list %}
+  <div>
+    <div class="music-container">
+  <div>
+    <img src="https://img.freepik.com/free-photo/people-making-hands-heart-shape-silhouette-sunset_53876-15987.jpg" alt="">
+  </div>
+  <div>
+    <img src="https://thumbs.dreamstime.com/b/bee-flower-27533578.jpg" alt="">
+  </div>
+  <div>
+    <img src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTszhmO2dRnPW3Co-zQF_rqipldQM77r2Ut6Q&usqp=CAU" alt="">
+  </div>
+  <div>
+    <img src="https://img.freepik.com/free-photo/wide-angle-shot-single-tree-growing-clouded-sky-during-sunset-surrounded-by-grass_181624-22807.jpg" alt="">
+  </div>
+</div>
+  </div>
+  {{ playlist }}
+{% endfor %}
+  </div>
+{% endblock %}

akarpov/templates/music/radio.html

@@ -1,330 +1,77 @@
 {% extends 'base.html' %}
+{% load static %}
 
 {% block css %}
 <style>
 body {
-  background-color: #EDEDED;
+    background-color: #f8f9fa;
 }
 
-h1 {
+#song-info .card {
-  font-family: "Open Sans", sans-serif;
+    margin-bottom: 30px;
-  font-size: 13pt;
-  font-weight: 600;
-  text-transform: uppercase;
-  color: white;
-  cursor: default;
 }
 
-h4 {
+#history .card {
-  font-family: "Open Sans", sans-serif;
+    margin-bottom: 15px;
-  font-size: 8pt;
-  font-weight: 400;
-  cursor: default;
 }
 
-h2 {
+#admin-controls {
-  font-family: "Open Sans", sans-serif;
+    margin-bottom: 30px;
-  font-size: 13pt;
-  font-weight: 300;
-  color: white;
-  cursor: default;
 }
 
-.player {
+.card-img-top {
-  height: 190px;
+    width: 100%;
-  width: 430px;
+    height: 200px; /* You can adjust as needed */
-  background-color: #1E2125;
+    object-fit: cover;
-  position: absolute;
-  -webkit-touch-callout: none;
-  -webkit-user-select: none;
-  -moz-user-select: none;
-  -ms-user-select: none;
-  user-select: none;
-  top: 50%;
-  left: 50%;
-  transform: translate(-50%, -50%);
-  -webkit-transform: translate(-50%, -50%);
-}
-.player ul {
-  list-style: none;
-}
-.player ul li {
-  display: inline-block;
 }
 
-.cover {
+.audio-player {
-  position: absolute;
+    width: 100%;
-  top: 0;
+    margin-top: 10px;
-  left: 0;
-}
-.cover img {
-  height: 190px;
-  width: 190px;
 }
 
-.info h1 {
+.range-slider {
-  margin-top: 15px;
+    width: 100%;
-  margin-left: 180px;
+    margin-top: 10px;
-  line-height: 0;
-}
-.info h4 {
-  margin-left: 180px;
-  line-height: 20px;
-  color: #636367;
-}
-.info h2 {
-  margin-left: 180px;
 }
 
-.button-items {
+.range-slider input[type="range"] {
-  margin-left: 180px;
+    width: 100%;
-}
-
-#slider {
-  width: 182px;
-  height: 4px;
-  background: #151518;
-  border-radius: 2px;
-}
-#slider div {
-  width: 4px;
-  height: 4px;
-  margin-top: 1px;
-  background: #EF6DBC;
-  border-radius: 2px;
-}
-
-#timer {
-  color: #494B4E;
-  line-height: 0;
-  font-size: 9pt;
-  float: right;
-  font-family: Arial, Sans-Serif;
-}
-
-.controls {
-  margin-top: 20px;
-}
-.controls svg:nth-child(2) {
-  margin-left: 5px;
-  margin-right: 5px;
-}
-
-#play {
-  padding: 0 3px;
-  width: 30px;
-  height: 30px;
-  x: 0px;
-  y: 0px;
-  enable-background: new 0 0 25 25;
-}
-#play g {
-  stroke: #FEFEFE;
-  stroke-width: 1;
-  stroke-miterlimit: 10;
-}
-#play g path {
-  fill: #FEFEFE;
-}
-
-#play:hover {
-  cursor: pointer;
-}
-#play:hover g {
-  stroke: #8F4DA9;
-  cursor: pointer;
-}
-#play:hover g path {
-  fill: #9b59b6;
-  cursor: pointer;
-}
-
-.step-backward {
-  width: 18px;
-  height: 18px;
-  x: 0px;
-  y: 0px;
-  enable-background: new 0 0 25 25;
-  margin-bottom: 5px;
-}
-.step-backward g polygon {
-  fill: #FEFEFE;
-}
-
-.step-foreward {
-  width: 18px;
-  height: 18px;
-  x: 0px;
-  y: 0px;
-  enable-background: new 0 0 25 25;
-  margin-bottom: 5px;
-}
-.step-foreward g polygon {
-  fill: #FEFEFE;
-}
-
-#pause {
-  x: 0px;
-  y: 0px;
-  enable-background: new 0 0 25 25;
-  width: 30px;
-  height: 30px;
-  position: absolute;
-  margin-left: -38px;
-  cursor: pointer;
-}
-#pause rect {
-  fill: white;
-}
-
-#pause:hover rect {
-  fill: #8F4DA9;
-}
-
-.step-backward g polygon:hover, .step-foreward g polygon:hover {
-  fill: #EF6DBC;
-  cursor: pointer;
-}
-#skip p {
-  color: #2980b9;
-}
-#skip p:hover {
-  color: #e74c3c;
-  cursor: pointer;
-}
-
-.expend {
-  padding: 0.5px;
-  cursor: pointer;
-}
-.expend svg:hover g polygon {
-  fill: #EF6DBC;
 }
 </style>
 {% endblock %}
 
 {% block content %}
-    <div class="player">
+    <div class="container mt-5 music-body">
-   <ul>
+        <!-- Music Player and Visualization -->
-      <li class="cover"><img id="cover" src="" alt=""/></li>
+        <div class="music-player-container">
-      <li class="info">
+            <canvas id="audio-visualization" width="800" height="100"></canvas>
-         <h1 id="artist"></h1>
+            <audio id="audio-player" controls>
-         <h4 id="album"></h4>
+                <source src="" type="audio/mpeg">
-         <h2 id="name">I Need You Back</h2>
+                Your browser does not support the audio element.
-
-         <div class="button-items">
-            <audio id="music">
-               <source id="music-src" type="audio/mp3">
             </audio>
-            <div id="slider"><div id="elapsed"></div></div>
+        </div>
-            <p id="timer">0:00</p>
-            <div class="controls">
-               <span class="expend"><svg class="step-backward" viewBox="0 0 25 25" xml:space="preserve">
-                  <g><polygon points="4.9,4.3 9,4.3 9,11.6 21.4,4.3 21.4,20.7 9,13.4 9,20.7 4.9,20.7"/></g>
-               </svg></span>
 
-               <svg id="play" viewBox="0 0 25 25" xml:space="preserve">
+        <!-- Now Playing Information -->
-                   <defs><rect x="-49.5" y="-132.9" width="446.4" height="366.4"/></defs>
+        <div class="now-playing">
-                  <g><circle fill="none" cx="12.5" cy="12.5" r="10.8"/>
+            <div class="track-image">
-                       <path fill-rule="evenodd" clip-rule="evenodd" d="M8.7,6.9V18c0,0,0.2,1.4,1.8,0l8.1-4.8c0,0,1.2-1.1-1-2L9.8,6.5 C9.8,6.5,9.1,6,8.7,6.9z"/>
+                <img id="track-image" src="" alt="Track Image">
-                  </g>
+            </div>
-               </svg>
+            <div class="artist-info">
-
+                <!-- Dynamically populate artist info -->
-
-               <svg id="pause" viewBox="0 0 25 25" xml:space="preserve">
-                  <g>
-                     <rect x="6" y="4.6" width="3.8" height="15.7"/>
-                     <rect x="14" y="4.6" width="3.9" height="15.7"/>
-                  </g>
-               </svg>
-
-               <span class="expend"><svg class="step-foreward" viewBox="0 0 25 25" xml:space="preserve">
-                  <g><polygon points="20.7,4.3 16.6,4.3 16.6,11.6 4.3,4.3 4.3,20.7 16.7,13.4 16.6,20.7 20.7,20.7"/></g>
-                  </svg></span>
             </div>
         </div>
-        </li>
+
-     </ul>
+        <div id="admin-controls" class="mb-3" style="display: none;">
+            <!-- Admin Controls will be inserted here -->
         </div>
-{% endblock %}
+
-
+        <h2>History</h2>
-{% block inline_javascript %}
+        <div id="history" class="mb-3">
-
+            <!-- Song History will be inserted here -->
-    <script>
+        </div>
-        const music = document.getElementById("music");
+    </div>
-
+
-        async function playAudio() {
+    <script src="{% static 'js/jquery.js' %}"></script>
-            try {
+    <script src="{% static 'js/ws_script.js' %}"></script>
-              await music.play();
-              playButton.style.visibility = "hidden";
-              pause.style.visibility = "visible";
-            } catch (err) {
-              playButton.style.visibility = "visible";
-              pause.style.visibility = "hidden";
-            }
-          }
-
-        const music_src = document.getElementById("music-src")
-        const cover_src = document.getElementById("cover")
-        const name = document.getElementById("name")
-        const album = document.getElementById("album")
-        const artist = document.getElementById("artist")
-
-        const socket = new WebSocket(
-            'ws://'
-            + window.location.host
-            + '/ws/radio/'
-        );
-
-        socket.onmessage = function(e) {
-            const data = JSON.parse(e.data);
-            console.log(data)
-            music_src.src = data.file
-            if (data.image !== null) {
-              cover_src.src = data.image
-            }
-            name.innerText = data.name
-            playAudio()
-        };
-
-        socket.onclose = function(e) {
-            console.error('Chat socket closed unexpectedly');
-        };
-        const playButton = document.getElementById("play");
-        const pauseButton = document.getElementById("pause");
-        const playhead = document.getElementById("elapsed");
-        const timeline = document.getElementById("slider");
-        const timer = document.getElementById("timer");
-        let duration;
-        pauseButton.style.visibility = "hidden";
-
-        const timelineWidth = timeline.offsetWidth - playhead.offsetWidth;
-        music.addEventListener("timeupdate", timeUpdate, false);
-
-        function timeUpdate() {
-          const playPercent = timelineWidth * (music.currentTime / duration);
-          playhead.style.width = playPercent + "px";
-
-          const secondsIn = Math.floor(((music.currentTime / duration) / 3.5) * 100);
-          if (secondsIn <= 9) {
-            timer.innerHTML = "0:0" + secondsIn;
-          } else {
-            timer.innerHTML = "0:" + secondsIn;
-          }
-        }
-
-        playButton.onclick = function() {
-            playAudio()
-        }
-
-        pauseButton.onclick = function() {
-          music.pause();
-          playButton.style.visibility = "visible";
-          pause.style.visibility = "hidden";
-        }
-
-        music.addEventListener("canplaythrough", function () {
-          duration = music.duration;
-        }, false);
-    </script>
 {% endblock %}

akarpov/templates/search/indexes/files/file_text.txt

@@ -0,0 +1,3 @@
+{{ object.name }}
+{{ object.description }}
+{{ object.content }}

akarpov/templates/socialaccount/signup.html

@@ -0,0 +1,26 @@
+{% extends "socialaccount/base.html" %}
+
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block head_title %}{% trans "Signup" %}{% endblock %}
+
+{% block content %}
+    <h1>{% trans "Sign Up" %}</h1>
+
+<p>{% blocktrans with provider_name=account.get_provider.name site_name=site.name %}You are about to use your {{provider_name}} account to login to
+{{site_name}}. As a final step, please complete the following form:{% endblocktrans %}</p>
+
+<form class="signup" id="signup_form" method="post" action="{% url 'socialaccount_signup' %}">
+  {% csrf_token %}
+    {{ form.media }}
+    {% for field in form %}
+        {{ field|as_crispy_field }}
+    {% endfor %}
+  {% if redirect_field_value %}
+  <input type="hidden" name="{{ redirect_field_name }}" value="{{ redirect_field_value }}" />
+  {% endif %}
+  <button class="btn btn-success" type="submit">{% trans "Sign Up" %} &raquo;</button>
+</form>
+
+{% endblock %}

akarpov/templates/tools/uuid/main.html

@@ -0,0 +1,58 @@
+{% extends 'base.html' %}
+{% load static %}
+
+{% block content %}
+<div class="row align-items-center justify-content-center d-flex">
+  <div class="col-lg-12 d-flex align-items-stretch m-3">
+    <div style="width: 100%" class="card text-center">
+      <form class="card-body row justify-content-end ml-5 mr-5" method="get">
+        <div class="col-lg-11 col-sm-10">
+          <label for="uuid">Lookup uuid info</label><input {% if uuid %}value="{{ uuid }}" {% endif %}name="uuid" class="form-control" id="uuid" type="text" placeholder="insert uuid" />
+        </div>
+        <div class="col-lg-1 col-sm-2">
+          <button class="btn btn-success mt-4" type="submit"><i class="bi-search"></i></button>
+        </div>
+      </form>
+    </div>
+  </div>
+  {% if data %}
+  <div class="col-lg-10">
+    <table class="table">
+    <thead>
+      <tr>
+        <th scope="col">name</th>
+        <th scope="col">value</th>
+      </tr>
+    </thead>
+        <tbody>
+      {% for key, val in data.items %}
+      <tr>
+        <th scope="row">{{ key }}</th>
+        <td>{{ val }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+    </table>
+    {% endif %}
+  </div>
+  {% for token, version in tokens %}
+  <div class="col-lg-5 d-flex align-items-stretch justify-content-center m-3">
+    <div class="card text-center ml-1">
+      <div class="card-header">
+        <h4 class="bg-gray-300 ">{{ token }}<button class="btn" data-clipboard-text="{{ token }}">
+      <i style="font-size: 0.8em" class="bi bi-clipboard ml-2"></i>
+    </button></h4>
+      </div>
+      <div class="card-body">
+        Generated: {{ now }}, Version: {{ version }}
+      </div>
+    </div>
+  </div>
+  {% endfor %}
+</div>
+
+  <script src="{% static 'js/clipboard.min.js' %}"></script>
+  <script>
+  new ClipboardJS('.btn');
+  </script>
+{% endblock %}

akarpov/tools/urls.py

@@ -3,6 +3,7 @@
 app_name = "tools"
 urlpatterns = [
     path("qr/", include("akarpov.tools.qr.urls", namespace="qr")),
+    path("uuid/", include("akarpov.tools.uuidtools.urls", namespace="uuid")),
     path(
         "promocodes/", include("akarpov.tools.promocodes.urls", namespace="promocodes")
     ),

akarpov/tools/uuidtools/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class UuidtoolsConfig(AppConfig):
+    name = "akarpov.tools.uuidtools"

akarpov/tools/uuidtools/forms.py

@@ -0,0 +1,5 @@
+from django import forms
+
+
+class UUIDForm(forms.Form):
+    token = forms.UUIDField()

akarpov/tools/uuidtools/services.py

@@ -0,0 +1,56 @@
+import datetime
+from uuid import UUID
+
+
+def uuid1_time_to_datetime(time: int):
+    return datetime.datetime(1582, 10, 15) + datetime.timedelta(microseconds=time // 10)
+
+
+version_info = {
+    1: """UUID Version 1 (Time-Based): The current time and the specific MAC address of the computer generating the
+    UUID are used to construct IDs in UUID version 1. This means that even if they generate UUIDs simultaneously,
+    separate machines are probably going to produce different ones. If your clock also knew your computer’s unique
+    number, it would display a different number each time you glanced at it depending on the time and who you were.
+    When generating a UUID that is specific to a certain computer and linked to the moment it was generated,
+    this version is frequently utilised.""",
+    2: """V2 UUIDs include the MAC address of the generator, lossy timestamp, and an account ID such as user ID or
+    group ID on the local computer. Because of the information included in the UUID, there is limited space for
+    randomness. The clock section of the UUID only advances every 429.47 seconds (~7 minutes). During any 7 minute
+    period, there are only 64 available different UUIDs! """,
+    3: """UUID Version 3 (Name-Based, Using MD5):The UUID version 3 uses MD5 hashing to create IDs.
+    It uses the MD5 technique to combine the “namespace UUID” (a unique UUID) and the name you supply to get a unique
+    identification. Imagine it as a secret recipe book where you add a secret ingredient (namespace UUID) to a standard
+    ingredient (name), and when you combine them using a specific method (MD5), you get a unique dish (UUID).
+    When you need to consistently produce UUIDs based on particular names, this variant is frequently utilised.""",
+    4: """UUID Version 4 (Random): UUID version 4 uses random integers to create identifiers. It doesn’t depend on
+    any particular details like names or times. Instead, it simply generates a slew of random numbers and characters.
+    Imagine shaking a dice-filled box and then examining the face-up numbers that came out. It resembles receiving an
+    unpredictable combination each time. When you just require a single unique identification without any kind of
+    pattern or order, this version is fantastic.""",
+    5: """UUID Version 5 (Name-Based, Using SHA-1): Similar to version 3, UUID version 5 generates identifiers using
+    the SHA-1 algorithm rather than the MD5 technique. Similar to version 3, you provide it a “namespace UUID” and a
+    name, and it uses the SHA-1 technique to combine them to get a unique identification. Consider it similar to
+    baking a cake: you need a special pan (namespace UUID), your recipe (name), and a particular baking technique (
+    SHA-1). No matter how many times you cook this recipe, you will always end up with a unique cake (UUID). Similar
+    to version 3, UUID version 5 is frequently used in situations where a consistent and distinctive identity based
+    on particular names is required. The better encryption capabilities of SHA-1 make its use preferred over MD5 in
+    terms of security.""",
+}
+
+
+def decode_uuid(token: str) -> dict:
+    data = {"token": token}
+    try:
+        uuid = UUID(token)
+    except ValueError:
+        return {"message": "not a valid UUID token"}
+    data["version"] = f"{uuid.version}, {uuid.variant}"
+    data["hex"] = uuid.hex
+    data["bytes"] = bin(uuid.int)[2:]
+    data["num"] = uuid.int
+    if uuid.version == 1:
+        data["time"] = uuid1_time_to_datetime(uuid.time)
+
+    if uuid.version in version_info:
+        data["info"] = version_info[uuid.version]
+    return data

akarpov/tools/uuidtools/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+
+from . import views
+
+app_name = "uuidtools"
+
+urlpatterns = [
+    path("", views.MainView.as_view(), name="main"),
+]

akarpov/tools/uuidtools/views.py

@@ -0,0 +1,28 @@
+import uuid
+
+import uuid6
+from django.utils.timezone import now
+from django.views import generic
+
+from akarpov.tools.uuidtools.services import decode_uuid
+
+
+class MainView(generic.TemplateView):
+    template_name = "tools/uuid/main.html"
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+
+        if "uuid" in self.request.GET:
+            data = decode_uuid(str(self.request.GET["uuid"]))
+            context["data"] = data
+            context["uuid"] = self.request.GET["uuid"]
+
+        context["tokens"] = [
+            (uuid.uuid4(), 4),
+            (uuid6.uuid6(), 6),
+            (uuid6.uuid8(), 8),
+            (uuid.uuid1(), 1),
+        ]
+        context["now"] = now()
+        return context

akarpov/users/api/views.py

@@ -3,7 +3,7 @@
 from rest_framework import generics, permissions, status, views
 from rest_framework.response import Response
 
-from akarpov.common.api import SmallResultsSetPagination
+from akarpov.common.api.pagination import SmallResultsSetPagination
 from akarpov.common.jwt import sign_jwt
 from akarpov.users.api.serializers import (
     UserEmailVerification,

akarpov/users/models.py

@@ -8,6 +8,7 @@
 
 from akarpov.common.models import BaseImageModel
 from akarpov.tools.shortener.models import ShortLinkModel
+from akarpov.users.themes.models import Theme
 
 
 class User(AbstractUser, BaseImageModel, ShortLinkModel):
@@ -29,6 +30,11 @@ class User(AbstractUser, BaseImageModel, ShortLinkModel):
     )
     theme = models.ForeignKey("themes.Theme", null=True, on_delete=models.SET_NULL)
 
+    def get_theme_url(self):
+        if self.theme_id:
+            return Theme.objects.cache().get(id=self.theme_id).file.url
+        return ""
+
     def get_absolute_url(self):
         """Get url for user's detail view.
 

akarpov/users/tests/test_forms.py

@@ -0,0 +1,36 @@
+from django.test import TestCase
+
+from akarpov.users.forms import UserAdminCreationForm
+
+
+class UserFormTest(TestCase):
+    def test_valid_form(self):
+        form = UserAdminCreationForm(
+            data={
+                "username": "testuser",
+                "password1": "P4sSw0rD!",
+                "password2": "P4sSw0rD!",
+            }
+        )
+        self.assertTrue(form.is_valid())
+
+    def test_insecure_password(self):
+        form = UserAdminCreationForm(
+            data={
+                "username": "testuser",
+                "password1": "password",
+                "password2": "password",
+            }
+        )
+        self.assertFalse(form.is_valid())
+        self.assertEqual(len(form.errors), 1)
+        self.assertEqual(
+            form.errors["password2"],
+            ["This password is too common."],
+        )
+
+    def test_invalid_form(self):
+        form = UserAdminCreationForm(data={})
+        self.assertFalse(form.is_valid())
+        self.assertEqual(len(form.errors), 3)
+        self.assertEqual(form.errors["username"], ["This field is required."])

akarpov/users/tests/test_models.py

@@ -1,4 +1,11 @@
 from akarpov.files.consts import USER_INITIAL_FILE_UPLOAD
+from akarpov.users.models import User
+
+
+def test_user_creation(user_factory):
+    user = user_factory(username="testuser", email="test@example.com")
+    assert isinstance(user, User)
+    assert user.__str__() == user.username
 
 
 def test_user_create(user_factory):

akarpov/users/tests/test_views.py

@@ -0,0 +1,18 @@
+from django.test import Client, TestCase
+from django.urls import reverse
+
+from akarpov.users.tests.factories import UserFactory
+
+
+class UserViewTest(TestCase):
+    def setUp(self):
+        self.client = Client()
+        self.user = UserFactory()
+
+    def test_user_detail_view(self):
+        self.client.force_login(self.user)
+        response = self.client.get(
+            reverse("users:detail", kwargs={"username": self.user.username})
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, self.user.username)

akarpov/utils/celery.py

@@ -1,7 +1,7 @@
 from config.celery_app import app
 
 
-def get_scheduled_tasks_name() -> [str]:
+def get_scheduled_tasks_name() -> list[str]:
     i = app.control.inspect()
     t = i.scheduled()
     all_tasks = []

compose/local/django/Dockerfile

@@ -41,6 +41,7 @@ RUN poetry export --without-hashes -f requirements.txt | /venv/bin/pip install -
 
 COPY . .
 RUN poetry build && /venv/bin/pip install dist/*.whl
+RUN /venv/bin/python -m nltk.downloader punkt stopwords wordnet
 
 
 COPY ./compose/production/django/entrypoint /entrypoint
@@ -51,10 +52,6 @@ COPY ./compose/local/django/start /start
 RUN sed -i 's/\r$//g' /start
 RUN chmod +x /start
 
-COPY ./compose/local/django/spacy_setup /spacy_setup
-RUN sed -i 's/\r$//g' /spacy_setup
-RUN chmod +x /spacy_setup
-
 COPY ./compose/local/django/start-redirect /start-redirect
 RUN sed -i 's/\r$//g' /start-redirect
 RUN chmod +x /start-redirect

compose/local/django/celery/worker/start

@@ -3,5 +3,6 @@
 set -o errexit
 set -o nounset
 
+/install_preview_dependencies
 
-watchfiles celery.__main__.main --args '-A config.celery_app worker -l INFO'
+celery -A config.celery_app worker --loglevel=info -c 5

compose/local/django/install_preview_dependencies

@@ -1,13 +1,13 @@
 #!/bin/bash
 
 apt-get update
+apt-get install wget libnotify4 scribus libappindicator3-1 libayatana-indicator3-7 libdbusmenu-glib4 libdbusmenu-gtk3-4
 apt-get install -y poppler-utils libfile-mimeinfo-perl ghostscript libsecret-1-0 zlib1g-dev libjpeg-dev imagemagick libmagic1 libreoffice inkscape xvfb
-apt-get install -y libxml2-dev libxslt1-dev antiword unrtf pstotext tesseract-ocr flac lame libmad0 libsox-fmt-mp3 sox swig
+apt-get install -y libxml2-dev libxslt1-dev antiword unrtf tesseract-ocr flac lame libmad0 libsox-fmt-mp3 sox swig
-apt-get install -y python-dev libxml2-dev libxslt1-dev antiword unrtf poppler-utils pstotext tesseract-ocr \
+apt-get install -y python-dev-is-python3 libxml2-dev libxslt1-dev antiword unrtf poppler-utils tesseract-ocr \
 flac ffmpeg lame libmad0 libsox-fmt-mp3 sox libjpeg-dev swig
 wget https://github.com/jgraph/drawio-desktop/releases/download/v13.0.3/draw.io-amd64-13.0.3.deb
 dpkg -i draw.io-amd64-13.0.3.deb
-/spacy_setup
 rm draw.io-amd64-13.0.3.deb
 apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
 rm -rf /var/lib/apt/lists/*

compose/local/django/spacy_setup

@@ -1,3 +0,0 @@
-python -m spacy download en_core_web_lg
-python -m spacy download xx_sent_ud_sm
-python -m spacy download ru_core_news_lg

compose/local/django/start

@@ -7,4 +7,4 @@ set -o nounset
 
 python manage.py migrate auth
 python manage.py migrate
-python manage.py runserver_plus 0.0.0.0:8000
+daphne config.asgi:application --port 8000 --bind 0.0.0.0

compose/production/traefik/Dockerfile

@@ -1,4 +1,4 @@
-FROM traefik:2.10.1
+FROM traefik:2.10.7
 RUN mkdir -p /etc/traefik/acme \
   && touch /etc/traefik/acme/acme.json \
   && chmod 600 /etc/traefik/acme/acme.json

config/api_router.py

@@ -24,6 +24,10 @@
         "users/",
         include("akarpov.users.api.urls", namespace="users"),
     ),
+    path(
+        "gallery/",
+        include("akarpov.gallery.api.urls", namespace="gallery"),
+    ),
     path(
         "notifications/",
         include("akarpov.notifications.providers.urls", namespace="notifications"),

config/asgi.py

@@ -4,8 +4,14 @@
 from django.conf import settings
 from django.core.asgi import get_asgi_application
 
-from akarpov.common.channels import HeaderAuthMiddleware
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
-from config import routing
+# Initialize Django ASGI application early to ensure the AppRegistry
+# is populated before importing code that may import ORM models.
+django_asgi_app = get_asgi_application()
+
+
+from akarpov.common.channels import HeaderAuthMiddleware  # noqa
+from config import routing  # noqa
 
 if settings.DEBUG:
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
@@ -14,7 +20,7 @@
 
 application = ProtocolTypeRouter(
     {
-        "http": get_asgi_application(),
+        "http": django_asgi_app,
         "websocket": HeaderAuthMiddleware(URLRouter(routing.websocket_urlpatterns)),
     }
 )

config/settings/base.py

@@ -5,6 +5,7 @@
 
 import environ
 import structlog
+from celery.schedules import crontab
 from sentry_sdk.integrations.celery import CeleryIntegration
 
 ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent
@@ -53,7 +54,15 @@
 
 # CACHES
 # ------------------------------------------------------------------------------
-CACHES = {"default": env.cache_url("REDIS_CACHE")}
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": env("REDIS_CACHE_URL", default="redis://localhost:6379/1"),
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    }
+}
 CACHE_MIDDLEWARE_KEY_PREFIX = "cache_middleware"
 CACHE_MIDDLEWARE_SECONDS = 0
 CACHE_TTL = 60 * 10
@@ -62,8 +71,12 @@
 CACHEOPS = {
     "auth.user": {"ops": "get", "timeout": 60 * 15},
     "auth.*": {"ops": ("fetch", "get"), "timeout": 60 * 2},
-    "blog.post": {"ops": ("fetch", "get"), "timeout": 15},
+    "blog.post": {"ops": ("fetch", "get"), "timeout": 20 * 15},
+    "themes.theme": {"ops": ("fetch", "get"), "timeout": 60 * 60},
+    "gallery.*": {"ops": "all", "timeout": 60 * 15},
+    "files.*": {"ops": ("fetch", "get"), "timeout": 60},
     "auth.permission": {"ops": "all", "timeout": 60 * 15},
+    "music.*": {"ops": "all", "timeout": 60 * 15},
 }
 CACHEOPS_REDIS = env.str("REDIS_URL")
 
@@ -106,6 +119,7 @@
 ]
 
 THIRD_PARTY_APPS = [
+    "django.contrib.postgres",
     "crispy_forms",
     "crispy_bootstrap5",
     "allauth",
@@ -128,6 +142,7 @@
     "django_filters",
     "django_tables2",
     "location_field",
+    "django_elasticsearch_dsl",
 ]
 
 HEALTH_CHECKS = [
@@ -144,9 +159,9 @@
 
 ALLAUTH_PROVIDERS = [
     "allauth.socialaccount.providers.github",
-    # "allauth.socialaccount.providers.google",
+    "allauth.socialaccount.providers.google",
-    # "allauth.socialaccount.providers.telegram",
+    # "allauth.socialaccount.providers.telegram", TODO
-    # "allauth.socialaccount.providers.yandex",
+    # "allauth.socialaccount.providers.yandex", TODO
 ]
 
 LOCAL_APPS = [
@@ -302,6 +317,17 @@
 
 # EMAIL
 # ------------------------------------------------------------------------------
+"""
+    host: EMAIL_HOST
+    port: EMAIL_PORT
+    username: EMAIL_HOST_USER
+    password: EMAIL_HOST_PASSWORD
+    use_tls: EMAIL_USE_TLS
+    use_ssl: EMAIL_USE_SSL
+    timeout: EMAIL_TIMEOUT
+    ssl_keyfile: EMAIL_SSL_KEYFILE
+    ssl_certfile: EMAIL_SSL_CERTFILE
+"""
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
 EMAIL_BACKEND = env(
     "DJANGO_EMAIL_BACKEND",
@@ -310,11 +336,11 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-timeout
 EMAIL_TIMEOUT = 5
 EMAIL_HOST_PASSWORD = env(
-    "EMAIL_PASSWORD",
+    "EMAIL_HOST_PASSWORD",
     default="",
 )
 EMAIL_HOST_USER = env(
-    "EMAIL_USER",
+    "EMAIL_HOST_USER",
     default="",
 )
 EMAIL_USE_SSL = env(
@@ -322,6 +348,13 @@
     default=False,
 )
 
+EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
+# https://docs.djangoproject.com/en/dev/ref/settings/#email-port
+EMAIL_PORT = env("EMAIL_PORT", default="1025")
+EMAIL_FROM = env("EMAIL_FROM", default="noreply@akarpov.ru")
+DEFAULT_FROM_EMAIL = env("EMAIL_FROM", default="noreply@akarpov.ru")
+SERVER_EMAIL = env("EMAIL_FROM", default="noreply@akarpov.ru")
+
 # ADMIN
 # ------------------------------------------------------------------------------
 # Django Admin URL.
@@ -398,6 +431,7 @@
         structlog.processors.UnicodeDecoder(),
         structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
     ],
+    context_class=dict,
     logger_factory=structlog.stdlib.LoggerFactory(),
     cache_logger_on_first_use=True,
 )
@@ -425,6 +459,13 @@
 CELERY_TASK_SOFT_TIME_LIMIT = 10 * 60
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#beat-scheduler
 CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
+CELERY_BEAT_SCHEDULE = {
+    "update-index-every-hour": {
+        "task": "akarpov.files.tasks.update_index_task",
+        "schedule": crontab(minute="0"),
+    },
+}
+
 
 # django-allauth
 # ------------------------------------------------------------------------------
@@ -463,6 +504,7 @@
         },
     },
 }
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = env("HTTP_PROTOCOL", default="http")
 
 # django-rest-framework
 # -------------------------------------------------------------------------------
@@ -591,3 +633,13 @@
             CeleryIntegration(monitor_beat_tasks=True, propagate_traces=True),
         ],
     )
+
+
+# ELASTICSEARCH
+# ------------------------------------------------------------------------------
+ELASTICSEARCH_DSL = {
+    "default": {"hosts": env("ELASTIC_SEARCH", default="http://127.0.0.1:9200/")},
+}
+
+
+USE_DEBUG_TOOLBAR = False

config/settings/local.py

@@ -15,14 +15,6 @@
 ALLOWED_HOSTS = ["*"]
 CSRF_TRUSTED_ORIGINS = ["http://127.0.0.1", "https://*.akarpov.ru"]
 
-# EMAIL
-# ------------------------------------------------------------------------------
-# https://docs.djangoproject.com/en/dev/ref/settings/#email-host
-EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
-# https://docs.djangoproject.com/en/dev/ref/settings/#email-port
-EMAIL_PORT = env("EMAIL_PORT", default="1025")
-EMAIL_FROM = env("EMAIL_FROM", default="noreply@akarpov.ru")
-
 # WhiteNoise
 # ------------------------------------------------------------------------------
 # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development
@@ -32,17 +24,19 @@
 # django-debug-toolbar
 # ------------------------------------------------------------------------------
 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#prerequisites
-INSTALLED_APPS += ["debug_toolbar"]  # noqa F405
+USE_DEBUG_TOOLBAR = DEBUG and not env.bool("USE_DOCKER", default=False)
-# https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware
+if USE_DEBUG_TOOLBAR:
-MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]  # noqa F405
+    INSTALLED_APPS += ["debug_toolbar"]  # noqa F405
-# https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config
+    # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware
-DEBUG_TOOLBAR_CONFIG = {
+    MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]  # noqa F405
+    # https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config
+    DEBUG_TOOLBAR_CONFIG = {
         "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
         "SHOW_TEMPLATE_CONTEXT": True,
-}
+    }
-# https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips
+    # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips
-INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"]
+    INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"]
-if env("USE_DOCKER") == "yes":
+    if env("USE_DOCKER") == "yes":
         import socket
 
         hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())

config/urls.py

@@ -68,7 +68,7 @@
     ),
 ]
 
-if settings.DEBUG:
+if settings.USE_DEBUG_TOOLBAR:
     # This allows the error pages to be debugged during development, just visit
     # these url in browser to see how these error pages look like.
     urlpatterns += [

local.yml

@@ -3,6 +3,7 @@ version: '3'
 volumes:
   akarpov_local_postgres_data: {}
   akarpov_local_postgres_data_backups: {}
+  akarpov_local_elasticsearch_data: {}
 
 services:
   django: &django
@@ -16,7 +17,9 @@ services:
       - postgres
       - redis
       - mailhog
+      - elasticsearch
     volumes:
+      - /var/www/media:/app/akarpov/media
       - .:/app:z
     env_file:
       - ./.envs/.local/.django
@@ -42,7 +45,7 @@ services:
       - ./.envs/.local/.django
       - ./.envs/.local/.postgres
     ports:
-      - "3000:3000"
+      - "3025:3000"
     command: /start-redirect
 
   postgres:
@@ -96,3 +99,18 @@ services:
     ports:
       - "5555:5555"
     command: /start-flower
+
+  elasticsearch:
+    image: elasticsearch:8.11.1
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    environment:
+      - xpack.security.enabled=false
+      - node.name=activity
+      - discovery.type=single-node
+      - cluster.name=ws-es-data-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
+    volumes:
+      - akarpov_local_elasticsearch_data:/usr/share/elasticsearch/data

prof_start.py

@@ -0,0 +1,32 @@
+import cProfile
+import os
+import pstats
+
+import django
+
+
+# Function to run the Django setup process, which you want to profile
+def django_setup():
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
+    django.setup()
+
+
+# Create a Profile object and run the django_setup function
+profiler = cProfile.Profile()
+profiler.enable()
+django_setup()
+profiler.disable()
+
+# Write the stats to a .prof file
+profiler.dump_stats("django_setup.prof")
+
+# Create a Stats object and print the sorted stats to a text file
+with open("django_setup.txt", "w") as stream:
+    stats = pstats.Stats(profiler, stream=stream)
+    stats.sort_stats("cumtime")  # Sort the statistics by cumulative time
+    stats.print_stats()  # Print the statistics to the stream
+
+# Optionally, print the stats to the console as well
+stats = pstats.Stats(profiler)
+stats.sort_stats("cumtime")
+stats.print_stats()

pyproject.toml

@@ -88,18 +88,13 @@ django-tables2 = "^2.5.3"
 django-filter = "^23.2"
 tablib = "^3.4.0"
 django-location-field = "^2.7.0"
-pydantic = "^2.0.2"
 channels-redis = "^4.1.0"
 django-ipware = "^5.0.0"
-fastapi = {extras = ["all"], version = "^0.101.0"}
 sqlalchemy = "^2.0.19"
-pydantic-settings = "^2.0.2"
 yt-dlp = "^2023.7.6"
 pytube = "^15.0.0"
 urllib3 = ">=1.26"
 requests = ">=2.25"
-spacy = {extras = ["lookups"], version = "^3.6.1"}
-spacy-transformers = "^1.2.5"
 extract-msg = "0.28.7"
 pytest-factoryboy = "2.3.1"
 pytest-xdist = "^3.3.1"
@@ -109,6 +104,16 @@ pytest-lambda = "^2.2.0"
 pgvector = "^0.2.2"
 pycld2 = "^0.41"
 textract = "^1.6.5"
+uuid6 = "^2023.5.2"
+uvicorn = "^0.24.0.post1"
+nltk = "^3.8.1"
+pymorphy3 = "^1.2.1"
+pymorphy3-dicts-ru = "^2.4.417150.4580142"
+fastapi = "^0.104.1"
+pydantic-settings = "^2.0.3"
+django-elasticsearch-dsl = "^8.0"
+elasticsearch-dsl = "^8.11.0"
+numpy = "1.25.2"
 
 
 [build-system]

search/pipeline.py

@@ -0,0 +1,6 @@
+from haystack import Document
+from milvus_haystack import MilvusDocumentStore
+
+ds = MilvusDocumentStore()
+ds.write_documents([Document("Some Content")])
+ds.get_all_documents()

search/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "search"
+version = "0.1.0"
+description = ""
+authors = ["Alexander-D-Karpov <alexandr.d.karpov@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+fastapi = "0.99.1"
+pydantic = "1.10.13"
+transformers = {version = "4.34.1", extras = ["torch"]}
+torch = ">=2.0.0, !=2.0.1, !=2.1.0"
+farm-haystack = {extras = ["faiss"], version = "^1.21.2"}
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"