Merge 1517c4dbf0 into a2da7e724f

Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.21.1 to 0.23.5.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.21.1...v0.23.5)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

akarpov/music/api/serializers.py

@@ -22,9 +22,17 @@ class Meta:
 
 
 class ListAlbumSerializer(serializers.ModelSerializer):
+    authors = serializers.SerializerMethodField(method_name="get_authors")
+
+    @extend_schema_field(ListAuthorSerializer(many=True))
+    def get_authors(self, obj):
+        return ListAuthorSerializer(
+            Author.objects.cache().filter(albums__id=obj.id), many=True
+        ).data
+
     class Meta:
         model = Album
-        fields = ["name", "slug", "image_cropped"]
+        fields = ["name", "slug", "image_cropped", "authors"]
 
 
 class SongSerializer(serializers.ModelSerializer):
@@ -133,10 +141,32 @@ class Meta:
 
 class PlaylistSerializer(SetUserModelSerializer):
     creator = UserPublicInfoSerializer(read_only=True)
+    images = serializers.SerializerMethodField(method_name="get_images")
+
+    @extend_schema_field(serializers.ListField(child=serializers.ImageField()))
+    def get_images(self, obj):
+        # Get distinct album images from songs
+        images = (
+            Song.objects.cache()
+            .filter(
+                playlists__id__in=PlaylistSong.objects.cache()
+                .filter(playlist=obj)
+                .values("id")
+            )
+            .values_list("album__image", flat=True)
+            .distinct()[:4]
+        )
+        # Build absolute URI for each image
+        images = [
+            self.context["request"].build_absolute_uri(image)
+            for image in images
+            if image
+        ]
+        return images
 
     class Meta:
         model = Playlist
-        fields = ["name", "length", "slug", "private", "creator"]
+        fields = ["name", "length", "slug", "images", "private", "creator"]
         extra_kwargs = {
             "slug": {"read_only": True},
             "creator": {"read_only": True},
@@ -144,13 +174,21 @@ class Meta:
         }
 
 
-class FullPlaylistSerializer(serializers.ModelSerializer):
-    songs = ListSongSerializer(many=True, read_only=True)
+class FullPlaylistSerializer(PlaylistSerializer):
+    songs = serializers.SerializerMethodField(method_name="get_songs")
     creator = UserPublicInfoSerializer(read_only=True)
 
+    @extend_schema_field(ListSongSerializer(many=True))
+    def get_songs(self, obj):
+        return ListSongSerializer(
+            Song.objects.cache().filter(playlists__id=obj.id),
+            many=True,
+            context=self.context,
+        ).data
+
     class Meta:
         model = Playlist
-        fields = ["name", "private", "creator", "songs"]
+        fields = ["name", "private", "creator", "images", "songs"]
         extra_kwargs = {
             "slug": {"read_only": True},
             "creator": {"read_only": True},
@@ -158,8 +196,8 @@ class Meta:
 
 
 class AddSongToPlaylistSerializer(serializers.ModelSerializer):
-    song = serializers.SlugField()
-    playlist = serializers.SlugField()
+    song = serializers.SlugField(write_only=True)
+    playlist = serializers.SlugField(write_only=True)
 
     class Meta:
         model = Playlist

akarpov/music/documents.py

@@ -39,24 +39,70 @@ class SongDocument(Document):
         },
     )
 
-    meta = fields.ObjectField(dynamic=True)  # Added meta field here as dynamic object
+    meta = fields.ObjectField(dynamic=True)
 
     class Index:
         name = "songs"
-        settings = {"number_of_shards": 1, "number_of_replicas": 0}
-        # settings = {
-        #     "number_of_shards": 1,
-        #     "number_of_replicas": 0,
-        #     "analysis": {
-        #         "analyzer": {
-        #             "russian_icu": {
-        #                 "type": "custom",
-        #                 "tokenizer": "icu_tokenizer",
-        #                 "filter": ["icu_folding","icu_normalizer"]
-        #             }
-        #         }
-        #     }
-        # } TODO
+        settings = {
+            "number_of_shards": 1,
+            "number_of_replicas": 0,
+            "analysis": {
+                "filter": {
+                    "russian_stop": {
+                        "type": "stop",
+                        "stopwords": "_russian_",
+                    },
+                    "russian_keywords": {
+                        "type": "keyword_marker",
+                        "keywords": ["пример"],
+                    },
+                    "russian_stemmer": {
+                        "type": "stemmer",
+                        "language": "russian",
+                    },
+                    "autocomplete_filter": {
+                        "type": "edge_ngram",
+                        "min_gram": 1,
+                        "max_gram": 20,
+                    },
+                    "synonym_filter": {
+                        "type": "synonym",
+                        "synonyms": [
+                            "бит,трек,песня,музыка,песня,мелодия,композиция",
+                            "певец,исполнитель,артист,музыкант",
+                            "альбом,диск,пластинка,сборник,коллекция",
+                        ],
+                    },
+                },
+                "analyzer": {
+                    "russian": {
+                        "tokenizer": "standard",
+                        "filter": [
+                            "russian_stop",
+                            "russian_keywords",
+                            "russian_stemmer",
+                        ],
+                    },
+                    "russian_icu": {
+                        "tokenizer": "icu_tokenizer",
+                        "filter": [
+                            "russian_stop",
+                            "russian_keywords",
+                            "russian_stemmer",
+                        ],
+                    },
+                    "autocomplete": {
+                        "type": "custom",
+                        "tokenizer": "standard",
+                        "filter": [
+                            "lowercase",
+                            "autocomplete_filter",
+                            "synonym_filter",
+                        ],
+                    },
+                },
+            },
+        }
 
     class Django:
         model = Song

akarpov/music/services/db.py

@@ -72,6 +72,16 @@ def load_track(
             name = search_info["title"]
         elif not name:
             name = process_track_name(" ".join(p_name.strip().split("-")))
+            clear_name = [
+                "(Official HD Video)",
+                "(Official Music Video)",
+                "(Official Video)",
+                "Official Video",
+                "Official Music Video",
+                "Official HD Video",
+            ]
+            for c in clear_name:
+                name = name.replace(c, "")
 
     if not name:
         name = orig_name

akarpov/music/services/info.py

@@ -360,6 +360,7 @@ def update_author_info(author: Author) -> None:
 
 
 def search_all_platforms(track_name: str) -> dict:
+    print(track_name)
     session = spotipy.Spotify(
         auth_manager=spotipy.SpotifyClientCredentials(
             client_id=settings.MUSIC_SPOTIFY_ID,

akarpov/music/services/search.py

@@ -13,9 +13,9 @@ def search_song(query):
             ES_Q(
                 "multi_match",
                 query=query,
-                fields=["name^3", "authors.name^2", "album.name"],
+                fields=["name^5", "authors.name^3", "album.name^3"],
                 fuzziness="AUTO",
-            ),  # Change here
+            ),
             ES_Q("wildcard", name__raw=f"*{query.lower()}*"),
             ES_Q(
                 "nested",
@@ -27,6 +27,7 @@ def search_song(query):
                 path="album",
                 query=ES_Q("wildcard", album__name__raw=f"*{query.lower()}*"),
             ),
+            ES_Q("wildcard", meta__raw=f"*{query.lower()}*"),
         ],
         minimum_should_match=1,
     )

akarpov/music/services/youtube.py

@@ -157,22 +157,25 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
         print(f"[processing] loading {title}")
 
         info = search_all_platforms(title)
-        if not info["album_image"].startswith("/"):
-            r = requests.get(info["album_image"])
-            img_pth = str(
-                settings.MEDIA_ROOT
-                + f"/{info['album_image'].split('/')[-1]}_{str(randint(100, 999))}"
-            )
-            with open(img_pth, "wb") as f:
-                f.write(r.content)
+        if "album_image" in info and info["album_image"]:
+            if not info["album_image"].startswith("/"):
+                r = requests.get(info["album_image"])
+                img_pth = str(
+                    settings.MEDIA_ROOT
+                    + f"/{info['album_image'].split('/')[-1]}_{str(randint(100, 999))}"
+                )
+                with open(img_pth, "wb") as f:
+                    f.write(r.content)
 
-            im = Image.open(img_pth)
-            im.save(str(f"{img_pth}.png"))
+                im = Image.open(img_pth)
+                im.save(str(f"{img_pth}.png"))
 
-            os.remove(img_pth)
-            img_pth = f"{img_pth}.png"
+                os.remove(img_pth)
+                img_pth = f"{img_pth}.png"
+            else:
+                img_pth = info["album_image"]
         else:
-            img_pth = info["album_image"]
+            img_pth = None
         if "genre" in info:
             song = load_track(
                 path,

akarpov/music/tasks.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+from urllib.parse import parse_qs, urlparse
 
 import pylast
 import spotipy
@@ -54,9 +55,19 @@ def list_tracks(url, user_id):
 
         elif "playlist" in url or "&list=" in url:
             ytmusic = ytmusicapi.YTMusic()
-            playlist_id = url.split("=")[-1]
-            playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
 
+            # Parse the URL and the query string
+            parsed_url = urlparse(url)
+            parsed_qs = parse_qs(parsed_url.query)
+
+            # Get the playlist ID from the parsed query string
+            playlist_id = parsed_qs.get("list", [None])[0]
+
+            if playlist_id:
+                playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]
+
+            else:
+                raise ValueError("No playlist ID found in the URL.")
             for song in playlist_songs:
                 process_yb.apply_async(
                     kwargs={

compose/local/django/celery/worker/start

@@ -5,4 +5,4 @@ set -o nounset
 
 /install_preview_dependencies
 
-celery -A config.celery_app worker --loglevel=info -c 5
+celery -A config.celery_app  worker --autoscale 20 -l INFO

compose/production/elasticsearch/Dockerfile

@@ -0,0 +1,4 @@
+FROM elasticsearch:8.11.1
+
+# Install the ICU plugin
+RUN bin/elasticsearch-plugin install https://akarpov.ru/media/analysis-icu-8.11.1.zip

local.yml

@@ -101,7 +101,9 @@ services:
     command: /start-flower
 
   elasticsearch:
-    image: elasticsearch:8.11.1
+    build:
+      context: .
+      dockerfile: ./compose/production/elasticsearch/Dockerfile
     ports:
       - "9200:9200"
       - "9300:9300"