Merge b7a3a447dc into 6a7e7d5ade

akarpov/common/signals.py

@@ -13,7 +13,7 @@ def create_cropped_model_image(sender, instance, created, **kwargs):
                     "app_label": model._meta.app_label,
                     "model_name": model._meta.model_name,
                 },
-                countdown=2,
+                countdown=5,
             )
 
 

akarpov/common/tasks.py

@@ -5,7 +5,7 @@
 from akarpov.utils.files import crop_image
 
 
-@shared_task()
+@shared_task(max_retries=3)
 def crop_model_image(pk: int, app_label: str, model_name: str):
     model = apps.get_model(app_label=app_label, model_name=model_name)
     instance = model.objects.get(pk=pk)

akarpov/files/views.py

@@ -63,7 +63,7 @@ def filter(self, queryset):
 
             if search_type in search_classes:
                 search_instance = search_classes[search_type](
-                    queryset=File.objects.filter(user=self.request.user)
+                    queryset=File.objects.filter(user=self.request.user).nocache()
                 )
                 queryset = search_instance.search(query)
         return queryset

akarpov/music/api/serializers.py

@@ -5,6 +5,7 @@
 from akarpov.common.api.serializers import SetUserModelSerializer
 from akarpov.music.models import (
     Album,
+    AnonMusicUser,
     Author,
     Playlist,
     PlaylistSong,
@@ -198,6 +199,26 @@ def create(self, validated_data):
         return playlist_song
 
 
+class ListenSongSerializer(serializers.Serializer):
+    song = serializers.SlugField()
+    user_id = serializers.CharField(required=False)
+
+    def validate(self, attrs):
+        if not Song.objects.filter(slug=attrs["song"]).exists():
+            raise serializers.ValidationError("Song not found")
+
+        if "user_id" in attrs and attrs["user_id"]:
+            if not AnonMusicUser.objects.filter(id=attrs["user_id"]).exists():
+                raise serializers.ValidationError("User not found")
+        return attrs
+
+
+class AnonMusicUserSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = AnonMusicUser
+        fields = ["id"]
+
+
 class LikeDislikeSongSerializer(serializers.ModelSerializer):
     song = serializers.SlugField()
 
@@ -250,6 +271,10 @@ def create(self, validated_data):
         return song_user_rating
 
 
+class ListSongSlugsSerializer(serializers.Serializer):
+    slugs = serializers.ListField(child=serializers.SlugField())
+
+
 class ListPlaylistSerializer(serializers.ModelSerializer):
     class Meta:
         model = Playlist

akarpov/music/api/urls.py

@@ -2,6 +2,7 @@
 
 from akarpov.music.api.views import (
     AddSongToPlaylistAPIView,
+    CreateAnonMusicUserAPIView,
     DislikeSongAPIView,
     LikeSongAPIView,
     ListAlbumsAPIView,
@@ -13,6 +14,7 @@
     ListLikedSongsAPIView,
     ListPublicPlaylistAPIView,
     ListSongPlaylistsAPIView,
+    ListSongSlugsAPIView,
     ListUserListenedSongsAPIView,
     RemoveSongFromPlaylistAPIView,
     RetrieveUpdateDestroyAlbumAPIView,
@@ -40,6 +42,7 @@
         name="retrieve_update_delete_playlist",
     ),
     path("song/", ListCreateSongAPIView.as_view(), name="list_create_song"),
+    path("song/slugs/", ListSongSlugsAPIView.as_view(), name="list_songs_slugs"),
     path(
         "song/<str:slug>",
         RetrieveUpdateDestroySongAPIView.as_view(),
@@ -76,4 +79,5 @@
         RetrieveUpdateDestroyAuthorAPIView.as_view(),
         name="retrieve_update_delete_author",
     ),
+    path("anon/create/", CreateAnonMusicUserAPIView.as_view(), name="create-anon"),
 ]

akarpov/music/api/views.py

@@ -6,14 +6,17 @@
 from akarpov.common.api.permissions import IsAdminOrReadOnly, IsCreatorOrReadOnly
 from akarpov.music.api.serializers import (
     AddSongToPlaylistSerializer,
+    AnonMusicUserSerializer,
     FullAlbumSerializer,
     FullAuthorSerializer,
     FullPlaylistSerializer,
     LikeDislikeSongSerializer,
     ListAlbumSerializer,
     ListAuthorSerializer,
+    ListenSongSerializer,
     ListPlaylistSerializer,
     ListSongSerializer,
+    ListSongSlugsSerializer,
     PlaylistSerializer,
     SongSerializer,
 )
@@ -78,11 +81,7 @@ def get_queryset(self):
         return qs.select_related("creator")
 
 
-class ListCreateSongAPIView(LikedSongsContextMixin, generics.ListCreateAPIView):
+class ListBaseSongAPIView(generics.ListAPIView):
-    serializer_class = ListSongSerializer
-    permission_classes = [IsAdminOrReadOnly]
-    pagination_class = StandardResultsSetPagination
-
     def get_queryset(self):
         search = self.request.query_params.get("search", None)
         if search:
@@ -114,6 +113,14 @@ def get_queryset(self):
             )
         return qs
 
+
+class ListCreateSongAPIView(
+    LikedSongsContextMixin, generics.ListCreateAPIView, ListBaseSongAPIView
+):
+    serializer_class = ListSongSerializer
+    permission_classes = [IsAdminOrReadOnly]
+    pagination_class = StandardResultsSetPagination
+
     @extend_schema(
         parameters=[
             OpenApiParameter(
@@ -166,6 +173,67 @@ def get(self, request, *args, **kwargs):
         return self.list(request, *args, **kwargs)
 
 
+class ListSongSlugsAPIView(ListBaseSongAPIView):
+    serializer_class = ListSongSlugsSerializer
+    permission_classes = [permissions.AllowAny]
+
+    @extend_schema(
+        parameters=[
+            OpenApiParameter(
+                name="search",
+                description="Search query",
+                required=False,
+                type=str,
+            ),
+            OpenApiParameter(
+                name="sort",
+                description="Sorting algorithm",
+                required=False,
+                type=str,
+                examples=[
+                    OpenApiExample(
+                        "Default",
+                        description="by date added",
+                        value=None,
+                    ),
+                    OpenApiExample(
+                        "played",
+                        description="by total times played",
+                        value="played",
+                    ),
+                    OpenApiExample(
+                        "likes",
+                        description="by total likes",
+                        value="likes",
+                    ),
+                    OpenApiExample(
+                        "likes reversed",
+                        description="by total likes",
+                        value="-likes",
+                    ),
+                    OpenApiExample(
+                        "length",
+                        description="by track length",
+                        value="length",
+                    ),
+                    OpenApiExample(
+                        "uploaded",
+                        description="by date uploaded",
+                        value="uploaded",
+                    ),
+                ],
+            ),
+        ]
+    )
+    def get(self, request, *args, **kwargs):
+        songs = self.get_queryset()
+        return Response(
+            data={
+                "songs": songs.values_list("slug", flat=True),
+            }
+        )
+
+
 class RetrieveUpdateDestroySongAPIView(generics.RetrieveUpdateDestroyAPIView):
     lookup_field = "slug"
     lookup_url_kwarg = "slug"
@@ -314,7 +382,7 @@ class RetrieveUpdateDestroyAuthorAPIView(
 
 
 class ListenSongAPIView(generics.GenericAPIView):
-    serializer_class = LikeDislikeSongSerializer
+    serializer_class = ListenSongSerializer
     permission_classes = [permissions.AllowAny]
 
     def get_queryset(self):
@@ -331,12 +399,25 @@ def post(self, request, *args, **kwargs):
             return Response(status=404)
         if self.request.user.is_authenticated:
             listen_to_song.apply_async(
-                kwargs={"song_id": song.id, "user_id": self.request.user.id},
+                kwargs={
+                    "song_id": song.id,
+                    "user_id": self.request.user.id,
+                    "anon": False,
+                },
+                countdown=2,
+            )
+        elif "user_id" in data:
+            listen_to_song.apply_async(
+                kwargs={
+                    "song_id": song.id,
+                    "user_id": data["user_id"],
+                    "anon": True,
+                },
                 countdown=2,
             )
         else:
             listen_to_song.apply_async(
-                kwargs={"song_id": song.id},
+                kwargs={"song_id": song.id, "user_id": None, "anon": True},
                 countdown=2,
             )
         return Response(status=201)
@@ -353,3 +434,8 @@ def get_queryset(self):
             .filter(user=self.request.user)
             .values_list("song_id", flat=True)
         )
+
+
+class CreateAnonMusicUserAPIView(generics.CreateAPIView):
+    serializer_class = AnonMusicUserSerializer
+    permission_classes = [permissions.AllowAny]

akarpov/music/migrations/0016_anonmusicuser_song_created_song_volume_and_more.py

@@ -0,0 +1,79 @@
+# Generated by Django 4.2.8 on 2024-01-17 13:13
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("music", "0015_usermusicprofile"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="AnonMusicUser",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name="song",
+            name="created",
+            field=models.DateTimeField(
+                auto_now_add=True, default=django.utils.timezone.now
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="song",
+            name="volume",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.IntegerField(), null=True, size=None
+            ),
+        ),
+        migrations.CreateModel(
+            name="AnonMusicUserHistory",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                (
+                    "song",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="anon_listeners",
+                        to="music.song",
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="songs_listened",
+                        to="music.anonmusicuser",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["-created"],
+            },
+        ),
+    ]

akarpov/music/migrations/0016_song_volume.py

@@ -1,20 +0,0 @@
-# Generated by Django 4.2.8 on 2024-01-15 17:28
-
-import django.contrib.postgres.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("music", "0015_usermusicprofile"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="song",
-            name="volume",
-            field=django.contrib.postgres.fields.ArrayField(
-                base_field=models.IntegerField(), null=True, size=None
-            ),
-        ),
-    ]

akarpov/music/models.py

@@ -1,3 +1,5 @@
+import uuid
+
 from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from django.urls import reverse
@@ -46,6 +48,7 @@ class Song(BaseImageModel, ShortLinkModel):
     creator = models.ForeignKey(
         "users.User", related_name="songs", on_delete=models.SET_NULL, null=True
     )
+    created = models.DateTimeField(auto_now_add=True)
     meta = models.JSONField(blank=True, null=True)
     likes = models.IntegerField(default=0)
     volume = ArrayField(models.IntegerField(), null=True)
@@ -154,6 +157,29 @@ class Meta:
         ordering = ["-created"]
 
 
+class AnonMusicUser(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+
+    def __str__(self):
+        return f"AnonMusicUser {self.id}"
+
+
+class AnonMusicUserHistory(models.Model):
+    user = models.ForeignKey(
+        "music.AnonMusicUser", related_name="songs_listened", on_delete=models.CASCADE
+    )
+    song = models.ForeignKey(
+        "Song", related_name="anon_listeners", on_delete=models.CASCADE
+    )
+    created = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["-created"]
+
+    def __str__(self):
+        return f"{self.user} - {self.song}"
+
+
 class UserListenHistory(models.Model):
     user = models.ForeignKey(
         "users.User", related_name="songs_listened", on_delete=models.CASCADE

akarpov/music/services/db.py

@@ -1,7 +1,10 @@
 import os
+import re
 
+import requests
 from deep_translator import GoogleTranslator
 from django.core.files import File
+from django.db import transaction
 from django.utils.text import slugify
 from mutagen import File as MutagenFile
 from mutagen.id3 import APIC, ID3, TCON, TORY, TextFrame
@@ -10,9 +13,32 @@
 from pydub import AudioSegment
 
 from akarpov.music.models import Album, Author, Song
-from akarpov.music.services.info import search_all_platforms
+from akarpov.music.services.info import generate_readable_slug, search_all_platforms
 from akarpov.users.models import User
-from akarpov.utils.generators import generate_charset
+
+
+def get_or_create_author(author_name):
+    with transaction.atomic():
+        author = Author.objects.filter(name__iexact=author_name).order_by("id").first()
+        if author is None:
+            author = Author.objects.create(name=author_name)
+        return author
+
+
+def process_track_name(track_name: str) -> str:
+    # Split the track name by dash and parentheses
+    parts = track_name.split(" - ")
+    processed_parts = []
+
+    for part in parts:
+        if "feat" in part:
+            continue
+        if "(" in part:
+            part = part.split("(")[0].strip()
+        processed_parts.append(part)
+
+    processed_track_name = " - ".join(processed_parts)
+    return processed_track_name
 
 
 def load_track(
@@ -25,14 +51,31 @@ def load_track(
     link: str | None = None,
     **kwargs,
 ) -> Song:
-    p_name = path.split("/")[-1]
+    p_name = process_track_name(
-    query = f"{name if name else p_name} - {album if album else ''} - {', '.join(authors) if authors else ''}"
+        " ".join(path.split("/")[-1].split(".")[0].strip().split())
+    )
+    query = (
+        f"{process_track_name(name) if name else p_name} "
+        f"- {album if album else ''} - {', '.join(authors) if authors else ''}"
+    )
     search_info = search_all_platforms(query)
+    orig_name = name if name else p_name
 
     if image_path and search_info.get("album_image", None):
         os.remove(search_info["album_image"])
+    if "title" in search_info:
+        title = re.sub(r"\W+", "", search_info["title"]).lower()
+        name_clean = re.sub(r"\W+", "", orig_name).lower()
+
+        # Check if title is in name
+        if title in name_clean:
+            name = search_info["title"]
+        elif not name:
+            name = process_track_name(" ".join(p_name.strip().split("-")))
+
+    if not name:
+        name = orig_name
 
-    name = name or search_info.get("title", p_name)
     album = album or search_info.get("album_name", None)
     authors = authors or search_info.get("artists", [])
     genre = kwargs.get("genre") or search_info.get("genre", None)
@@ -44,15 +87,6 @@ def load_track(
     if album and type(album) is str and album.startswith("['"):
         album = album.replace("['", "").replace("']", "")
 
-    re_authors = []
-    if authors:
-        for x in authors:
-            try:
-                re_authors.append(Author.objects.get(name=x))
-            except Author.DoesNotExist:
-                re_authors.append(Author.objects.create(name=x))
-    authors = re_authors
-    album_name = None
     if album:
         if type(album) is str:
             album_name = album
@@ -61,12 +95,16 @@ def load_track(
         else:
             album_name = None
         if album_name:
-            try:
+            album, created = Album.objects.get_or_create(
-                album = Album.objects.get(name=album_name)
+                name__iexact=album_name, defaults={"name": album_name}
-            except Album.DoesNotExist:
+            )
-                album = Album.objects.create(name=album_name)
+
-    if not album_name:
+    processed_authors = []
-        album = None
+    if authors:
+        for author_name in authors:
+            author = get_or_create_author(author_name)
+            processed_authors.append(author)
+    authors = processed_authors
 
     if sng := Song.objects.filter(
         name=name if name else p_name,
@@ -82,6 +120,14 @@ def load_track(
         path = mp3_path
 
     tag = MP3(path, ID3=ID3)
+
+    if image_path and image_path.startswith("http"):
+        response = requests.get(image_path)
+        se = image_path.split("/")[-1]
+        image_path = f'/tmp/{generate_readable_slug(name, Song)}.{"png" if "." not in se else se.split(".")[-1]}'
+        with open(image_path, "wb") as f:
+            f.write(response.content)
+
     if image_path:
         if not image_path.endswith(".png"):
             nm = image_path
@@ -173,19 +219,7 @@ def load_track(
     if os.path.exists(image_path):
         os.remove(image_path)
 
-    if generated_name and not Song.objects.filter(slug=generated_name).exists():
+    song.slug = generate_readable_slug(song.name, Song)
-        if len(generated_name) > 20:
+    song.save()
-            generated_name = generated_name.split("-")[0]
-            if len(generated_name) > 20:
-                generated_name = generated_name[:20]
-            if not Song.objects.filter(slug=generated_name).exists():
-                song.slug = generated_name
-                song.save()
-            else:
-                song.slug = generated_name[:14] + "_" + generate_charset(5)
-                song.save()
-        else:
-            song.slug = generated_name
-            song.save()
 
     return song

akarpov/music/services/info.py

@@ -6,6 +6,7 @@
 from deep_translator import GoogleTranslator
 from django.conf import settings
 from django.core.files import File
+from django.db import transaction
 from django.utils.text import slugify
 from spotipy import SpotifyClientCredentials
 from yandex_music import Client, Cover
@@ -16,6 +17,34 @@
 from akarpov.utils.text import is_similar_artist, normalize_text
 
 
+def generate_readable_slug(name: str, model) -> str:
+    # Translate and slugify the name
+    slug = str(
+        slugify(
+            GoogleTranslator(source="auto", target="en").translate(
+                name,
+                target_language="en",
+            )
+        )
+    )
+
+    if len(slug) > 20:
+        slug = slug[:20]
+        last_dash = slug.rfind("-")
+        if last_dash != -1:
+            slug = slug[:last_dash]
+
+    while model.objects.filter(slug=slug).exists():
+        if len(slug) > 14:
+            slug = slug[:14]
+            last_dash = slug.rfind("-")
+            if last_dash != -1:
+                slug = slug[:last_dash]
+        slug = slug + "_" + generate_charset(5)
+
+    return slug
+
+
 def create_spotify_session() -> spotipy.Spotify:
     if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
         raise ConnectionError("No spotify credentials provided")
@@ -197,15 +226,6 @@ def update_album_info(album: AlbumModel, author_name: str = None) -> None:
 
     # Combine and prioritize Spotify data
     album_data = {}
-    if yandex_album_info:
-        album_data.update(
-            {
-                "name": album_data.get("name", yandex_album_info.title),
-                "genre": album_data.get("genre", yandex_album_info.genre),
-                "description": yandex_album_info.description,
-                "type": yandex_album_info.type,
-            }
-        )
 
     if spotify_album_info:
         album_data = {
@@ -215,6 +235,15 @@ def update_album_info(album: AlbumModel, author_name: str = None) -> None:
             "link": spotify_album_info["external_urls"]["spotify"],
             "genre": spotify_album_info.get("genres", []),
         }
+    if yandex_album_info:
+        album_data.update(
+            {
+                "name": album_data.get("name", yandex_album_info.title),
+                "genre": album_data.get("genre", yandex_album_info.genre),
+                "description": yandex_album_info.description,
+                "type": yandex_album_info.type,
+            }
+        )
 
     album.meta = album_data
     album.save()
@@ -262,20 +291,8 @@ def update_album_info(album: AlbumModel, author_name: str = None) -> None:
             album_authors.append(author)
         album.authors.set(album_authors)
 
-    if generated_name and not AlbumModel.objects.filter(slug=generated_name).exists():
+    album.slug = generate_readable_slug(album.name, AlbumModel)
-        if len(generated_name) > 20:
+    album.save()
-            generated_name = generated_name.split("-")[0]
-            if len(generated_name) > 20:
-                generated_name = generated_name[:20]
-            if not AlbumModel.objects.filter(slug=generated_name).exists():
-                album.slug = generated_name
-                album.save()
-            else:
-                album.slug = generated_name[:14] + "_" + generate_charset(5)
-                album.save()
-        else:
-            album.slug = generated_name
-            album.save()
 
 
 def update_author_info(author: Author) -> None:
@@ -288,6 +305,13 @@ def update_author_info(author: Author) -> None:
 
     # Combine and prioritize Spotify data
     author_data = {}
+    if spotify_artist_info:
+        author_data = {
+            "name": spotify_artist_info.get("name", author.name),
+            "genres": spotify_artist_info.get("genres", []),
+            "popularity": spotify_artist_info.get("popularity", 0),
+            "link": spotify_artist_info["external_urls"]["spotify"],
+        }
     if yandex_artist_info:
         author_data.update(
             {
@@ -297,16 +321,9 @@ def update_author_info(author: Author) -> None:
             }
         )
 
-    if spotify_artist_info:
-        author_data = {
-            "name": spotify_artist_info.get("name", author.name),
-            "genres": spotify_artist_info.get("genres", []),
-            "popularity": spotify_artist_info.get("popularity", 0),
-            "link": spotify_artist_info["external_urls"]["spotify"],
-        }
-
     author.meta = author_data
-    author.save()
+    with transaction.atomic():
+        author.save()
 
     # Handle Author Image - Prefer Spotify, fallback to Yandex
     image_path = None
@@ -337,20 +354,9 @@ def update_author_info(author: Author) -> None:
         os.remove(image_path)
         author.save()
 
-    if generated_name and not Author.objects.filter(slug=generated_name).exists():
+    author.slug = generate_readable_slug(author.name, Author)
-        if len(generated_name) > 20:
+    with transaction.atomic():
-            generated_name = generated_name.split("-")[0]
+        author.save()
-            if len(generated_name) > 20:
-                generated_name = generated_name[:20]
-            if not Author.objects.filter(slug=generated_name).exists():
-                author.slug = generated_name
-                author.save()
-            else:
-                author.slug = generated_name[:14] + "_" + generate_charset(5)
-                author.save()
-        else:
-            author.slug = generated_name
-            author.save()
 
 
 def search_all_platforms(track_name: str) -> dict:
@@ -373,7 +379,6 @@ def search_all_platforms(track_name: str) -> dict:
             for existing_artist in combined_artists
         ):
             combined_artists.add(normalized_artist)
-
     genre = spotify_info.get("genre") or yandex_info.get("genre")
     if type(genre) is list:
         genre = sorted(genre, key=lambda x: len(x))

akarpov/music/services/spotify.py

@@ -1,7 +1,12 @@
+import threading
+
 import spotipy
 from django.conf import settings
+from spotdl import Song, Spotdl
 from spotipy.oauth2 import SpotifyClientCredentials
 
+from akarpov.music.services.db import load_track
+
 
 def create_session() -> spotipy.Spotify:
     if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
@@ -18,3 +23,72 @@ def create_session() -> spotipy.Spotify:
 def search(name: str, session: spotipy.Spotify, search_type="track"):
     res = session.search(name, type=search_type)
     return res
+
+
+thread_local = threading.local()
+
+
+def get_spotdl_client():
+    if not hasattr(thread_local, "spotdl_client"):
+        spot_settings = {
+            "simple_tui": True,
+            "log_level": "ERROR",
+            "lyrics_providers": ["genius", "azlyrics", "musixmatch"],
+            "threads": 6,
+            "format": "mp3",
+            "ffmpeg": "ffmpeg",
+            "sponsor_block": True,
+        }
+        thread_local.spotdl_client = Spotdl(
+            client_id=settings.MUSIC_SPOTIFY_ID,
+            client_secret=settings.MUSIC_SPOTIFY_SECRET,
+            user_auth=False,
+            headless=False,
+            downloader_settings=spot_settings,
+        )
+    return thread_local.spotdl_client
+
+
+def download_url(url, user_id=None):
+    spotdl_client = get_spotdl_client()
+    session = create_session()
+
+    if "track" in url:
+        songs = [Song.from_url(url)]
+    elif "album" in url:
+        album_tracks = session.album(url)["tracks"]["items"]
+        songs = [
+            Song.from_url(track["external_urls"]["spotify"]) for track in album_tracks
+        ]
+    elif "artist" in url:
+        artist_top_tracks = session.artist_top_tracks(url)["tracks"]
+        songs = [
+            Song.from_url(track["external_urls"]["spotify"])
+            for track in artist_top_tracks
+        ]
+    elif "playlist" in url:
+        playlist_tracks = session.playlist_items(url)["items"]
+        songs = [
+            Song.from_url(track["track"]["external_urls"]["spotify"])
+            for track in playlist_tracks
+        ]
+    else:
+        return None
+
+    for song in songs:
+        res = spotdl_client.download(song)
+        if res:
+            song, path = res
+        else:
+            return None
+        load_track(
+            path=str(path),
+            image_path=song.cover_url,
+            user_id=user_id,
+            authors=song.artists,
+            album=song.album_name,
+            name=song.name,
+            link=song.url,
+            genre=song.genres[0] if song.genres else None,
+            release=song.date,
+        )

akarpov/music/services/youtube.py

@@ -6,9 +6,11 @@
 import requests
 import yt_dlp
 from django.conf import settings
+from django.utils.text import slugify
 from PIL import Image
 from pydub import AudioSegment
 from pytube import Search, YouTube
+from spotdl.providers.audio import YouTubeMusic
 
 from akarpov.music.models import Song
 from akarpov.music.services.db import load_track
@@ -17,22 +19,28 @@
 final_filename = None
 
 
-ydl_opts = {
+ytmusic = YouTubeMusic()
-    "format": "m4a/bestaudio/best",
-    "postprocessors": [
-        {  # Extract audio using ffmpeg
-            "key": "FFmpegExtractAudio",
-            "preferredcodec": "m4a",
-        }
-    ],
-    "outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
-}
 
 
 def download_file(url):
+    ydl_opts = {
+        "format": "bestaudio/best",
+        "outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
+        "postprocessors": [
+            {"key": "SponsorBlock"},  # Skip sponsor segments
+            {
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "mp3",
+                "preferredquality": "192",
+            },  # Extract audio
+            {"key": "EmbedThumbnail"},  # Embed Thumbnail
+            {"key": "FFmpegMetadata"},  # Apply correct metadata
+        ],
+    }
     with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        info = ydl.extract_info(url)
+        info = ydl.extract_info(url, download=True)
-    return info["requested_downloads"][0]["_filename"]
+        filename = ydl.prepare_filename(info)
+    return os.path.splitext(filename)[0] + ".mp3"
 
 
 def parse_description(description: str) -> list:
@@ -66,7 +74,7 @@ def parse_description(description: str) -> list:
 def download_from_youtube_link(link: str, user_id: int) -> Song:
     song = None
 
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+    with yt_dlp.YoutubeDL({"ignoreerrors": True, "extract_flat": True}) as ydl:
         info_dict = ydl.extract_info(link, download=False)
         title = info_dict.get("title", None)
         description = info_dict.get("description", None)
@@ -75,9 +83,18 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
 
     # convert to mp3
     print(f"[processing] {title} converting to mp3")
-    path = orig_path.replace(orig_path.split(".")[-1], "mp3")
+    path = (
-    AudioSegment.from_file(orig_path).export(path)
+        "/".join(orig_path.split("/")[:-1])
-    os.remove(orig_path)
+        + "/"
+        + slugify(orig_path.split("/")[-1].split(".")[0])
+        + ".mp3"
+    )
+    if orig_path.endswith(".mp3"):
+        os.rename(orig_path, path)
+    else:
+        AudioSegment.from_file(orig_path).export(path)
+        if orig_path != path:
+            os.remove(orig_path)
     print(f"[processing] {title} converting to mp3: done")
 
     # split in chapters
@@ -175,7 +192,8 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
                 info["album_name"],
                 title,
             )
-    os.remove(path)
+    if os.path.exists(path):
+        os.remove(path)
 
     return song
 

akarpov/music/signals.py

@@ -17,7 +17,7 @@ def auto_delete_file_on_delete(sender, instance, **kwargs):
 
 @receiver(post_save, sender=Song)
 def song_create(sender, instance: Song, created, **kwargs):
-    if instance.volume is None:
+    if instance.volume is None and instance.file:
         set_song_volume(instance)
 
 

akarpov/music/tasks.py

@@ -1,16 +1,27 @@
+from datetime import timedelta
+
 import pylast
+import spotipy
 import structlog
+import ytmusicapi
 from asgiref.sync import async_to_sync
 from celery import shared_task
 from channels.layers import get_channel_layer
 from django.conf import settings
 from django.utils import timezone
 from django.utils.timezone import now
-from pytube import Channel, Playlist
+from spotipy import SpotifyClientCredentials
 
 from akarpov.music.api.serializers import SongSerializer
-from akarpov.music.models import RadioSong, Song, UserListenHistory, UserMusicProfile
+from akarpov.music.models import (
-from akarpov.music.services import yandex, youtube
+    AnonMusicUser,
+    AnonMusicUserHistory,
+    RadioSong,
+    Song,
+    UserListenHistory,
+    UserMusicProfile,
+)
+from akarpov.music.services import spotify, yandex, youtube
 from akarpov.music.services.file import load_dir, load_file
 from akarpov.utils.celery import get_scheduled_tasks_name
 
@@ -19,18 +30,57 @@
 
 @shared_task
 def list_tracks(url, user_id):
-    if "music.yandex.ru" in url:
+    if "music.youtube.com" in url or "youtu.be" in url:
+        url = url.replace("music.youtube.com", "youtube.com")
+        url = url.replace("youtu.be", "youtube.com")
+    if "spotify.com" in url:
+        spotify.download_url(url, user_id)
+    elif "music.yandex.ru" in url:
         yandex.load_playlist(url, user_id)
-    elif "channel" in url or "/c/" in url:
+    if "youtube.com" in url:
-        p = Channel(url)
+        if "channel" in url or "/c/" in url:
-        for video in p.video_urls:
+            ytmusic = ytmusicapi.YTMusic()
-            process_yb.apply_async(kwargs={"url": video, "user_id": user_id})
+            channel_id = url.split("/")[-1]
-    elif "playlist" in url or "&list=" in url:
+            channel_songs = ytmusic.get_artist(channel_id)["songs"]["results"]
-        p = Playlist(url)
+            print(channel_songs)
-        for video in p.video_urls:
+
-            process_yb.apply_async(kwargs={"url": video, "user_id": user_id})
+            for song in channel_songs:
+                process_yb.apply_async(
+                    kwargs={
+                        "url": f"https://youtube.com/watch?v={song['videoId']}",
+                        "user_id": user_id,
+                    }
+                )
+
+        elif "playlist" in url or "&list=" in url:
+            ytmusic = ytmusicapi.YTMusic()
+            playlist_id = url.split("=")[-1]
+            playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
+
+            for song in playlist_songs:
+                process_yb.apply_async(
+                    kwargs={
+                        "url": f"https://music.youtube.com/watch?v={song['videoId']}",
+                        "user_id": user_id,
+                    }
+                )
+        else:
+            process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
     else:
-        process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
+        spotify_manager = SpotifyClientCredentials(
+            client_id=settings.MUSIC_SPOTIFY_ID,
+            client_secret=settings.MUSIC_SPOTIFY_SECRET,
+        )
+        spotify_search = spotipy.Spotify(client_credentials_manager=spotify_manager)
+
+        results = spotify_search.search(q=url, type="track", limit=1)
+        top_track = (
+            results["tracks"]["items"][0] if results["tracks"]["items"] else None
+        )
+
+        if top_track:
+            spotify.download_url(top_track["external_urls"]["spotify"], user_id)
+            url = top_track["external_urls"]["spotify"]
 
     return url
 
@@ -78,8 +128,6 @@ def start_next_song(previous_ids: list):
         async_to_sync(channel_layer.group_send)(
             "radio_main", {"type": "song", "data": data}
         )
-        song.played += 1
-        song.save(update_fields=["played"])
         if RadioSong.objects.filter(slug="").exists():
             r = RadioSong.objects.get(slug="")
             r.song = song
@@ -96,7 +144,7 @@ def start_next_song(previous_ids: list):
 
 
 @shared_task
-def listen_to_song(song_id, user_id=None):
+def listen_to_song(song_id, user_id=None, anon=True):
     # protection from multiple listen,
     # check that last listen by user was more than the length of the song
     # and last listened song is not the same
@@ -104,38 +152,63 @@ def listen_to_song(song_id, user_id=None):
     s.played += 1
     s.save(update_fields=["played"])
     if user_id:
-        try:
+        if anon:
-            last_listen = UserListenHistory.objects.filter(user_id=user_id).latest("id")
+            try:
-        except UserListenHistory.DoesNotExist:
+                anon_user = AnonMusicUser.objects.get(id=user_id)
-            last_listen = None
+            except AnonMusicUser.DoesNotExist:
-        if (
+                anon_user = AnonMusicUser.objects.create(id=user_id)
-            last_listen
+            try:
-            and last_listen.song_id == song_id
+                last_listen = AnonMusicUserHistory.objects.filter(
-            or last_listen
+                    user_id=user_id
-            and last_listen.created + s.length > now()
+                ).last()
-        ):
+            except AnonMusicUserHistory.DoesNotExist:
-            return
+                last_listen = None
-        UserListenHistory.objects.create(
+            if (
-            user_id=user_id,
+                last_listen
-            song_id=song_id,
+                and last_listen.song_id == song_id
-        )
+                or last_listen
-        try:
+                and last_listen.created + timedelta(seconds=s.length) > now()
-            user_profile = UserMusicProfile.objects.get(user_id=user_id)
+            ):
-            lastfm_token = user_profile.lastfm_token
+                return
-
+            AnonMusicUserHistory.objects.create(
-            # Initialize Last.fm network with the user's session key
+                user=anon_user,
-            network = pylast.LastFMNetwork(
+                song_id=song_id,
-                api_key=settings.LAST_FM_API_KEY,
-                api_secret=settings.LAST_FM_SECRET,
-                session_key=lastfm_token,
             )
-            song = Song.objects.get(id=song_id)
+        else:
-            artist_name = song.artists_names
+            try:
-            track_name = song.name
+                last_listen = UserListenHistory.objects.filter(user_id=user_id).last()
-            timestamp = int(timezone.now().timestamp())
+            except UserListenHistory.DoesNotExist:
-            network.scrobble(artist=artist_name, title=track_name, timestamp=timestamp)
+                last_listen = None
-        except UserMusicProfile.DoesNotExist:
+            if (
-            pass
+                last_listen
-        except Exception as e:
+                and last_listen.song_id == song_id
-            logger.error(f"Last.fm scrobble error: {e}")
+                or last_listen
+                and last_listen.created + timedelta(seconds=s.length) > now()
+            ):
+                return
+            UserListenHistory.objects.create(
+                user_id=user_id,
+                song_id=song_id,
+            )
+            try:
+                user_profile = UserMusicProfile.objects.get(user_id=user_id)
+                lastfm_token = user_profile.lastfm_token
+
+                # Initialize Last.fm network with the user's session key
+                network = pylast.LastFMNetwork(
+                    api_key=settings.LAST_FM_API_KEY,
+                    api_secret=settings.LAST_FM_SECRET,
+                    session_key=lastfm_token,
+                )
+                song = Song.objects.get(id=song_id)
+                artist_name = song.artists_names
+                track_name = song.name
+                timestamp = int(timezone.now().timestamp())
+                network.scrobble(
+                    artist=artist_name, title=track_name, timestamp=timestamp
+                )
+            except UserMusicProfile.DoesNotExist:
+                pass
+            except Exception as e:
+                logger.error(f"Last.fm scrobble error: {e}")
     return song_id

akarpov/tools/shortener/models.py

@@ -1,7 +1,7 @@
 from abc import abstractmethod
 
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from django.urls import reverse
 from model_utils.models import TimeStampedModel
 
@@ -79,7 +79,8 @@ def create_model_link(sender, instance, created, **kwargs):
 
         link.save()
         instance.short_link = link
-        instance.save()
+        with transaction.atomic():
+            instance.save()
 
 
 def update_model_link(sender, instance, **kwargs):

compose/local/django/Dockerfile

@@ -33,6 +33,13 @@ RUN apt-get update && \
     apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
     rm -rf /var/lib/apt/lists/*
 
+# Make ssh dir
+RUN mkdir -p /root/.ssh/
+
+# Create known_hosts and add github to it
+RUN touch /root/.ssh/known_hosts
+RUN ssh-keyscan -t rsa github.com >> /root/.ssh/known_hosts
+
 RUN pip install "poetry==$POETRY_VERSION"
 RUN python -m venv /venv
 

pyproject.toml

@@ -119,7 +119,7 @@ spotdl = "^4.2.4"
 fuzzywuzzy = "^0.18.0"
 python-levenshtein = "^0.23.0"
 pylast = "^5.2.0"
-textract = {git = "https://github.com/Alexander-D-Karpov/textract.git", branch = "master"}
+textract = {git = "https://github.com/Alexander-D-Karpov/textract", branch = "master"}
 librosa = "^0.10.1"