Merge 0b2d0b0c18 into 6a7e7d5ade

akarpov/files/views.py

@@ -63,7 +63,7 @@ def filter(self, queryset):
 
             if search_type in search_classes:
                 search_instance = search_classes[search_type](
-                    queryset=File.objects.filter(user=self.request.user)
+                    queryset=File.objects.filter(user=self.request.user).nocache()
                 )
                 queryset = search_instance.search(query)
         return queryset

akarpov/music/services/db.py

@@ -1,6 +1,7 @@
 import os
 import re
 
+import requests
 from deep_translator import GoogleTranslator
 from django.core.files import File
 from django.db import transaction
@@ -16,6 +17,14 @@
 from akarpov.users.models import User
 
 
+def get_or_create_author(author_name):
+    with transaction.atomic():
+        author = Author.objects.filter(name__iexact=author_name).order_by("id").first()
+        if author is None:
+            author = Author.objects.create(name=author_name)
+        return author
+
+
 def process_track_name(track_name: str) -> str:
     # Split the track name by dash and parentheses
     parts = track_name.split(" - ")
@@ -78,24 +87,6 @@ def load_track(
     if album and type(album) is str and album.startswith("['"):
         album = album.replace("['", "").replace("']", "")
 
-    re_authors = []
-    if authors:
-        for x in authors:
-            while True:
-                try:
-                    with transaction.atomic():
-                        author, created = Author.objects.get_or_create(
-                            name__iexact=x, defaults={"name": x}
-                        )
-                    re_authors.append(author)
-                    break
-                except Author.MultipleObjectsReturned:
-                    # If multiple authors are found, delete all but one
-                    Author.objects.filter(name__iexact=x).exclude(
-                        id=Author.objects.filter(name__iexact=x).first().id
-                    ).delete()
-    authors = re_authors
-
     if album:
         if type(album) is str:
             album_name = album
@@ -108,6 +99,13 @@ def load_track(
                 name__iexact=album_name, defaults={"name": album_name}
             )
 
+    processed_authors = []
+    if authors:
+        for author_name in authors:
+            author = get_or_create_author(author_name)
+            processed_authors.append(author)
+    authors = processed_authors
+
     if sng := Song.objects.filter(
         name=name if name else p_name,
         authors__id__in=[x.id for x in authors],
@@ -122,6 +120,14 @@ def load_track(
         path = mp3_path
 
     tag = MP3(path, ID3=ID3)
+
+    if image_path and image_path.startswith("http"):
+        response = requests.get(image_path)
+        se = image_path.split("/")[-1]
+        image_path = f'/tmp/{generate_readable_slug(name, Song)}.{"png" if "." not in se else se.split(".")[-1]}'
+        with open(image_path, "wb") as f:
+            f.write(response.content)
+
     if image_path:
         if not image_path.endswith(".png"):
             nm = image_path

akarpov/music/services/info.py

@@ -6,6 +6,7 @@
 from deep_translator import GoogleTranslator
 from django.conf import settings
 from django.core.files import File
+from django.db import transaction
 from django.utils.text import slugify
 from spotipy import SpotifyClientCredentials
 from yandex_music import Client, Cover
@@ -321,7 +322,8 @@ def update_author_info(author: Author) -> None:
         )
 
     author.meta = author_data
-    author.save()
+    with transaction.atomic():
+        author.save()
 
     # Handle Author Image - Prefer Spotify, fallback to Yandex
     image_path = None
@@ -353,7 +355,8 @@ def update_author_info(author: Author) -> None:
         author.save()
 
     author.slug = generate_readable_slug(author.name, Author)
-    author.save()
+    with transaction.atomic():
+        author.save()
 
 
 def search_all_platforms(track_name: str) -> dict:

akarpov/music/services/spotify.py

@@ -1,7 +1,12 @@
+import threading
+
 import spotipy
 from django.conf import settings
+from spotdl import Song, Spotdl
 from spotipy.oauth2 import SpotifyClientCredentials
 
+from akarpov.music.services.db import load_track
+
 
 def create_session() -> spotipy.Spotify:
     if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
@@ -18,3 +23,72 @@ def create_session() -> spotipy.Spotify:
 def search(name: str, session: spotipy.Spotify, search_type="track"):
     res = session.search(name, type=search_type)
     return res
+
+
+thread_local = threading.local()
+
+
+def get_spotdl_client():
+    if not hasattr(thread_local, "spotdl_client"):
+        spot_settings = {
+            "simple_tui": True,
+            "log_level": "ERROR",
+            "lyrics_providers": ["genius", "azlyrics", "musixmatch"],
+            "threads": 6,
+            "format": "mp3",
+            "ffmpeg": "ffmpeg",
+            "sponsor_block": True,
+        }
+        thread_local.spotdl_client = Spotdl(
+            client_id=settings.MUSIC_SPOTIFY_ID,
+            client_secret=settings.MUSIC_SPOTIFY_SECRET,
+            user_auth=False,
+            headless=False,
+            downloader_settings=spot_settings,
+        )
+    return thread_local.spotdl_client
+
+
+def download_url(url, user_id=None):
+    spotdl_client = get_spotdl_client()
+    session = create_session()
+
+    if "track" in url:
+        songs = [Song.from_url(url)]
+    elif "album" in url:
+        album_tracks = session.album(url)["tracks"]["items"]
+        songs = [
+            Song.from_url(track["external_urls"]["spotify"]) for track in album_tracks
+        ]
+    elif "artist" in url:
+        artist_top_tracks = session.artist_top_tracks(url)["tracks"]
+        songs = [
+            Song.from_url(track["external_urls"]["spotify"])
+            for track in artist_top_tracks
+        ]
+    elif "playlist" in url:
+        playlist_tracks = session.playlist_items(url)["items"]
+        songs = [
+            Song.from_url(track["track"]["external_urls"]["spotify"])
+            for track in playlist_tracks
+        ]
+    else:
+        return None
+
+    for song in songs:
+        res = spotdl_client.download(song)
+        if res:
+            song, path = res
+        else:
+            return None
+        load_track(
+            path=str(path),
+            image_path=song.cover_url,
+            user_id=user_id,
+            authors=song.artists,
+            album=song.album_name,
+            name=song.name,
+            link=song.url,
+            genre=song.genres[0] if song.genres else None,
+            release=song.date,
+        )

akarpov/music/services/youtube.py

@@ -10,6 +10,7 @@
 from PIL import Image
 from pydub import AudioSegment
 from pytube import Search, YouTube
+from spotdl.providers.audio import YouTubeMusic
 
 from akarpov.music.models import Song
 from akarpov.music.services.db import load_track
@@ -18,22 +19,28 @@
 final_filename = None
 
 
-ydl_opts = {
+ytmusic = YouTubeMusic()
-    "format": "m4a/bestaudio/best",
-    "postprocessors": [
-        {  # Extract audio using ffmpeg
-            "key": "FFmpegExtractAudio",
-            "preferredcodec": "m4a",
-        }
-    ],
-    "outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
-}
 
 
 def download_file(url):
+    ydl_opts = {
+        "format": "bestaudio/best",
+        "outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
+        "postprocessors": [
+            {"key": "SponsorBlock"},  # Skip sponsor segments
+            {
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "mp3",
+                "preferredquality": "192",
+            },  # Extract audio
+            {"key": "EmbedThumbnail"},  # Embed Thumbnail
+            {"key": "FFmpegMetadata"},  # Apply correct metadata
+        ],
+    }
     with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        info = ydl.extract_info(url)
+        info = ydl.extract_info(url, download=True)
-    return info["requested_downloads"][0]["_filename"]
+        filename = ydl.prepare_filename(info)
+    return os.path.splitext(filename)[0] + ".mp3"
 
 
 def parse_description(description: str) -> list:
@@ -67,7 +74,7 @@ def parse_description(description: str) -> list:
 def download_from_youtube_link(link: str, user_id: int) -> Song:
     song = None
 
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+    with yt_dlp.YoutubeDL({"ignoreerrors": True, "extract_flat": True}) as ydl:
         info_dict = ydl.extract_info(link, download=False)
         title = info_dict.get("title", None)
         description = info_dict.get("description", None)
@@ -82,9 +89,12 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
         + slugify(orig_path.split("/")[-1].split(".")[0])
         + ".mp3"
     )
-    AudioSegment.from_file(orig_path).export(path)
+    if orig_path.endswith(".mp3"):
-    if orig_path != path:
+        os.rename(orig_path, path)
-        os.remove(orig_path)
+    else:
+        AudioSegment.from_file(orig_path).export(path)
+        if orig_path != path:
+            os.remove(orig_path)
     print(f"[processing] {title} converting to mp3: done")
 
     # split in chapters

akarpov/music/tasks.py

@@ -1,14 +1,16 @@
 from datetime import timedelta
 
 import pylast
+import spotipy
 import structlog
+import ytmusicapi
 from asgiref.sync import async_to_sync
 from celery import shared_task
 from channels.layers import get_channel_layer
 from django.conf import settings
 from django.utils import timezone
 from django.utils.timezone import now
-from pytube import Channel, Playlist
+from spotipy import SpotifyClientCredentials
 
 from akarpov.music.api.serializers import SongSerializer
 from akarpov.music.models import (
@@ -19,7 +21,7 @@
     UserListenHistory,
     UserMusicProfile,
 )
-from akarpov.music.services import yandex, youtube
+from akarpov.music.services import spotify, yandex, youtube
 from akarpov.music.services.file import load_dir, load_file
 from akarpov.utils.celery import get_scheduled_tasks_name
 
@@ -28,18 +30,57 @@
 
 @shared_task
 def list_tracks(url, user_id):
-    if "music.yandex.ru" in url:
+    if "music.youtube.com" in url or "youtu.be" in url:
+        url = url.replace("music.youtube.com", "youtube.com")
+        url = url.replace("youtu.be", "youtube.com")
+    if "spotify.com" in url:
+        spotify.download_url(url, user_id)
+    elif "music.yandex.ru" in url:
         yandex.load_playlist(url, user_id)
-    elif "channel" in url or "/c/" in url:
+    if "youtube.com" in url:
-        p = Channel(url)
+        if "channel" in url or "/c/" in url:
-        for video in p.video_urls:
+            ytmusic = ytmusicapi.YTMusic()
-            process_yb.apply_async(kwargs={"url": video, "user_id": user_id})
+            channel_id = url.split("/")[-1]
-    elif "playlist" in url or "&list=" in url:
+            channel_songs = ytmusic.get_artist(channel_id)["songs"]["results"]
-        p = Playlist(url)
+            print(channel_songs)
-        for video in p.video_urls:
+
-            process_yb.apply_async(kwargs={"url": video, "user_id": user_id})
+            for song in channel_songs:
+                process_yb.apply_async(
+                    kwargs={
+                        "url": f"https://youtube.com/watch?v={song['videoId']}",
+                        "user_id": user_id,
+                    }
+                )
+
+        elif "playlist" in url or "&list=" in url:
+            ytmusic = ytmusicapi.YTMusic()
+            playlist_id = url.split("=")[-1]
+            playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
+
+            for song in playlist_songs:
+                process_yb.apply_async(
+                    kwargs={
+                        "url": f"https://music.youtube.com/watch?v={song['videoId']}",
+                        "user_id": user_id,
+                    }
+                )
+        else:
+            process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
     else:
-        process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
+        spotify_manager = SpotifyClientCredentials(
+            client_id=settings.MUSIC_SPOTIFY_ID,
+            client_secret=settings.MUSIC_SPOTIFY_SECRET,
+        )
+        spotify_search = spotipy.Spotify(client_credentials_manager=spotify_manager)
+
+        results = spotify_search.search(q=url, type="track", limit=1)
+        top_track = (
+            results["tracks"]["items"][0] if results["tracks"]["items"] else None
+        )
+
+        if top_track:
+            spotify.download_url(top_track["external_urls"]["spotify"], user_id)
+            url = top_track["external_urls"]["spotify"]
 
     return url
 

akarpov/tools/shortener/models.py

@@ -1,7 +1,7 @@
 from abc import abstractmethod
 
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from django.urls import reverse
 from model_utils.models import TimeStampedModel
 
@@ -79,7 +79,8 @@ def create_model_link(sender, instance, created, **kwargs):
 
         link.save()
         instance.short_link = link
-        instance.save()
+        with transaction.atomic():
+            instance.save()
 
 
 def update_model_link(sender, instance, **kwargs):

compose/local/django/Dockerfile

@@ -33,6 +33,13 @@ RUN apt-get update && \
     apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
     rm -rf /var/lib/apt/lists/*
 
+# Make ssh dir
+RUN mkdir -p /root/.ssh/
+
+# Create known_hosts and add github to it
+RUN touch /root/.ssh/known_hosts
+RUN ssh-keyscan -t rsa github.com >> /root/.ssh/known_hosts
+
 RUN pip install "poetry==$POETRY_VERSION"
 RUN python -m venv /venv
 

pyproject.toml

@@ -119,7 +119,7 @@ spotdl = "^4.2.4"
 fuzzywuzzy = "^0.18.0"
 python-levenshtein = "^0.23.0"
 pylast = "^5.2.0"
-textract = {git = "https://github.com/Alexander-D-Karpov/textract.git", branch = "master"}
+textract = {git = "https://github.com/Alexander-D-Karpov/textract", branch = "master"}
 librosa = "^0.10.1"