Compare commits

..

11 Commits

9 changed files with 194 additions and 52 deletions

View File

@ -63,7 +63,7 @@ def filter(self, queryset):
if search_type in search_classes: if search_type in search_classes:
search_instance = search_classes[search_type]( search_instance = search_classes[search_type](
queryset=File.objects.filter(user=self.request.user) queryset=File.objects.filter(user=self.request.user).nocache()
) )
queryset = search_instance.search(query) queryset = search_instance.search(query)
return queryset return queryset

View File

@ -1,6 +1,7 @@
import os import os
import re import re
import requests
from deep_translator import GoogleTranslator from deep_translator import GoogleTranslator
from django.core.files import File from django.core.files import File
from django.db import transaction from django.db import transaction
@ -16,6 +17,14 @@
from akarpov.users.models import User from akarpov.users.models import User
def get_or_create_author(author_name):
with transaction.atomic():
author = Author.objects.filter(name__iexact=author_name).order_by("id").first()
if author is None:
author = Author.objects.create(name=author_name)
return author
def process_track_name(track_name: str) -> str: def process_track_name(track_name: str) -> str:
# Split the track name by dash and parentheses # Split the track name by dash and parentheses
parts = track_name.split(" - ") parts = track_name.split(" - ")
@ -78,24 +87,6 @@ def load_track(
if album and type(album) is str and album.startswith("['"): if album and type(album) is str and album.startswith("['"):
album = album.replace("['", "").replace("']", "") album = album.replace("['", "").replace("']", "")
re_authors = []
if authors:
for x in authors:
while True:
try:
with transaction.atomic():
author, created = Author.objects.get_or_create(
name__iexact=x, defaults={"name": x}
)
re_authors.append(author)
break
except Author.MultipleObjectsReturned:
# If multiple authors are found, delete all but one
Author.objects.filter(name__iexact=x).exclude(
id=Author.objects.filter(name__iexact=x).first().id
).delete()
authors = re_authors
if album: if album:
if type(album) is str: if type(album) is str:
album_name = album album_name = album
@ -108,6 +99,13 @@ def load_track(
name__iexact=album_name, defaults={"name": album_name} name__iexact=album_name, defaults={"name": album_name}
) )
processed_authors = []
if authors:
for author_name in authors:
author = get_or_create_author(author_name)
processed_authors.append(author)
authors = processed_authors
if sng := Song.objects.filter( if sng := Song.objects.filter(
name=name if name else p_name, name=name if name else p_name,
authors__id__in=[x.id for x in authors], authors__id__in=[x.id for x in authors],
@ -122,6 +120,14 @@ def load_track(
path = mp3_path path = mp3_path
tag = MP3(path, ID3=ID3) tag = MP3(path, ID3=ID3)
if image_path and image_path.startswith("http"):
response = requests.get(image_path)
se = image_path.split("/")[-1]
image_path = f'/tmp/{generate_readable_slug(name, Song)}.{"png" if "." not in se else se.split(".")[-1]}'
with open(image_path, "wb") as f:
f.write(response.content)
if image_path: if image_path:
if not image_path.endswith(".png"): if not image_path.endswith(".png"):
nm = image_path nm = image_path

View File

@ -6,6 +6,7 @@
from deep_translator import GoogleTranslator from deep_translator import GoogleTranslator
from django.conf import settings from django.conf import settings
from django.core.files import File from django.core.files import File
from django.db import transaction
from django.utils.text import slugify from django.utils.text import slugify
from spotipy import SpotifyClientCredentials from spotipy import SpotifyClientCredentials
from yandex_music import Client, Cover from yandex_music import Client, Cover
@ -321,7 +322,8 @@ def update_author_info(author: Author) -> None:
) )
author.meta = author_data author.meta = author_data
author.save() with transaction.atomic():
author.save()
# Handle Author Image - Prefer Spotify, fallback to Yandex # Handle Author Image - Prefer Spotify, fallback to Yandex
image_path = None image_path = None
@ -353,7 +355,8 @@ def update_author_info(author: Author) -> None:
author.save() author.save()
author.slug = generate_readable_slug(author.name, Author) author.slug = generate_readable_slug(author.name, Author)
author.save() with transaction.atomic():
author.save()
def search_all_platforms(track_name: str) -> dict: def search_all_platforms(track_name: str) -> dict:

View File

@ -1,7 +1,12 @@
import threading
import spotipy import spotipy
from django.conf import settings from django.conf import settings
from spotdl import Song, Spotdl
from spotipy.oauth2 import SpotifyClientCredentials from spotipy.oauth2 import SpotifyClientCredentials
from akarpov.music.services.db import load_track
def create_session() -> spotipy.Spotify: def create_session() -> spotipy.Spotify:
if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET: if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
@ -18,3 +23,72 @@ def create_session() -> spotipy.Spotify:
def search(name: str, session: spotipy.Spotify, search_type="track"): def search(name: str, session: spotipy.Spotify, search_type="track"):
res = session.search(name, type=search_type) res = session.search(name, type=search_type)
return res return res
thread_local = threading.local()
def get_spotdl_client():
if not hasattr(thread_local, "spotdl_client"):
spot_settings = {
"simple_tui": True,
"log_level": "ERROR",
"lyrics_providers": ["genius", "azlyrics", "musixmatch"],
"threads": 6,
"format": "mp3",
"ffmpeg": "ffmpeg",
"sponsor_block": True,
}
thread_local.spotdl_client = Spotdl(
client_id=settings.MUSIC_SPOTIFY_ID,
client_secret=settings.MUSIC_SPOTIFY_SECRET,
user_auth=False,
headless=False,
downloader_settings=spot_settings,
)
return thread_local.spotdl_client
def download_url(url, user_id=None):
spotdl_client = get_spotdl_client()
session = create_session()
if "track" in url:
songs = [Song.from_url(url)]
elif "album" in url:
album_tracks = session.album(url)["tracks"]["items"]
songs = [
Song.from_url(track["external_urls"]["spotify"]) for track in album_tracks
]
elif "artist" in url:
artist_top_tracks = session.artist_top_tracks(url)["tracks"]
songs = [
Song.from_url(track["external_urls"]["spotify"])
for track in artist_top_tracks
]
elif "playlist" in url:
playlist_tracks = session.playlist_items(url)["items"]
songs = [
Song.from_url(track["track"]["external_urls"]["spotify"])
for track in playlist_tracks
]
else:
return None
for song in songs:
res = spotdl_client.download(song)
if res:
song, path = res
else:
return None
load_track(
path=str(path),
image_path=song.cover_url,
user_id=user_id,
authors=song.artists,
album=song.album_name,
name=song.name,
link=song.url,
genre=song.genres[0] if song.genres else None,
release=song.date,
)

View File

@ -10,6 +10,7 @@
from PIL import Image from PIL import Image
from pydub import AudioSegment from pydub import AudioSegment
from pytube import Search, YouTube from pytube import Search, YouTube
from spotdl.providers.audio import YouTubeMusic
from akarpov.music.models import Song from akarpov.music.models import Song
from akarpov.music.services.db import load_track from akarpov.music.services.db import load_track
@ -18,22 +19,28 @@
final_filename = None final_filename = None
ydl_opts = { ytmusic = YouTubeMusic()
"format": "m4a/bestaudio/best",
"postprocessors": [
{ # Extract audio using ffmpeg
"key": "FFmpegExtractAudio",
"preferredcodec": "m4a",
}
],
"outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
}
def download_file(url): def download_file(url):
ydl_opts = {
"format": "bestaudio/best",
"outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
"postprocessors": [
{"key": "SponsorBlock"}, # Skip sponsor segments
{
"key": "FFmpegExtractAudio",
"preferredcodec": "mp3",
"preferredquality": "192",
}, # Extract audio
{"key": "EmbedThumbnail"}, # Embed Thumbnail
{"key": "FFmpegMetadata"}, # Apply correct metadata
],
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(url) info = ydl.extract_info(url, download=True)
return info["requested_downloads"][0]["_filename"] filename = ydl.prepare_filename(info)
return os.path.splitext(filename)[0] + ".mp3"
def parse_description(description: str) -> list: def parse_description(description: str) -> list:
@ -67,7 +74,7 @@ def parse_description(description: str) -> list:
def download_from_youtube_link(link: str, user_id: int) -> Song: def download_from_youtube_link(link: str, user_id: int) -> Song:
song = None song = None
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL({"ignoreerrors": True, "extract_flat": True}) as ydl:
info_dict = ydl.extract_info(link, download=False) info_dict = ydl.extract_info(link, download=False)
title = info_dict.get("title", None) title = info_dict.get("title", None)
description = info_dict.get("description", None) description = info_dict.get("description", None)
@ -82,9 +89,12 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
+ slugify(orig_path.split("/")[-1].split(".")[0]) + slugify(orig_path.split("/")[-1].split(".")[0])
+ ".mp3" + ".mp3"
) )
AudioSegment.from_file(orig_path).export(path) if orig_path.endswith(".mp3"):
if orig_path != path: os.rename(orig_path, path)
os.remove(orig_path) else:
AudioSegment.from_file(orig_path).export(path)
if orig_path != path:
os.remove(orig_path)
print(f"[processing] {title} converting to mp3: done") print(f"[processing] {title} converting to mp3: done")
# split in chapters # split in chapters

View File

@ -1,14 +1,16 @@
from datetime import timedelta from datetime import timedelta
import pylast import pylast
import spotipy
import structlog import structlog
import ytmusicapi
from asgiref.sync import async_to_sync from asgiref.sync import async_to_sync
from celery import shared_task from celery import shared_task
from channels.layers import get_channel_layer from channels.layers import get_channel_layer
from django.conf import settings from django.conf import settings
from django.utils import timezone from django.utils import timezone
from django.utils.timezone import now from django.utils.timezone import now
from pytube import Channel, Playlist from spotipy import SpotifyClientCredentials
from akarpov.music.api.serializers import SongSerializer from akarpov.music.api.serializers import SongSerializer
from akarpov.music.models import ( from akarpov.music.models import (
@ -19,7 +21,7 @@
UserListenHistory, UserListenHistory,
UserMusicProfile, UserMusicProfile,
) )
from akarpov.music.services import yandex, youtube from akarpov.music.services import spotify, yandex, youtube
from akarpov.music.services.file import load_dir, load_file from akarpov.music.services.file import load_dir, load_file
from akarpov.utils.celery import get_scheduled_tasks_name from akarpov.utils.celery import get_scheduled_tasks_name
@ -28,18 +30,57 @@
@shared_task @shared_task
def list_tracks(url, user_id): def list_tracks(url, user_id):
if "music.yandex.ru" in url: if "music.youtube.com" in url or "youtu.be" in url:
url = url.replace("music.youtube.com", "youtube.com")
url = url.replace("youtu.be", "youtube.com")
if "spotify.com" in url:
spotify.download_url(url, user_id)
elif "music.yandex.ru" in url:
yandex.load_playlist(url, user_id) yandex.load_playlist(url, user_id)
elif "channel" in url or "/c/" in url: if "youtube.com" in url:
p = Channel(url) if "channel" in url or "/c/" in url:
for video in p.video_urls: ytmusic = ytmusicapi.YTMusic()
process_yb.apply_async(kwargs={"url": video, "user_id": user_id}) channel_id = url.split("/")[-1]
elif "playlist" in url or "&list=" in url: channel_songs = ytmusic.get_artist(channel_id)["songs"]["results"]
p = Playlist(url) print(channel_songs)
for video in p.video_urls:
process_yb.apply_async(kwargs={"url": video, "user_id": user_id}) for song in channel_songs:
process_yb.apply_async(
kwargs={
"url": f"https://youtube.com/watch?v={song['videoId']}",
"user_id": user_id,
}
)
elif "playlist" in url or "&list=" in url:
ytmusic = ytmusicapi.YTMusic()
playlist_id = url.split("=")[-1]
playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
for song in playlist_songs:
process_yb.apply_async(
kwargs={
"url": f"https://music.youtube.com/watch?v={song['videoId']}",
"user_id": user_id,
}
)
else:
process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
else: else:
process_yb.apply_async(kwargs={"url": url, "user_id": user_id}) spotify_manager = SpotifyClientCredentials(
client_id=settings.MUSIC_SPOTIFY_ID,
client_secret=settings.MUSIC_SPOTIFY_SECRET,
)
spotify_search = spotipy.Spotify(client_credentials_manager=spotify_manager)
results = spotify_search.search(q=url, type="track", limit=1)
top_track = (
results["tracks"]["items"][0] if results["tracks"]["items"] else None
)
if top_track:
spotify.download_url(top_track["external_urls"]["spotify"], user_id)
url = top_track["external_urls"]["spotify"]
return url return url

View File

@ -1,7 +1,7 @@
from abc import abstractmethod from abc import abstractmethod
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models, transaction
from django.urls import reverse from django.urls import reverse
from model_utils.models import TimeStampedModel from model_utils.models import TimeStampedModel
@ -79,7 +79,8 @@ def create_model_link(sender, instance, created, **kwargs):
link.save() link.save()
instance.short_link = link instance.short_link = link
instance.save() with transaction.atomic():
instance.save()
def update_model_link(sender, instance, **kwargs): def update_model_link(sender, instance, **kwargs):

View File

@ -33,6 +33,13 @@ RUN apt-get update && \
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \ apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
# Make ssh dir
RUN mkdir -p /root/.ssh/
# Create known_hosts and add github to it
RUN touch /root/.ssh/known_hosts
RUN ssh-keyscan -t rsa github.com >> /root/.ssh/known_hosts
RUN pip install "poetry==$POETRY_VERSION" RUN pip install "poetry==$POETRY_VERSION"
RUN python -m venv /venv RUN python -m venv /venv

View File

@ -119,7 +119,7 @@ spotdl = "^4.2.4"
fuzzywuzzy = "^0.18.0" fuzzywuzzy = "^0.18.0"
python-levenshtein = "^0.23.0" python-levenshtein = "^0.23.0"
pylast = "^5.2.0" pylast = "^5.2.0"
textract = {git = "https://github.com/Alexander-D-Karpov/textract.git", branch = "master"} textract = {git = "https://github.com/Alexander-D-Karpov/textract", branch = "master"}
librosa = "^0.10.1" librosa = "^0.10.1"