mirror of
https://github.com/Alexander-D-Karpov/akarpov
synced 2024-11-14 20:16:34 +03:00
Compare commits
11 Commits
b7110ec83d
...
1501e0bdf5
Author | SHA1 | Date | |
---|---|---|---|
|
1501e0bdf5 | ||
6a7e7d5ade | |||
1524791779 | |||
6d9edbf95d | |||
6a21158a62 | |||
ffa1e9c69f | |||
a87385db78 | |||
aa49e4afc3 | |||
a309d5653d | |||
c81b387689 | |||
db72084d64 |
|
@ -63,7 +63,7 @@ def filter(self, queryset):
|
||||||
|
|
||||||
if search_type in search_classes:
|
if search_type in search_classes:
|
||||||
search_instance = search_classes[search_type](
|
search_instance = search_classes[search_type](
|
||||||
queryset=File.objects.filter(user=self.request.user)
|
queryset=File.objects.filter(user=self.request.user).nocache()
|
||||||
)
|
)
|
||||||
queryset = search_instance.search(query)
|
queryset = search_instance.search(query)
|
||||||
return queryset
|
return queryset
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
import requests
|
||||||
from deep_translator import GoogleTranslator
|
from deep_translator import GoogleTranslator
|
||||||
from django.core.files import File
|
from django.core.files import File
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
|
@ -16,6 +17,14 @@
|
||||||
from akarpov.users.models import User
|
from akarpov.users.models import User
|
||||||
|
|
||||||
|
|
||||||
|
def get_or_create_author(author_name):
|
||||||
|
with transaction.atomic():
|
||||||
|
author = Author.objects.filter(name__iexact=author_name).order_by("id").first()
|
||||||
|
if author is None:
|
||||||
|
author = Author.objects.create(name=author_name)
|
||||||
|
return author
|
||||||
|
|
||||||
|
|
||||||
def process_track_name(track_name: str) -> str:
|
def process_track_name(track_name: str) -> str:
|
||||||
# Split the track name by dash and parentheses
|
# Split the track name by dash and parentheses
|
||||||
parts = track_name.split(" - ")
|
parts = track_name.split(" - ")
|
||||||
|
@ -78,24 +87,6 @@ def load_track(
|
||||||
if album and type(album) is str and album.startswith("['"):
|
if album and type(album) is str and album.startswith("['"):
|
||||||
album = album.replace("['", "").replace("']", "")
|
album = album.replace("['", "").replace("']", "")
|
||||||
|
|
||||||
re_authors = []
|
|
||||||
if authors:
|
|
||||||
for x in authors:
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
with transaction.atomic():
|
|
||||||
author, created = Author.objects.get_or_create(
|
|
||||||
name__iexact=x, defaults={"name": x}
|
|
||||||
)
|
|
||||||
re_authors.append(author)
|
|
||||||
break
|
|
||||||
except Author.MultipleObjectsReturned:
|
|
||||||
# If multiple authors are found, delete all but one
|
|
||||||
Author.objects.filter(name__iexact=x).exclude(
|
|
||||||
id=Author.objects.filter(name__iexact=x).first().id
|
|
||||||
).delete()
|
|
||||||
authors = re_authors
|
|
||||||
|
|
||||||
if album:
|
if album:
|
||||||
if type(album) is str:
|
if type(album) is str:
|
||||||
album_name = album
|
album_name = album
|
||||||
|
@ -108,6 +99,13 @@ def load_track(
|
||||||
name__iexact=album_name, defaults={"name": album_name}
|
name__iexact=album_name, defaults={"name": album_name}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
processed_authors = []
|
||||||
|
if authors:
|
||||||
|
for author_name in authors:
|
||||||
|
author = get_or_create_author(author_name)
|
||||||
|
processed_authors.append(author)
|
||||||
|
authors = processed_authors
|
||||||
|
|
||||||
if sng := Song.objects.filter(
|
if sng := Song.objects.filter(
|
||||||
name=name if name else p_name,
|
name=name if name else p_name,
|
||||||
authors__id__in=[x.id for x in authors],
|
authors__id__in=[x.id for x in authors],
|
||||||
|
@ -122,6 +120,14 @@ def load_track(
|
||||||
path = mp3_path
|
path = mp3_path
|
||||||
|
|
||||||
tag = MP3(path, ID3=ID3)
|
tag = MP3(path, ID3=ID3)
|
||||||
|
|
||||||
|
if image_path and image_path.startswith("http"):
|
||||||
|
response = requests.get(image_path)
|
||||||
|
se = image_path.split("/")[-1]
|
||||||
|
image_path = f'/tmp/{generate_readable_slug(name, Song)}.{"png" if "." not in se else se.split(".")[-1]}'
|
||||||
|
with open(image_path, "wb") as f:
|
||||||
|
f.write(response.content)
|
||||||
|
|
||||||
if image_path:
|
if image_path:
|
||||||
if not image_path.endswith(".png"):
|
if not image_path.endswith(".png"):
|
||||||
nm = image_path
|
nm = image_path
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
from deep_translator import GoogleTranslator
|
from deep_translator import GoogleTranslator
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.files import File
|
from django.core.files import File
|
||||||
|
from django.db import transaction
|
||||||
from django.utils.text import slugify
|
from django.utils.text import slugify
|
||||||
from spotipy import SpotifyClientCredentials
|
from spotipy import SpotifyClientCredentials
|
||||||
from yandex_music import Client, Cover
|
from yandex_music import Client, Cover
|
||||||
|
@ -321,7 +322,8 @@ def update_author_info(author: Author) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
author.meta = author_data
|
author.meta = author_data
|
||||||
author.save()
|
with transaction.atomic():
|
||||||
|
author.save()
|
||||||
|
|
||||||
# Handle Author Image - Prefer Spotify, fallback to Yandex
|
# Handle Author Image - Prefer Spotify, fallback to Yandex
|
||||||
image_path = None
|
image_path = None
|
||||||
|
@ -353,7 +355,8 @@ def update_author_info(author: Author) -> None:
|
||||||
author.save()
|
author.save()
|
||||||
|
|
||||||
author.slug = generate_readable_slug(author.name, Author)
|
author.slug = generate_readable_slug(author.name, Author)
|
||||||
author.save()
|
with transaction.atomic():
|
||||||
|
author.save()
|
||||||
|
|
||||||
|
|
||||||
def search_all_platforms(track_name: str) -> dict:
|
def search_all_platforms(track_name: str) -> dict:
|
||||||
|
|
|
@ -1,7 +1,12 @@
|
||||||
|
import threading
|
||||||
|
|
||||||
import spotipy
|
import spotipy
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
from spotdl import Song, Spotdl
|
||||||
from spotipy.oauth2 import SpotifyClientCredentials
|
from spotipy.oauth2 import SpotifyClientCredentials
|
||||||
|
|
||||||
|
from akarpov.music.services.db import load_track
|
||||||
|
|
||||||
|
|
||||||
def create_session() -> spotipy.Spotify:
|
def create_session() -> spotipy.Spotify:
|
||||||
if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
|
if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
|
||||||
|
@ -18,3 +23,72 @@ def create_session() -> spotipy.Spotify:
|
||||||
def search(name: str, session: spotipy.Spotify, search_type="track"):
|
def search(name: str, session: spotipy.Spotify, search_type="track"):
|
||||||
res = session.search(name, type=search_type)
|
res = session.search(name, type=search_type)
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
thread_local = threading.local()
|
||||||
|
|
||||||
|
|
||||||
|
def get_spotdl_client():
|
||||||
|
if not hasattr(thread_local, "spotdl_client"):
|
||||||
|
spot_settings = {
|
||||||
|
"simple_tui": True,
|
||||||
|
"log_level": "ERROR",
|
||||||
|
"lyrics_providers": ["genius", "azlyrics", "musixmatch"],
|
||||||
|
"threads": 6,
|
||||||
|
"format": "mp3",
|
||||||
|
"ffmpeg": "ffmpeg",
|
||||||
|
"sponsor_block": True,
|
||||||
|
}
|
||||||
|
thread_local.spotdl_client = Spotdl(
|
||||||
|
client_id=settings.MUSIC_SPOTIFY_ID,
|
||||||
|
client_secret=settings.MUSIC_SPOTIFY_SECRET,
|
||||||
|
user_auth=False,
|
||||||
|
headless=False,
|
||||||
|
downloader_settings=spot_settings,
|
||||||
|
)
|
||||||
|
return thread_local.spotdl_client
|
||||||
|
|
||||||
|
|
||||||
|
def download_url(url, user_id=None):
|
||||||
|
spotdl_client = get_spotdl_client()
|
||||||
|
session = create_session()
|
||||||
|
|
||||||
|
if "track" in url:
|
||||||
|
songs = [Song.from_url(url)]
|
||||||
|
elif "album" in url:
|
||||||
|
album_tracks = session.album(url)["tracks"]["items"]
|
||||||
|
songs = [
|
||||||
|
Song.from_url(track["external_urls"]["spotify"]) for track in album_tracks
|
||||||
|
]
|
||||||
|
elif "artist" in url:
|
||||||
|
artist_top_tracks = session.artist_top_tracks(url)["tracks"]
|
||||||
|
songs = [
|
||||||
|
Song.from_url(track["external_urls"]["spotify"])
|
||||||
|
for track in artist_top_tracks
|
||||||
|
]
|
||||||
|
elif "playlist" in url:
|
||||||
|
playlist_tracks = session.playlist_items(url)["items"]
|
||||||
|
songs = [
|
||||||
|
Song.from_url(track["track"]["external_urls"]["spotify"])
|
||||||
|
for track in playlist_tracks
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
for song in songs:
|
||||||
|
res = spotdl_client.download(song)
|
||||||
|
if res:
|
||||||
|
song, path = res
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
load_track(
|
||||||
|
path=str(path),
|
||||||
|
image_path=song.cover_url,
|
||||||
|
user_id=user_id,
|
||||||
|
authors=song.artists,
|
||||||
|
album=song.album_name,
|
||||||
|
name=song.name,
|
||||||
|
link=song.url,
|
||||||
|
genre=song.genres[0] if song.genres else None,
|
||||||
|
release=song.date,
|
||||||
|
)
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from pydub import AudioSegment
|
from pydub import AudioSegment
|
||||||
from pytube import Search, YouTube
|
from pytube import Search, YouTube
|
||||||
|
from spotdl.providers.audio import YouTubeMusic
|
||||||
|
|
||||||
from akarpov.music.models import Song
|
from akarpov.music.models import Song
|
||||||
from akarpov.music.services.db import load_track
|
from akarpov.music.services.db import load_track
|
||||||
|
@ -18,22 +19,28 @@
|
||||||
final_filename = None
|
final_filename = None
|
||||||
|
|
||||||
|
|
||||||
ydl_opts = {
|
ytmusic = YouTubeMusic()
|
||||||
"format": "m4a/bestaudio/best",
|
|
||||||
"postprocessors": [
|
|
||||||
{ # Extract audio using ffmpeg
|
|
||||||
"key": "FFmpegExtractAudio",
|
|
||||||
"preferredcodec": "m4a",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def download_file(url):
|
def download_file(url):
|
||||||
|
ydl_opts = {
|
||||||
|
"format": "bestaudio/best",
|
||||||
|
"outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
|
||||||
|
"postprocessors": [
|
||||||
|
{"key": "SponsorBlock"}, # Skip sponsor segments
|
||||||
|
{
|
||||||
|
"key": "FFmpegExtractAudio",
|
||||||
|
"preferredcodec": "mp3",
|
||||||
|
"preferredquality": "192",
|
||||||
|
}, # Extract audio
|
||||||
|
{"key": "EmbedThumbnail"}, # Embed Thumbnail
|
||||||
|
{"key": "FFmpegMetadata"}, # Apply correct metadata
|
||||||
|
],
|
||||||
|
}
|
||||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||||
info = ydl.extract_info(url)
|
info = ydl.extract_info(url, download=True)
|
||||||
return info["requested_downloads"][0]["_filename"]
|
filename = ydl.prepare_filename(info)
|
||||||
|
return os.path.splitext(filename)[0] + ".mp3"
|
||||||
|
|
||||||
|
|
||||||
def parse_description(description: str) -> list:
|
def parse_description(description: str) -> list:
|
||||||
|
@ -67,7 +74,7 @@ def parse_description(description: str) -> list:
|
||||||
def download_from_youtube_link(link: str, user_id: int) -> Song:
|
def download_from_youtube_link(link: str, user_id: int) -> Song:
|
||||||
song = None
|
song = None
|
||||||
|
|
||||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
with yt_dlp.YoutubeDL({"ignoreerrors": True, "extract_flat": True}) as ydl:
|
||||||
info_dict = ydl.extract_info(link, download=False)
|
info_dict = ydl.extract_info(link, download=False)
|
||||||
title = info_dict.get("title", None)
|
title = info_dict.get("title", None)
|
||||||
description = info_dict.get("description", None)
|
description = info_dict.get("description", None)
|
||||||
|
@ -82,9 +89,12 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
|
||||||
+ slugify(orig_path.split("/")[-1].split(".")[0])
|
+ slugify(orig_path.split("/")[-1].split(".")[0])
|
||||||
+ ".mp3"
|
+ ".mp3"
|
||||||
)
|
)
|
||||||
AudioSegment.from_file(orig_path).export(path)
|
if orig_path.endswith(".mp3"):
|
||||||
if orig_path != path:
|
os.rename(orig_path, path)
|
||||||
os.remove(orig_path)
|
else:
|
||||||
|
AudioSegment.from_file(orig_path).export(path)
|
||||||
|
if orig_path != path:
|
||||||
|
os.remove(orig_path)
|
||||||
print(f"[processing] {title} converting to mp3: done")
|
print(f"[processing] {title} converting to mp3: done")
|
||||||
|
|
||||||
# split in chapters
|
# split in chapters
|
||||||
|
|
|
@ -1,14 +1,16 @@
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
import pylast
|
import pylast
|
||||||
|
import spotipy
|
||||||
import structlog
|
import structlog
|
||||||
|
import ytmusicapi
|
||||||
from asgiref.sync import async_to_sync
|
from asgiref.sync import async_to_sync
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from channels.layers import get_channel_layer
|
from channels.layers import get_channel_layer
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from django.utils.timezone import now
|
from django.utils.timezone import now
|
||||||
from pytube import Channel, Playlist
|
from spotipy import SpotifyClientCredentials
|
||||||
|
|
||||||
from akarpov.music.api.serializers import SongSerializer
|
from akarpov.music.api.serializers import SongSerializer
|
||||||
from akarpov.music.models import (
|
from akarpov.music.models import (
|
||||||
|
@ -19,7 +21,7 @@
|
||||||
UserListenHistory,
|
UserListenHistory,
|
||||||
UserMusicProfile,
|
UserMusicProfile,
|
||||||
)
|
)
|
||||||
from akarpov.music.services import yandex, youtube
|
from akarpov.music.services import spotify, yandex, youtube
|
||||||
from akarpov.music.services.file import load_dir, load_file
|
from akarpov.music.services.file import load_dir, load_file
|
||||||
from akarpov.utils.celery import get_scheduled_tasks_name
|
from akarpov.utils.celery import get_scheduled_tasks_name
|
||||||
|
|
||||||
|
@ -28,18 +30,57 @@
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def list_tracks(url, user_id):
|
def list_tracks(url, user_id):
|
||||||
if "music.yandex.ru" in url:
|
if "music.youtube.com" in url or "youtu.be" in url:
|
||||||
|
url = url.replace("music.youtube.com", "youtube.com")
|
||||||
|
url = url.replace("youtu.be", "youtube.com")
|
||||||
|
if "spotify.com" in url:
|
||||||
|
spotify.download_url(url, user_id)
|
||||||
|
elif "music.yandex.ru" in url:
|
||||||
yandex.load_playlist(url, user_id)
|
yandex.load_playlist(url, user_id)
|
||||||
elif "channel" in url or "/c/" in url:
|
if "youtube.com" in url:
|
||||||
p = Channel(url)
|
if "channel" in url or "/c/" in url:
|
||||||
for video in p.video_urls:
|
ytmusic = ytmusicapi.YTMusic()
|
||||||
process_yb.apply_async(kwargs={"url": video, "user_id": user_id})
|
channel_id = url.split("/")[-1]
|
||||||
elif "playlist" in url or "&list=" in url:
|
channel_songs = ytmusic.get_artist(channel_id)["songs"]["results"]
|
||||||
p = Playlist(url)
|
print(channel_songs)
|
||||||
for video in p.video_urls:
|
|
||||||
process_yb.apply_async(kwargs={"url": video, "user_id": user_id})
|
for song in channel_songs:
|
||||||
|
process_yb.apply_async(
|
||||||
|
kwargs={
|
||||||
|
"url": f"https://youtube.com/watch?v={song['videoId']}",
|
||||||
|
"user_id": user_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
elif "playlist" in url or "&list=" in url:
|
||||||
|
ytmusic = ytmusicapi.YTMusic()
|
||||||
|
playlist_id = url.split("=")[-1]
|
||||||
|
playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
|
||||||
|
|
||||||
|
for song in playlist_songs:
|
||||||
|
process_yb.apply_async(
|
||||||
|
kwargs={
|
||||||
|
"url": f"https://music.youtube.com/watch?v={song['videoId']}",
|
||||||
|
"user_id": user_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
|
||||||
else:
|
else:
|
||||||
process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
|
spotify_manager = SpotifyClientCredentials(
|
||||||
|
client_id=settings.MUSIC_SPOTIFY_ID,
|
||||||
|
client_secret=settings.MUSIC_SPOTIFY_SECRET,
|
||||||
|
)
|
||||||
|
spotify_search = spotipy.Spotify(client_credentials_manager=spotify_manager)
|
||||||
|
|
||||||
|
results = spotify_search.search(q=url, type="track", limit=1)
|
||||||
|
top_track = (
|
||||||
|
results["tracks"]["items"][0] if results["tracks"]["items"] else None
|
||||||
|
)
|
||||||
|
|
||||||
|
if top_track:
|
||||||
|
spotify.download_url(top_track["external_urls"]["spotify"], user_id)
|
||||||
|
url = top_track["external_urls"]["spotify"]
|
||||||
|
|
||||||
return url
|
return url
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db import models
|
from django.db import models, transaction
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
from model_utils.models import TimeStampedModel
|
from model_utils.models import TimeStampedModel
|
||||||
|
|
||||||
|
@ -79,7 +79,8 @@ def create_model_link(sender, instance, created, **kwargs):
|
||||||
|
|
||||||
link.save()
|
link.save()
|
||||||
instance.short_link = link
|
instance.short_link = link
|
||||||
instance.save()
|
with transaction.atomic():
|
||||||
|
instance.save()
|
||||||
|
|
||||||
|
|
||||||
def update_model_link(sender, instance, **kwargs):
|
def update_model_link(sender, instance, **kwargs):
|
||||||
|
|
|
@ -33,6 +33,13 @@ RUN apt-get update && \
|
||||||
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
|
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Make ssh dir
|
||||||
|
RUN mkdir -p /root/.ssh/
|
||||||
|
|
||||||
|
# Create known_hosts and add github to it
|
||||||
|
RUN touch /root/.ssh/known_hosts
|
||||||
|
RUN ssh-keyscan -t rsa github.com >> /root/.ssh/known_hosts
|
||||||
|
|
||||||
RUN pip install "poetry==$POETRY_VERSION"
|
RUN pip install "poetry==$POETRY_VERSION"
|
||||||
RUN python -m venv /venv
|
RUN python -m venv /venv
|
||||||
|
|
||||||
|
|
|
@ -119,7 +119,7 @@ spotdl = "^4.2.4"
|
||||||
fuzzywuzzy = "^0.18.0"
|
fuzzywuzzy = "^0.18.0"
|
||||||
python-levenshtein = "^0.23.0"
|
python-levenshtein = "^0.23.0"
|
||||||
pylast = "^5.2.0"
|
pylast = "^5.2.0"
|
||||||
textract = {git = "https://github.com/Alexander-D-Karpov/textract.git", branch = "master"}
|
textract = {git = "https://github.com/Alexander-D-Karpov/textract", branch = "master"}
|
||||||
librosa = "^0.10.1"
|
librosa = "^0.10.1"
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user