Compare commits

...

18 Commits

Author SHA1 Message Date
dependabot[bot]
c9aa12457f
Merge b7a3a447dc into 6a7e7d5ade 2024-02-01 14:13:08 +00:00
6a7e7d5ade Fixing git error 2024-02-01 17:13:01 +03:00
1524791779 Fixing git error 2024-02-01 17:06:07 +03:00
6d9edbf95d Fixing git error 2024-02-01 17:04:11 +03:00
6a21158a62 Update textract dependency URL in pyproject.toml 2024-02-01 16:55:42 +03:00
ffa1e9c69f Refine YouTube and Spotify music download process 2024-02-01 16:46:45 +03:00
a87385db78 Implemented URL parsing and music identification improvements 2024-02-01 15:41:12 +03:00
aa49e4afc3 update author save 2024-02-01 03:51:44 +03:00
a309d5653d updated cache root 2024-02-01 03:15:22 +03:00
c81b387689 updated cache root 2024-02-01 03:00:27 +03:00
db72084d64 Refactor music service and add Spotify support 2024-02-01 02:47:29 +03:00
0189377aeb fixed track name 2024-01-19 11:11:27 +03:00
d3b1fe5fa1 fixed track name 2024-01-18 23:10:12 +03:00
b76a40aa02 fixed track processing, youtube handling 2024-01-18 22:15:17 +03:00
9c32235926 fixed migrations 2024-01-17 16:15:07 +03:00
7fafb8e484 fixed migrations 2024-01-17 16:14:10 +03:00
c26a2ea8d0 added track slug api view 2024-01-17 16:00:17 +03:00
6f70c38ecf added anon user listen history 2024-01-17 15:43:20 +03:00
18 changed files with 594 additions and 182 deletions

View File

@ -13,7 +13,7 @@ def create_cropped_model_image(sender, instance, created, **kwargs):
"app_label": model._meta.app_label, "app_label": model._meta.app_label,
"model_name": model._meta.model_name, "model_name": model._meta.model_name,
}, },
countdown=2, countdown=5,
) )

View File

@ -5,7 +5,7 @@
from akarpov.utils.files import crop_image from akarpov.utils.files import crop_image
@shared_task() @shared_task(max_retries=3)
def crop_model_image(pk: int, app_label: str, model_name: str): def crop_model_image(pk: int, app_label: str, model_name: str):
model = apps.get_model(app_label=app_label, model_name=model_name) model = apps.get_model(app_label=app_label, model_name=model_name)
instance = model.objects.get(pk=pk) instance = model.objects.get(pk=pk)

View File

@ -63,7 +63,7 @@ def filter(self, queryset):
if search_type in search_classes: if search_type in search_classes:
search_instance = search_classes[search_type]( search_instance = search_classes[search_type](
queryset=File.objects.filter(user=self.request.user) queryset=File.objects.filter(user=self.request.user).nocache()
) )
queryset = search_instance.search(query) queryset = search_instance.search(query)
return queryset return queryset

View File

@ -5,6 +5,7 @@
from akarpov.common.api.serializers import SetUserModelSerializer from akarpov.common.api.serializers import SetUserModelSerializer
from akarpov.music.models import ( from akarpov.music.models import (
Album, Album,
AnonMusicUser,
Author, Author,
Playlist, Playlist,
PlaylistSong, PlaylistSong,
@ -198,6 +199,26 @@ def create(self, validated_data):
return playlist_song return playlist_song
class ListenSongSerializer(serializers.Serializer):
song = serializers.SlugField()
user_id = serializers.CharField(required=False)
def validate(self, attrs):
if not Song.objects.filter(slug=attrs["song"]).exists():
raise serializers.ValidationError("Song not found")
if "user_id" in attrs and attrs["user_id"]:
if not AnonMusicUser.objects.filter(id=attrs["user_id"]).exists():
raise serializers.ValidationError("User not found")
return attrs
class AnonMusicUserSerializer(serializers.ModelSerializer):
class Meta:
model = AnonMusicUser
fields = ["id"]
class LikeDislikeSongSerializer(serializers.ModelSerializer): class LikeDislikeSongSerializer(serializers.ModelSerializer):
song = serializers.SlugField() song = serializers.SlugField()
@ -250,6 +271,10 @@ def create(self, validated_data):
return song_user_rating return song_user_rating
class ListSongSlugsSerializer(serializers.Serializer):
slugs = serializers.ListField(child=serializers.SlugField())
class ListPlaylistSerializer(serializers.ModelSerializer): class ListPlaylistSerializer(serializers.ModelSerializer):
class Meta: class Meta:
model = Playlist model = Playlist

View File

@ -2,6 +2,7 @@
from akarpov.music.api.views import ( from akarpov.music.api.views import (
AddSongToPlaylistAPIView, AddSongToPlaylistAPIView,
CreateAnonMusicUserAPIView,
DislikeSongAPIView, DislikeSongAPIView,
LikeSongAPIView, LikeSongAPIView,
ListAlbumsAPIView, ListAlbumsAPIView,
@ -13,6 +14,7 @@
ListLikedSongsAPIView, ListLikedSongsAPIView,
ListPublicPlaylistAPIView, ListPublicPlaylistAPIView,
ListSongPlaylistsAPIView, ListSongPlaylistsAPIView,
ListSongSlugsAPIView,
ListUserListenedSongsAPIView, ListUserListenedSongsAPIView,
RemoveSongFromPlaylistAPIView, RemoveSongFromPlaylistAPIView,
RetrieveUpdateDestroyAlbumAPIView, RetrieveUpdateDestroyAlbumAPIView,
@ -40,6 +42,7 @@
name="retrieve_update_delete_playlist", name="retrieve_update_delete_playlist",
), ),
path("song/", ListCreateSongAPIView.as_view(), name="list_create_song"), path("song/", ListCreateSongAPIView.as_view(), name="list_create_song"),
path("song/slugs/", ListSongSlugsAPIView.as_view(), name="list_songs_slugs"),
path( path(
"song/<str:slug>", "song/<str:slug>",
RetrieveUpdateDestroySongAPIView.as_view(), RetrieveUpdateDestroySongAPIView.as_view(),
@ -76,4 +79,5 @@
RetrieveUpdateDestroyAuthorAPIView.as_view(), RetrieveUpdateDestroyAuthorAPIView.as_view(),
name="retrieve_update_delete_author", name="retrieve_update_delete_author",
), ),
path("anon/create/", CreateAnonMusicUserAPIView.as_view(), name="create-anon"),
] ]

View File

@ -6,14 +6,17 @@
from akarpov.common.api.permissions import IsAdminOrReadOnly, IsCreatorOrReadOnly from akarpov.common.api.permissions import IsAdminOrReadOnly, IsCreatorOrReadOnly
from akarpov.music.api.serializers import ( from akarpov.music.api.serializers import (
AddSongToPlaylistSerializer, AddSongToPlaylistSerializer,
AnonMusicUserSerializer,
FullAlbumSerializer, FullAlbumSerializer,
FullAuthorSerializer, FullAuthorSerializer,
FullPlaylistSerializer, FullPlaylistSerializer,
LikeDislikeSongSerializer, LikeDislikeSongSerializer,
ListAlbumSerializer, ListAlbumSerializer,
ListAuthorSerializer, ListAuthorSerializer,
ListenSongSerializer,
ListPlaylistSerializer, ListPlaylistSerializer,
ListSongSerializer, ListSongSerializer,
ListSongSlugsSerializer,
PlaylistSerializer, PlaylistSerializer,
SongSerializer, SongSerializer,
) )
@ -78,11 +81,7 @@ def get_queryset(self):
return qs.select_related("creator") return qs.select_related("creator")
class ListCreateSongAPIView(LikedSongsContextMixin, generics.ListCreateAPIView): class ListBaseSongAPIView(generics.ListAPIView):
serializer_class = ListSongSerializer
permission_classes = [IsAdminOrReadOnly]
pagination_class = StandardResultsSetPagination
def get_queryset(self): def get_queryset(self):
search = self.request.query_params.get("search", None) search = self.request.query_params.get("search", None)
if search: if search:
@ -114,6 +113,14 @@ def get_queryset(self):
) )
return qs return qs
class ListCreateSongAPIView(
LikedSongsContextMixin, generics.ListCreateAPIView, ListBaseSongAPIView
):
serializer_class = ListSongSerializer
permission_classes = [IsAdminOrReadOnly]
pagination_class = StandardResultsSetPagination
@extend_schema( @extend_schema(
parameters=[ parameters=[
OpenApiParameter( OpenApiParameter(
@ -166,6 +173,67 @@ def get(self, request, *args, **kwargs):
return self.list(request, *args, **kwargs) return self.list(request, *args, **kwargs)
class ListSongSlugsAPIView(ListBaseSongAPIView):
serializer_class = ListSongSlugsSerializer
permission_classes = [permissions.AllowAny]
@extend_schema(
parameters=[
OpenApiParameter(
name="search",
description="Search query",
required=False,
type=str,
),
OpenApiParameter(
name="sort",
description="Sorting algorithm",
required=False,
type=str,
examples=[
OpenApiExample(
"Default",
description="by date added",
value=None,
),
OpenApiExample(
"played",
description="by total times played",
value="played",
),
OpenApiExample(
"likes",
description="by total likes",
value="likes",
),
OpenApiExample(
"likes reversed",
description="by total likes",
value="-likes",
),
OpenApiExample(
"length",
description="by track length",
value="length",
),
OpenApiExample(
"uploaded",
description="by date uploaded",
value="uploaded",
),
],
),
]
)
def get(self, request, *args, **kwargs):
songs = self.get_queryset()
return Response(
data={
"songs": songs.values_list("slug", flat=True),
}
)
class RetrieveUpdateDestroySongAPIView(generics.RetrieveUpdateDestroyAPIView): class RetrieveUpdateDestroySongAPIView(generics.RetrieveUpdateDestroyAPIView):
lookup_field = "slug" lookup_field = "slug"
lookup_url_kwarg = "slug" lookup_url_kwarg = "slug"
@ -314,7 +382,7 @@ class RetrieveUpdateDestroyAuthorAPIView(
class ListenSongAPIView(generics.GenericAPIView): class ListenSongAPIView(generics.GenericAPIView):
serializer_class = LikeDislikeSongSerializer serializer_class = ListenSongSerializer
permission_classes = [permissions.AllowAny] permission_classes = [permissions.AllowAny]
def get_queryset(self): def get_queryset(self):
@ -331,12 +399,25 @@ def post(self, request, *args, **kwargs):
return Response(status=404) return Response(status=404)
if self.request.user.is_authenticated: if self.request.user.is_authenticated:
listen_to_song.apply_async( listen_to_song.apply_async(
kwargs={"song_id": song.id, "user_id": self.request.user.id}, kwargs={
"song_id": song.id,
"user_id": self.request.user.id,
"anon": False,
},
countdown=2,
)
elif "user_id" in data:
listen_to_song.apply_async(
kwargs={
"song_id": song.id,
"user_id": data["user_id"],
"anon": True,
},
countdown=2, countdown=2,
) )
else: else:
listen_to_song.apply_async( listen_to_song.apply_async(
kwargs={"song_id": song.id}, kwargs={"song_id": song.id, "user_id": None, "anon": True},
countdown=2, countdown=2,
) )
return Response(status=201) return Response(status=201)
@ -353,3 +434,8 @@ def get_queryset(self):
.filter(user=self.request.user) .filter(user=self.request.user)
.values_list("song_id", flat=True) .values_list("song_id", flat=True)
) )
class CreateAnonMusicUserAPIView(generics.CreateAPIView):
serializer_class = AnonMusicUserSerializer
permission_classes = [permissions.AllowAny]

View File

@ -0,0 +1,79 @@
# Generated by Django 4.2.8 on 2024-01-17 13:13
import django.contrib.postgres.fields
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import uuid
class Migration(migrations.Migration):
dependencies = [
("music", "0015_usermusicprofile"),
]
operations = [
migrations.CreateModel(
name="AnonMusicUser",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
],
),
migrations.AddField(
model_name="song",
name="created",
field=models.DateTimeField(
auto_now_add=True, default=django.utils.timezone.now
),
preserve_default=False,
),
migrations.AddField(
model_name="song",
name="volume",
field=django.contrib.postgres.fields.ArrayField(
base_field=models.IntegerField(), null=True, size=None
),
),
migrations.CreateModel(
name="AnonMusicUserHistory",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created", models.DateTimeField(auto_now_add=True)),
(
"song",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="anon_listeners",
to="music.song",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="songs_listened",
to="music.anonmusicuser",
),
),
],
options={
"ordering": ["-created"],
},
),
]

View File

@ -1,20 +0,0 @@
# Generated by Django 4.2.8 on 2024-01-15 17:28
import django.contrib.postgres.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("music", "0015_usermusicprofile"),
]
operations = [
migrations.AddField(
model_name="song",
name="volume",
field=django.contrib.postgres.fields.ArrayField(
base_field=models.IntegerField(), null=True, size=None
),
),
]

View File

@ -1,3 +1,5 @@
import uuid
from django.contrib.postgres.fields import ArrayField from django.contrib.postgres.fields import ArrayField
from django.db import models from django.db import models
from django.urls import reverse from django.urls import reverse
@ -46,6 +48,7 @@ class Song(BaseImageModel, ShortLinkModel):
creator = models.ForeignKey( creator = models.ForeignKey(
"users.User", related_name="songs", on_delete=models.SET_NULL, null=True "users.User", related_name="songs", on_delete=models.SET_NULL, null=True
) )
created = models.DateTimeField(auto_now_add=True)
meta = models.JSONField(blank=True, null=True) meta = models.JSONField(blank=True, null=True)
likes = models.IntegerField(default=0) likes = models.IntegerField(default=0)
volume = ArrayField(models.IntegerField(), null=True) volume = ArrayField(models.IntegerField(), null=True)
@ -154,6 +157,29 @@ class Meta:
ordering = ["-created"] ordering = ["-created"]
class AnonMusicUser(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
def __str__(self):
return f"AnonMusicUser {self.id}"
class AnonMusicUserHistory(models.Model):
user = models.ForeignKey(
"music.AnonMusicUser", related_name="songs_listened", on_delete=models.CASCADE
)
song = models.ForeignKey(
"Song", related_name="anon_listeners", on_delete=models.CASCADE
)
created = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ["-created"]
def __str__(self):
return f"{self.user} - {self.song}"
class UserListenHistory(models.Model): class UserListenHistory(models.Model):
user = models.ForeignKey( user = models.ForeignKey(
"users.User", related_name="songs_listened", on_delete=models.CASCADE "users.User", related_name="songs_listened", on_delete=models.CASCADE

View File

@ -1,7 +1,10 @@
import os import os
import re
import requests
from deep_translator import GoogleTranslator from deep_translator import GoogleTranslator
from django.core.files import File from django.core.files import File
from django.db import transaction
from django.utils.text import slugify from django.utils.text import slugify
from mutagen import File as MutagenFile from mutagen import File as MutagenFile
from mutagen.id3 import APIC, ID3, TCON, TORY, TextFrame from mutagen.id3 import APIC, ID3, TCON, TORY, TextFrame
@ -10,9 +13,32 @@
from pydub import AudioSegment from pydub import AudioSegment
from akarpov.music.models import Album, Author, Song from akarpov.music.models import Album, Author, Song
from akarpov.music.services.info import search_all_platforms from akarpov.music.services.info import generate_readable_slug, search_all_platforms
from akarpov.users.models import User from akarpov.users.models import User
from akarpov.utils.generators import generate_charset
def get_or_create_author(author_name):
with transaction.atomic():
author = Author.objects.filter(name__iexact=author_name).order_by("id").first()
if author is None:
author = Author.objects.create(name=author_name)
return author
def process_track_name(track_name: str) -> str:
# Split the track name by dash and parentheses
parts = track_name.split(" - ")
processed_parts = []
for part in parts:
if "feat" in part:
continue
if "(" in part:
part = part.split("(")[0].strip()
processed_parts.append(part)
processed_track_name = " - ".join(processed_parts)
return processed_track_name
def load_track( def load_track(
@ -25,14 +51,31 @@ def load_track(
link: str | None = None, link: str | None = None,
**kwargs, **kwargs,
) -> Song: ) -> Song:
p_name = path.split("/")[-1] p_name = process_track_name(
query = f"{name if name else p_name} - {album if album else ''} - {', '.join(authors) if authors else ''}" " ".join(path.split("/")[-1].split(".")[0].strip().split())
)
query = (
f"{process_track_name(name) if name else p_name} "
f"- {album if album else ''} - {', '.join(authors) if authors else ''}"
)
search_info = search_all_platforms(query) search_info = search_all_platforms(query)
orig_name = name if name else p_name
if image_path and search_info.get("album_image", None): if image_path and search_info.get("album_image", None):
os.remove(search_info["album_image"]) os.remove(search_info["album_image"])
if "title" in search_info:
title = re.sub(r"\W+", "", search_info["title"]).lower()
name_clean = re.sub(r"\W+", "", orig_name).lower()
# Check if title is in name
if title in name_clean:
name = search_info["title"]
elif not name:
name = process_track_name(" ".join(p_name.strip().split("-")))
if not name:
name = orig_name
name = name or search_info.get("title", p_name)
album = album or search_info.get("album_name", None) album = album or search_info.get("album_name", None)
authors = authors or search_info.get("artists", []) authors = authors or search_info.get("artists", [])
genre = kwargs.get("genre") or search_info.get("genre", None) genre = kwargs.get("genre") or search_info.get("genre", None)
@ -44,15 +87,6 @@ def load_track(
if album and type(album) is str and album.startswith("['"): if album and type(album) is str and album.startswith("['"):
album = album.replace("['", "").replace("']", "") album = album.replace("['", "").replace("']", "")
re_authors = []
if authors:
for x in authors:
try:
re_authors.append(Author.objects.get(name=x))
except Author.DoesNotExist:
re_authors.append(Author.objects.create(name=x))
authors = re_authors
album_name = None
if album: if album:
if type(album) is str: if type(album) is str:
album_name = album album_name = album
@ -61,12 +95,16 @@ def load_track(
else: else:
album_name = None album_name = None
if album_name: if album_name:
try: album, created = Album.objects.get_or_create(
album = Album.objects.get(name=album_name) name__iexact=album_name, defaults={"name": album_name}
except Album.DoesNotExist: )
album = Album.objects.create(name=album_name)
if not album_name: processed_authors = []
album = None if authors:
for author_name in authors:
author = get_or_create_author(author_name)
processed_authors.append(author)
authors = processed_authors
if sng := Song.objects.filter( if sng := Song.objects.filter(
name=name if name else p_name, name=name if name else p_name,
@ -82,6 +120,14 @@ def load_track(
path = mp3_path path = mp3_path
tag = MP3(path, ID3=ID3) tag = MP3(path, ID3=ID3)
if image_path and image_path.startswith("http"):
response = requests.get(image_path)
se = image_path.split("/")[-1]
image_path = f'/tmp/{generate_readable_slug(name, Song)}.{"png" if "." not in se else se.split(".")[-1]}'
with open(image_path, "wb") as f:
f.write(response.content)
if image_path: if image_path:
if not image_path.endswith(".png"): if not image_path.endswith(".png"):
nm = image_path nm = image_path
@ -173,19 +219,7 @@ def load_track(
if os.path.exists(image_path): if os.path.exists(image_path):
os.remove(image_path) os.remove(image_path)
if generated_name and not Song.objects.filter(slug=generated_name).exists(): song.slug = generate_readable_slug(song.name, Song)
if len(generated_name) > 20: song.save()
generated_name = generated_name.split("-")[0]
if len(generated_name) > 20:
generated_name = generated_name[:20]
if not Song.objects.filter(slug=generated_name).exists():
song.slug = generated_name
song.save()
else:
song.slug = generated_name[:14] + "_" + generate_charset(5)
song.save()
else:
song.slug = generated_name
song.save()
return song return song

View File

@ -6,6 +6,7 @@
from deep_translator import GoogleTranslator from deep_translator import GoogleTranslator
from django.conf import settings from django.conf import settings
from django.core.files import File from django.core.files import File
from django.db import transaction
from django.utils.text import slugify from django.utils.text import slugify
from spotipy import SpotifyClientCredentials from spotipy import SpotifyClientCredentials
from yandex_music import Client, Cover from yandex_music import Client, Cover
@ -16,6 +17,34 @@
from akarpov.utils.text import is_similar_artist, normalize_text from akarpov.utils.text import is_similar_artist, normalize_text
def generate_readable_slug(name: str, model) -> str:
# Translate and slugify the name
slug = str(
slugify(
GoogleTranslator(source="auto", target="en").translate(
name,
target_language="en",
)
)
)
if len(slug) > 20:
slug = slug[:20]
last_dash = slug.rfind("-")
if last_dash != -1:
slug = slug[:last_dash]
while model.objects.filter(slug=slug).exists():
if len(slug) > 14:
slug = slug[:14]
last_dash = slug.rfind("-")
if last_dash != -1:
slug = slug[:last_dash]
slug = slug + "_" + generate_charset(5)
return slug
def create_spotify_session() -> spotipy.Spotify: def create_spotify_session() -> spotipy.Spotify:
if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET: if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
raise ConnectionError("No spotify credentials provided") raise ConnectionError("No spotify credentials provided")
@ -197,15 +226,6 @@ def update_album_info(album: AlbumModel, author_name: str = None) -> None:
# Combine and prioritize Spotify data # Combine and prioritize Spotify data
album_data = {} album_data = {}
if yandex_album_info:
album_data.update(
{
"name": album_data.get("name", yandex_album_info.title),
"genre": album_data.get("genre", yandex_album_info.genre),
"description": yandex_album_info.description,
"type": yandex_album_info.type,
}
)
if spotify_album_info: if spotify_album_info:
album_data = { album_data = {
@ -215,6 +235,15 @@ def update_album_info(album: AlbumModel, author_name: str = None) -> None:
"link": spotify_album_info["external_urls"]["spotify"], "link": spotify_album_info["external_urls"]["spotify"],
"genre": spotify_album_info.get("genres", []), "genre": spotify_album_info.get("genres", []),
} }
if yandex_album_info:
album_data.update(
{
"name": album_data.get("name", yandex_album_info.title),
"genre": album_data.get("genre", yandex_album_info.genre),
"description": yandex_album_info.description,
"type": yandex_album_info.type,
}
)
album.meta = album_data album.meta = album_data
album.save() album.save()
@ -262,20 +291,8 @@ def update_album_info(album: AlbumModel, author_name: str = None) -> None:
album_authors.append(author) album_authors.append(author)
album.authors.set(album_authors) album.authors.set(album_authors)
if generated_name and not AlbumModel.objects.filter(slug=generated_name).exists(): album.slug = generate_readable_slug(album.name, AlbumModel)
if len(generated_name) > 20: album.save()
generated_name = generated_name.split("-")[0]
if len(generated_name) > 20:
generated_name = generated_name[:20]
if not AlbumModel.objects.filter(slug=generated_name).exists():
album.slug = generated_name
album.save()
else:
album.slug = generated_name[:14] + "_" + generate_charset(5)
album.save()
else:
album.slug = generated_name
album.save()
def update_author_info(author: Author) -> None: def update_author_info(author: Author) -> None:
@ -288,6 +305,13 @@ def update_author_info(author: Author) -> None:
# Combine and prioritize Spotify data # Combine and prioritize Spotify data
author_data = {} author_data = {}
if spotify_artist_info:
author_data = {
"name": spotify_artist_info.get("name", author.name),
"genres": spotify_artist_info.get("genres", []),
"popularity": spotify_artist_info.get("popularity", 0),
"link": spotify_artist_info["external_urls"]["spotify"],
}
if yandex_artist_info: if yandex_artist_info:
author_data.update( author_data.update(
{ {
@ -297,16 +321,9 @@ def update_author_info(author: Author) -> None:
} }
) )
if spotify_artist_info:
author_data = {
"name": spotify_artist_info.get("name", author.name),
"genres": spotify_artist_info.get("genres", []),
"popularity": spotify_artist_info.get("popularity", 0),
"link": spotify_artist_info["external_urls"]["spotify"],
}
author.meta = author_data author.meta = author_data
author.save() with transaction.atomic():
author.save()
# Handle Author Image - Prefer Spotify, fallback to Yandex # Handle Author Image - Prefer Spotify, fallback to Yandex
image_path = None image_path = None
@ -337,20 +354,9 @@ def update_author_info(author: Author) -> None:
os.remove(image_path) os.remove(image_path)
author.save() author.save()
if generated_name and not Author.objects.filter(slug=generated_name).exists(): author.slug = generate_readable_slug(author.name, Author)
if len(generated_name) > 20: with transaction.atomic():
generated_name = generated_name.split("-")[0] author.save()
if len(generated_name) > 20:
generated_name = generated_name[:20]
if not Author.objects.filter(slug=generated_name).exists():
author.slug = generated_name
author.save()
else:
author.slug = generated_name[:14] + "_" + generate_charset(5)
author.save()
else:
author.slug = generated_name
author.save()
def search_all_platforms(track_name: str) -> dict: def search_all_platforms(track_name: str) -> dict:
@ -373,7 +379,6 @@ def search_all_platforms(track_name: str) -> dict:
for existing_artist in combined_artists for existing_artist in combined_artists
): ):
combined_artists.add(normalized_artist) combined_artists.add(normalized_artist)
genre = spotify_info.get("genre") or yandex_info.get("genre") genre = spotify_info.get("genre") or yandex_info.get("genre")
if type(genre) is list: if type(genre) is list:
genre = sorted(genre, key=lambda x: len(x)) genre = sorted(genre, key=lambda x: len(x))

View File

@ -1,7 +1,12 @@
import threading
import spotipy import spotipy
from django.conf import settings from django.conf import settings
from spotdl import Song, Spotdl
from spotipy.oauth2 import SpotifyClientCredentials from spotipy.oauth2 import SpotifyClientCredentials
from akarpov.music.services.db import load_track
def create_session() -> spotipy.Spotify: def create_session() -> spotipy.Spotify:
if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET: if not settings.MUSIC_SPOTIFY_ID or not settings.MUSIC_SPOTIFY_SECRET:
@ -18,3 +23,72 @@ def create_session() -> spotipy.Spotify:
def search(name: str, session: spotipy.Spotify, search_type="track"): def search(name: str, session: spotipy.Spotify, search_type="track"):
res = session.search(name, type=search_type) res = session.search(name, type=search_type)
return res return res
thread_local = threading.local()
def get_spotdl_client():
if not hasattr(thread_local, "spotdl_client"):
spot_settings = {
"simple_tui": True,
"log_level": "ERROR",
"lyrics_providers": ["genius", "azlyrics", "musixmatch"],
"threads": 6,
"format": "mp3",
"ffmpeg": "ffmpeg",
"sponsor_block": True,
}
thread_local.spotdl_client = Spotdl(
client_id=settings.MUSIC_SPOTIFY_ID,
client_secret=settings.MUSIC_SPOTIFY_SECRET,
user_auth=False,
headless=False,
downloader_settings=spot_settings,
)
return thread_local.spotdl_client
def download_url(url, user_id=None):
spotdl_client = get_spotdl_client()
session = create_session()
if "track" in url:
songs = [Song.from_url(url)]
elif "album" in url:
album_tracks = session.album(url)["tracks"]["items"]
songs = [
Song.from_url(track["external_urls"]["spotify"]) for track in album_tracks
]
elif "artist" in url:
artist_top_tracks = session.artist_top_tracks(url)["tracks"]
songs = [
Song.from_url(track["external_urls"]["spotify"])
for track in artist_top_tracks
]
elif "playlist" in url:
playlist_tracks = session.playlist_items(url)["items"]
songs = [
Song.from_url(track["track"]["external_urls"]["spotify"])
for track in playlist_tracks
]
else:
return None
for song in songs:
res = spotdl_client.download(song)
if res:
song, path = res
else:
return None
load_track(
path=str(path),
image_path=song.cover_url,
user_id=user_id,
authors=song.artists,
album=song.album_name,
name=song.name,
link=song.url,
genre=song.genres[0] if song.genres else None,
release=song.date,
)

View File

@ -6,9 +6,11 @@
import requests import requests
import yt_dlp import yt_dlp
from django.conf import settings from django.conf import settings
from django.utils.text import slugify
from PIL import Image from PIL import Image
from pydub import AudioSegment from pydub import AudioSegment
from pytube import Search, YouTube from pytube import Search, YouTube
from spotdl.providers.audio import YouTubeMusic
from akarpov.music.models import Song from akarpov.music.models import Song
from akarpov.music.services.db import load_track from akarpov.music.services.db import load_track
@ -17,22 +19,28 @@
final_filename = None final_filename = None
ydl_opts = { ytmusic = YouTubeMusic()
"format": "m4a/bestaudio/best",
"postprocessors": [
{ # Extract audio using ffmpeg
"key": "FFmpegExtractAudio",
"preferredcodec": "m4a",
}
],
"outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
}
def download_file(url): def download_file(url):
ydl_opts = {
"format": "bestaudio/best",
"outtmpl": f"{settings.MEDIA_ROOT}/%(uploader)s_%(title)s.%(ext)s",
"postprocessors": [
{"key": "SponsorBlock"}, # Skip sponsor segments
{
"key": "FFmpegExtractAudio",
"preferredcodec": "mp3",
"preferredquality": "192",
}, # Extract audio
{"key": "EmbedThumbnail"}, # Embed Thumbnail
{"key": "FFmpegMetadata"}, # Apply correct metadata
],
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(url) info = ydl.extract_info(url, download=True)
return info["requested_downloads"][0]["_filename"] filename = ydl.prepare_filename(info)
return os.path.splitext(filename)[0] + ".mp3"
def parse_description(description: str) -> list: def parse_description(description: str) -> list:
@ -66,7 +74,7 @@ def parse_description(description: str) -> list:
def download_from_youtube_link(link: str, user_id: int) -> Song: def download_from_youtube_link(link: str, user_id: int) -> Song:
song = None song = None
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL({"ignoreerrors": True, "extract_flat": True}) as ydl:
info_dict = ydl.extract_info(link, download=False) info_dict = ydl.extract_info(link, download=False)
title = info_dict.get("title", None) title = info_dict.get("title", None)
description = info_dict.get("description", None) description = info_dict.get("description", None)
@ -75,9 +83,18 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
# convert to mp3 # convert to mp3
print(f"[processing] {title} converting to mp3") print(f"[processing] {title} converting to mp3")
path = orig_path.replace(orig_path.split(".")[-1], "mp3") path = (
AudioSegment.from_file(orig_path).export(path) "/".join(orig_path.split("/")[:-1])
os.remove(orig_path) + "/"
+ slugify(orig_path.split("/")[-1].split(".")[0])
+ ".mp3"
)
if orig_path.endswith(".mp3"):
os.rename(orig_path, path)
else:
AudioSegment.from_file(orig_path).export(path)
if orig_path != path:
os.remove(orig_path)
print(f"[processing] {title} converting to mp3: done") print(f"[processing] {title} converting to mp3: done")
# split in chapters # split in chapters
@ -175,7 +192,8 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
info["album_name"], info["album_name"],
title, title,
) )
os.remove(path) if os.path.exists(path):
os.remove(path)
return song return song

View File

@ -17,7 +17,7 @@ def auto_delete_file_on_delete(sender, instance, **kwargs):
@receiver(post_save, sender=Song) @receiver(post_save, sender=Song)
def song_create(sender, instance: Song, created, **kwargs): def song_create(sender, instance: Song, created, **kwargs):
if instance.volume is None: if instance.volume is None and instance.file:
set_song_volume(instance) set_song_volume(instance)

View File

@ -1,16 +1,27 @@
from datetime import timedelta
import pylast import pylast
import spotipy
import structlog import structlog
import ytmusicapi
from asgiref.sync import async_to_sync from asgiref.sync import async_to_sync
from celery import shared_task from celery import shared_task
from channels.layers import get_channel_layer from channels.layers import get_channel_layer
from django.conf import settings from django.conf import settings
from django.utils import timezone from django.utils import timezone
from django.utils.timezone import now from django.utils.timezone import now
from pytube import Channel, Playlist from spotipy import SpotifyClientCredentials
from akarpov.music.api.serializers import SongSerializer from akarpov.music.api.serializers import SongSerializer
from akarpov.music.models import RadioSong, Song, UserListenHistory, UserMusicProfile from akarpov.music.models import (
from akarpov.music.services import yandex, youtube AnonMusicUser,
AnonMusicUserHistory,
RadioSong,
Song,
UserListenHistory,
UserMusicProfile,
)
from akarpov.music.services import spotify, yandex, youtube
from akarpov.music.services.file import load_dir, load_file from akarpov.music.services.file import load_dir, load_file
from akarpov.utils.celery import get_scheduled_tasks_name from akarpov.utils.celery import get_scheduled_tasks_name
@ -19,18 +30,57 @@
@shared_task @shared_task
def list_tracks(url, user_id): def list_tracks(url, user_id):
if "music.yandex.ru" in url: if "music.youtube.com" in url or "youtu.be" in url:
url = url.replace("music.youtube.com", "youtube.com")
url = url.replace("youtu.be", "youtube.com")
if "spotify.com" in url:
spotify.download_url(url, user_id)
elif "music.yandex.ru" in url:
yandex.load_playlist(url, user_id) yandex.load_playlist(url, user_id)
elif "channel" in url or "/c/" in url: if "youtube.com" in url:
p = Channel(url) if "channel" in url or "/c/" in url:
for video in p.video_urls: ytmusic = ytmusicapi.YTMusic()
process_yb.apply_async(kwargs={"url": video, "user_id": user_id}) channel_id = url.split("/")[-1]
elif "playlist" in url or "&list=" in url: channel_songs = ytmusic.get_artist(channel_id)["songs"]["results"]
p = Playlist(url) print(channel_songs)
for video in p.video_urls:
process_yb.apply_async(kwargs={"url": video, "user_id": user_id}) for song in channel_songs:
process_yb.apply_async(
kwargs={
"url": f"https://youtube.com/watch?v={song['videoId']}",
"user_id": user_id,
}
)
elif "playlist" in url or "&list=" in url:
ytmusic = ytmusicapi.YTMusic()
playlist_id = url.split("=")[-1]
playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
for song in playlist_songs:
process_yb.apply_async(
kwargs={
"url": f"https://music.youtube.com/watch?v={song['videoId']}",
"user_id": user_id,
}
)
else:
process_yb.apply_async(kwargs={"url": url, "user_id": user_id})
else: else:
process_yb.apply_async(kwargs={"url": url, "user_id": user_id}) spotify_manager = SpotifyClientCredentials(
client_id=settings.MUSIC_SPOTIFY_ID,
client_secret=settings.MUSIC_SPOTIFY_SECRET,
)
spotify_search = spotipy.Spotify(client_credentials_manager=spotify_manager)
results = spotify_search.search(q=url, type="track", limit=1)
top_track = (
results["tracks"]["items"][0] if results["tracks"]["items"] else None
)
if top_track:
spotify.download_url(top_track["external_urls"]["spotify"], user_id)
url = top_track["external_urls"]["spotify"]
return url return url
@ -78,8 +128,6 @@ def start_next_song(previous_ids: list):
async_to_sync(channel_layer.group_send)( async_to_sync(channel_layer.group_send)(
"radio_main", {"type": "song", "data": data} "radio_main", {"type": "song", "data": data}
) )
song.played += 1
song.save(update_fields=["played"])
if RadioSong.objects.filter(slug="").exists(): if RadioSong.objects.filter(slug="").exists():
r = RadioSong.objects.get(slug="") r = RadioSong.objects.get(slug="")
r.song = song r.song = song
@ -96,7 +144,7 @@ def start_next_song(previous_ids: list):
@shared_task @shared_task
def listen_to_song(song_id, user_id=None): def listen_to_song(song_id, user_id=None, anon=True):
# protection from multiple listen, # protection from multiple listen,
# check that last listen by user was more than the length of the song # check that last listen by user was more than the length of the song
# and last listened song is not the same # and last listened song is not the same
@ -104,38 +152,63 @@ def listen_to_song(song_id, user_id=None):
s.played += 1 s.played += 1
s.save(update_fields=["played"]) s.save(update_fields=["played"])
if user_id: if user_id:
try: if anon:
last_listen = UserListenHistory.objects.filter(user_id=user_id).latest("id") try:
except UserListenHistory.DoesNotExist: anon_user = AnonMusicUser.objects.get(id=user_id)
last_listen = None except AnonMusicUser.DoesNotExist:
if ( anon_user = AnonMusicUser.objects.create(id=user_id)
last_listen try:
and last_listen.song_id == song_id last_listen = AnonMusicUserHistory.objects.filter(
or last_listen user_id=user_id
and last_listen.created + s.length > now() ).last()
): except AnonMusicUserHistory.DoesNotExist:
return last_listen = None
UserListenHistory.objects.create( if (
user_id=user_id, last_listen
song_id=song_id, and last_listen.song_id == song_id
) or last_listen
try: and last_listen.created + timedelta(seconds=s.length) > now()
user_profile = UserMusicProfile.objects.get(user_id=user_id) ):
lastfm_token = user_profile.lastfm_token return
AnonMusicUserHistory.objects.create(
# Initialize Last.fm network with the user's session key user=anon_user,
network = pylast.LastFMNetwork( song_id=song_id,
api_key=settings.LAST_FM_API_KEY,
api_secret=settings.LAST_FM_SECRET,
session_key=lastfm_token,
) )
song = Song.objects.get(id=song_id) else:
artist_name = song.artists_names try:
track_name = song.name last_listen = UserListenHistory.objects.filter(user_id=user_id).last()
timestamp = int(timezone.now().timestamp()) except UserListenHistory.DoesNotExist:
network.scrobble(artist=artist_name, title=track_name, timestamp=timestamp) last_listen = None
except UserMusicProfile.DoesNotExist: if (
pass last_listen
except Exception as e: and last_listen.song_id == song_id
logger.error(f"Last.fm scrobble error: {e}") or last_listen
and last_listen.created + timedelta(seconds=s.length) > now()
):
return
UserListenHistory.objects.create(
user_id=user_id,
song_id=song_id,
)
try:
user_profile = UserMusicProfile.objects.get(user_id=user_id)
lastfm_token = user_profile.lastfm_token
# Initialize Last.fm network with the user's session key
network = pylast.LastFMNetwork(
api_key=settings.LAST_FM_API_KEY,
api_secret=settings.LAST_FM_SECRET,
session_key=lastfm_token,
)
song = Song.objects.get(id=song_id)
artist_name = song.artists_names
track_name = song.name
timestamp = int(timezone.now().timestamp())
network.scrobble(
artist=artist_name, title=track_name, timestamp=timestamp
)
except UserMusicProfile.DoesNotExist:
pass
except Exception as e:
logger.error(f"Last.fm scrobble error: {e}")
return song_id return song_id

View File

@ -1,7 +1,7 @@
from abc import abstractmethod from abc import abstractmethod
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models, transaction
from django.urls import reverse from django.urls import reverse
from model_utils.models import TimeStampedModel from model_utils.models import TimeStampedModel
@ -79,7 +79,8 @@ def create_model_link(sender, instance, created, **kwargs):
link.save() link.save()
instance.short_link = link instance.short_link = link
instance.save() with transaction.atomic():
instance.save()
def update_model_link(sender, instance, **kwargs): def update_model_link(sender, instance, **kwargs):

View File

@ -33,6 +33,13 @@ RUN apt-get update && \
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \ apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
# Make ssh dir
RUN mkdir -p /root/.ssh/
# Create known_hosts and add github to it
RUN touch /root/.ssh/known_hosts
RUN ssh-keyscan -t rsa github.com >> /root/.ssh/known_hosts
RUN pip install "poetry==$POETRY_VERSION" RUN pip install "poetry==$POETRY_VERSION"
RUN python -m venv /venv RUN python -m venv /venv

View File

@ -119,7 +119,7 @@ spotdl = "^4.2.4"
fuzzywuzzy = "^0.18.0" fuzzywuzzy = "^0.18.0"
python-levenshtein = "^0.23.0" python-levenshtein = "^0.23.0"
pylast = "^5.2.0" pylast = "^5.2.0"
textract = {git = "https://github.com/Alexander-D-Karpov/textract.git", branch = "master"} textract = {git = "https://github.com/Alexander-D-Karpov/textract", branch = "master"}
librosa = "^0.10.1" librosa = "^0.10.1"