Compare commits

...

5 Commits

Author SHA1 Message Date
dependabot[bot]
6631d5586c
Merge 1517c4dbf0 into a2da7e724f 2024-03-01 23:47:31 +00:00
dependabot[bot]
1517c4dbf0
Bump pytest-asyncio from 0.21.1 to 0.23.5
Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.21.1 to 0.23.5.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.21.1...v0.23.5)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-03-01 23:47:28 +00:00
a2da7e724f updated elastic build for ru ip 2024-03-02 02:34:44 +03:00
7c9890975b fixed youtube track download, improved search 2024-03-02 02:29:49 +03:00
3acd858598 added distinct images to playlist, authors to album list, minor bug fixes 2024-02-22 16:02:41 +03:00
12 changed files with 1490 additions and 1665 deletions

View File

@ -22,9 +22,17 @@ class Meta:
class ListAlbumSerializer(serializers.ModelSerializer):
authors = serializers.SerializerMethodField(method_name="get_authors")
@extend_schema_field(ListAuthorSerializer(many=True))
def get_authors(self, obj):
return ListAuthorSerializer(
Author.objects.cache().filter(albums__id=obj.id), many=True
).data
class Meta:
model = Album
fields = ["name", "slug", "image_cropped"]
fields = ["name", "slug", "image_cropped", "authors"]
class SongSerializer(serializers.ModelSerializer):
@ -133,10 +141,32 @@ class Meta:
class PlaylistSerializer(SetUserModelSerializer):
creator = UserPublicInfoSerializer(read_only=True)
images = serializers.SerializerMethodField(method_name="get_images")
@extend_schema_field(serializers.ListField(child=serializers.ImageField()))
def get_images(self, obj):
# Get distinct album images from songs
images = (
Song.objects.cache()
.filter(
playlists__id__in=PlaylistSong.objects.cache()
.filter(playlist=obj)
.values("id")
)
.values_list("album__image", flat=True)
.distinct()[:4]
)
# Build absolute URI for each image
images = [
self.context["request"].build_absolute_uri(image)
for image in images
if image
]
return images
class Meta:
model = Playlist
fields = ["name", "length", "slug", "private", "creator"]
fields = ["name", "length", "slug", "images", "private", "creator"]
extra_kwargs = {
"slug": {"read_only": True},
"creator": {"read_only": True},
@ -144,13 +174,21 @@ class Meta:
}
class FullPlaylistSerializer(serializers.ModelSerializer):
songs = ListSongSerializer(many=True, read_only=True)
class FullPlaylistSerializer(PlaylistSerializer):
songs = serializers.SerializerMethodField(method_name="get_songs")
creator = UserPublicInfoSerializer(read_only=True)
@extend_schema_field(ListSongSerializer(many=True))
def get_songs(self, obj):
return ListSongSerializer(
Song.objects.cache().filter(playlists__id=obj.id),
many=True,
context=self.context,
).data
class Meta:
model = Playlist
fields = ["name", "private", "creator", "songs"]
fields = ["name", "private", "creator", "images", "songs"]
extra_kwargs = {
"slug": {"read_only": True},
"creator": {"read_only": True},
@ -158,8 +196,8 @@ class Meta:
class AddSongToPlaylistSerializer(serializers.ModelSerializer):
song = serializers.SlugField()
playlist = serializers.SlugField()
song = serializers.SlugField(write_only=True)
playlist = serializers.SlugField(write_only=True)
class Meta:
model = Playlist

View File

@ -39,24 +39,70 @@ class SongDocument(Document):
},
)
meta = fields.ObjectField(dynamic=True) # Added meta field here as dynamic object
meta = fields.ObjectField(dynamic=True)
class Index:
name = "songs"
settings = {"number_of_shards": 1, "number_of_replicas": 0}
# settings = {
# "number_of_shards": 1,
# "number_of_replicas": 0,
# "analysis": {
# "analyzer": {
# "russian_icu": {
# "type": "custom",
# "tokenizer": "icu_tokenizer",
# "filter": ["icu_folding","icu_normalizer"]
# }
# }
# }
# } TODO
settings = {
"number_of_shards": 1,
"number_of_replicas": 0,
"analysis": {
"filter": {
"russian_stop": {
"type": "stop",
"stopwords": "_russian_",
},
"russian_keywords": {
"type": "keyword_marker",
"keywords": ["пример"],
},
"russian_stemmer": {
"type": "stemmer",
"language": "russian",
},
"autocomplete_filter": {
"type": "edge_ngram",
"min_gram": 1,
"max_gram": 20,
},
"synonym_filter": {
"type": "synonym",
"synonyms": [
"бит,трек,песня,музыка,песня,мелодия,композиция",
"певец,исполнитель,артист,музыкант",
"альбом,диск,пластинка,сборник,коллекция",
],
},
},
"analyzer": {
"russian": {
"tokenizer": "standard",
"filter": [
"russian_stop",
"russian_keywords",
"russian_stemmer",
],
},
"russian_icu": {
"tokenizer": "icu_tokenizer",
"filter": [
"russian_stop",
"russian_keywords",
"russian_stemmer",
],
},
"autocomplete": {
"type": "custom",
"tokenizer": "standard",
"filter": [
"lowercase",
"autocomplete_filter",
"synonym_filter",
],
},
},
},
}
class Django:
model = Song

View File

@ -72,6 +72,16 @@ def load_track(
name = search_info["title"]
elif not name:
name = process_track_name(" ".join(p_name.strip().split("-")))
clear_name = [
"(Official HD Video)",
"(Official Music Video)",
"(Official Video)",
"Official Video",
"Official Music Video",
"Official HD Video",
]
for c in clear_name:
name = name.replace(c, "")
if not name:
name = orig_name

View File

@ -360,6 +360,7 @@ def update_author_info(author: Author) -> None:
def search_all_platforms(track_name: str) -> dict:
print(track_name)
session = spotipy.Spotify(
auth_manager=spotipy.SpotifyClientCredentials(
client_id=settings.MUSIC_SPOTIFY_ID,

View File

@ -13,9 +13,9 @@ def search_song(query):
ES_Q(
"multi_match",
query=query,
fields=["name^3", "authors.name^2", "album.name"],
fields=["name^5", "authors.name^3", "album.name^3"],
fuzziness="AUTO",
), # Change here
),
ES_Q("wildcard", name__raw=f"*{query.lower()}*"),
ES_Q(
"nested",
@ -27,6 +27,7 @@ def search_song(query):
path="album",
query=ES_Q("wildcard", album__name__raw=f"*{query.lower()}*"),
),
ES_Q("wildcard", meta__raw=f"*{query.lower()}*"),
],
minimum_should_match=1,
)

View File

@ -157,22 +157,25 @@ def download_from_youtube_link(link: str, user_id: int) -> Song:
print(f"[processing] loading {title}")
info = search_all_platforms(title)
if not info["album_image"].startswith("/"):
r = requests.get(info["album_image"])
img_pth = str(
settings.MEDIA_ROOT
+ f"/{info['album_image'].split('/')[-1]}_{str(randint(100, 999))}"
)
with open(img_pth, "wb") as f:
f.write(r.content)
if "album_image" in info and info["album_image"]:
if not info["album_image"].startswith("/"):
r = requests.get(info["album_image"])
img_pth = str(
settings.MEDIA_ROOT
+ f"/{info['album_image'].split('/')[-1]}_{str(randint(100, 999))}"
)
with open(img_pth, "wb") as f:
f.write(r.content)
im = Image.open(img_pth)
im.save(str(f"{img_pth}.png"))
im = Image.open(img_pth)
im.save(str(f"{img_pth}.png"))
os.remove(img_pth)
img_pth = f"{img_pth}.png"
os.remove(img_pth)
img_pth = f"{img_pth}.png"
else:
img_pth = info["album_image"]
else:
img_pth = info["album_image"]
img_pth = None
if "genre" in info:
song = load_track(
path,

View File

@ -1,4 +1,5 @@
from datetime import timedelta
from urllib.parse import parse_qs, urlparse
import pylast
import spotipy
@ -54,9 +55,19 @@ def list_tracks(url, user_id):
elif "playlist" in url or "&list=" in url:
ytmusic = ytmusicapi.YTMusic()
playlist_id = url.split("=")[-1]
playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]["results"]
# Parse the URL and the query string
parsed_url = urlparse(url)
parsed_qs = parse_qs(parsed_url.query)
# Get the playlist ID from the parsed query string
playlist_id = parsed_qs.get("list", [None])[0]
if playlist_id:
playlist_songs = ytmusic.get_playlist(playlist_id)["tracks"]
else:
raise ValueError("No playlist ID found in the URL.")
for song in playlist_songs:
process_yb.apply_async(
kwargs={

View File

@ -5,4 +5,4 @@ set -o nounset
/install_preview_dependencies
celery -A config.celery_app worker --loglevel=info -c 5
celery -A config.celery_app worker --autoscale 20 -l INFO

View File

@ -0,0 +1,4 @@
FROM elasticsearch:8.11.1
# Install the ICU plugin
RUN bin/elasticsearch-plugin install https://akarpov.ru/media/analysis-icu-8.11.1.zip

View File

@ -101,7 +101,9 @@ services:
command: /start-flower
elasticsearch:
image: elasticsearch:8.11.1
build:
context: .
dockerfile: ./compose/production/elasticsearch/Dockerfile
ports:
- "9200:9200"
- "9300:9300"

2955
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -98,7 +98,7 @@ requests = ">=2.25"
pytest-factoryboy = "2.3.1"
pytest-xdist = "^3.3.1"
pytest-mock = "^3.11.1"
pytest-asyncio = "^0.21.1"
pytest-asyncio = "^0.23.5"
pytest-lambda = "^2.2.0"
pgvector = "^0.2.2"
pycld2 = "^0.41"