Merge d06d39f8b8 into 45cd860803

Bumps [pytest-factoryboy](https://github.com/pytest-dev/pytest-factoryboy) from 2.3.1 to 2.5.1.
- [Changelog](https://github.com/pytest-dev/pytest-factoryboy/blob/master/CHANGES.rst)
- [Commits](https://github.com/pytest-dev/pytest-factoryboy/compare/2.3.1...2.5.1)

---
updated-dependencies:
- dependency-name: pytest-factoryboy
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

README.md

@@ -4,6 +4,9 @@ My collection of apps and tools
 
 Writen in Python 3.11 and Django 4.2
 
+Local upstream mirror:
+https://git.akarpov.ru/sanspie/akarpov
+
 ## Start up
 
 ### installation

akarpov/common/channels.py

@@ -1,20 +1,46 @@
+from importlib import import_module
+
 from channels.db import database_sync_to_async
 from channels.generic.websocket import AsyncJsonWebsocketConsumer, JsonWebsocketConsumer
+from django.conf import settings
+from django.contrib.sessions.models import Session
 
 from akarpov.common.jwt import read_jwt
+from akarpov.users.models import User
+
+engine = import_module(settings.SESSION_ENGINE)
+sessionstore = engine.SessionStore
 
 
 @database_sync_to_async
 def get_user(headers):
     # WARNING headers type is bytes
-    if b"authorization" not in headers or not headers[b"authorization"]:
-        return False
-
+    if (b"authorization" not in headers or not headers[b"authorization"]) and (
+        b"cookie" not in headers or not headers[b"cookie"]
+    ):
+        return None
+    if b"authorization" in headers:
         jwt = headers[b"authorization"].decode()
-    payload = read_jwt(jwt)
+        data = read_jwt(jwt)
+        if not data:
+            return None
+        payload = data
+    elif b"cookie" in headers:
+        cookies = dict([x.split("=") for x in headers[b"cookie"].decode().split("; ")])
+        if "sessionid" not in cookies:
+            return None
+        try:
+            session = sessionstore(cookies["sessionid"])
+            user_id = session["_auth_user_id"]
+        except (Session.DoesNotExist, User.DoesNotExist, KeyError):
+            return None
+
+        payload = {"id": user_id}
+    else:
+        payload = {}
 
     if not payload or "id" not in payload:
-        return False
+        return None
 
     return payload["id"]
 
@@ -27,7 +53,7 @@ def __init__(self, app):
         self.app = app
 
     async def __call__(self, scope, receive, send):
-        scope["user"] = await get_user(dict(scope["headers"]))
+        scope["user_id"] = await get_user(dict(scope["headers"]))
         try:
             return await self.app(scope, receive, send)
         except ValueError:

akarpov/common/jwt.py

@@ -3,7 +3,7 @@
 import jwt
 import pytz
 from django.conf import settings
-from jwt import ExpiredSignatureError, InvalidSignatureError
+from jwt import DecodeError, ExpiredSignatureError, InvalidSignatureError
 
 TIMEZONE = pytz.timezone("Europe/Moscow")
 
@@ -24,7 +24,10 @@ def sign_jwt(data: dict, t_life: None | int = None) -> str:
 
 def read_jwt(token: str) -> dict | bool:
     """reads jwt, validates it and return payload if correct"""
+    try:
         header_data = jwt.get_unverified_header(token)
+    except DecodeError:
+        return False
     secret = settings.SECRET_KEY
     try:
         payload = jwt.decode(token, key=secret, algorithms=[header_data["alg"]])

akarpov/common/ml/text.py

@@ -0,0 +1,51 @@
+import pycld2 as cld2
+import spacy
+import torch
+from transformers import AutoModel, AutoTokenizer
+
+# load ml classes and models on first request
+# TODO: move to outer server/service
+nlp = None
+ru_nlp = None
+
+ru_model = None
+ru_tokenizer = None
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def get_text_embedding(text: str):
+    global nlp, ru_nlp, ru_model, ru_tokenizer
+
+    is_reliable, text_bytes_found, details = cld2.detect(text)
+    if is_reliable:
+        lang = details[0]
+        if lang[1] in ["ru", "en"]:
+            lang = lang[1]
+        else:
+            return None
+    else:
+        return None
+
+    if lang == "ru":
+        if not ru_nlp:
+            ru_nlp = spacy.load("ru_core_news_md", disable=["parser", "ner"])
+        lema = " ".join([token.lemma_ for token in ru_nlp(text)])
+        if not ru_model:
+            ru_model = AutoModel.from_pretrained("DeepPavlov/rubert-base-cased")
+        if not ru_tokenizer:
+            ru_tokenizer = AutoTokenizer.from_pretrained("DeepPavlov/rubert-base-cased")
+        encodings = ru_tokenizer(
+            lema,  # the texts to be tokenized
+            padding=True,  # pad the texts to the maximum length (so that all outputs have the same length)
+            return_tensors="pt",  # return the tensors (not lists)
+        )
+        with torch.no_grad():
+            # get the model embeddings
+            embeds = ru_model(**encodings)
+        embeds = embeds[0]
+    elif lang == "en":
+        embeds = None
+    else:
+        embeds = None
+
+    return embeds

akarpov/files/forms.py

@@ -6,7 +6,7 @@
 class FileForm(forms.ModelForm):
     class Meta:
         model = File
-        fields = ["name", "private", "description"]
+        fields = ["name", "private", "notify_user_on_view", "description"]
 
 
 class FolderForm(forms.ModelForm):

akarpov/files/migrations/0025_create_vector_ps.py

@@ -0,0 +1,10 @@
+from django.db import migrations
+from pgvector.django import VectorExtension
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0024_alter_file_options_alter_filereport_options_and_more"),
+    ]
+
+    operations = [VectorExtension()]

akarpov/files/migrations/0025_file_notify_user_on_view_alter_basefileitem_parent.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.5 on 2023-09-24 16:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0024_alter_file_options_alter_filereport_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="file",
+            name="notify_user_on_view",
+            field=models.BooleanField(
+                default=False, verbose_name="Receive notifications on file view"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="basefileitem",
+            name="parent",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="children",
+                to="files.folder",
+                verbose_name="Folder",
+            ),
+        ),
+    ]

akarpov/files/migrations/0026_file_embeddings.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.5 on 2023-09-16 18:33
+
+from django.db import migrations
+import pgvector.django
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0025_create_vector_ps"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="file",
+            name="embeddings",
+            field=pgvector.django.VectorField(dimensions=768, null=True),
+        ),
+    ]

akarpov/files/migrations/0027_merge_20230925_2023.py

@@ -0,0 +1,12 @@
+# Generated by Django 4.2.5 on 2023-09-25 17:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0025_file_notify_user_on_view_alter_basefileitem_parent"),
+        ("files", "0026_file_embeddings"),
+    ]
+
+    operations = []

akarpov/files/migrations/0028_file_content_file_lang.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.5 on 2023-09-26 09:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("files", "0027_merge_20230925_2023"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="file",
+            name="content",
+            field=models.TextField(default="", max_length=10000),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="file",
+            name="lang",
+            field=models.CharField(
+                choices=[("ru", "ru"), ("en", "en")], default="en", max_length=2
+            ),
+            preserve_default=False,
+        ),
+    ]

akarpov/files/models.py

@@ -17,6 +17,7 @@
 from django.urls import reverse
 from model_utils.fields import AutoCreatedField, AutoLastModifiedField
 from model_utils.models import TimeStampedModel
+from pgvector.django import VectorField
 from polymorphic.models import PolymorphicModel
 
 from akarpov.files.services.files import trash_file_upload, user_unique_file_upload
@@ -26,6 +27,7 @@
 
 class BaseFileItem(PolymorphicModel):
     parent = ForeignKey(
+        verbose_name="Folder",
         to="files.Folder",
         null=True,
         blank=True,
@@ -68,12 +70,20 @@ class File(BaseFileItem, TimeStampedModel, ShortLinkModel, UserHistoryModel):
 
     preview = FileField(blank=True, upload_to="file/previews/")
     file_obj = FileField(blank=False, upload_to=user_unique_file_upload)
+    embeddings = VectorField(dimensions=768, null=True)
+    content = TextField(max_length=10000)
+    lang = CharField(max_length=2, choices=[("ru", "ru"), ("en", "en")])
 
     # meta
     name = CharField(max_length=255, null=True, blank=True)
     description = TextField(blank=True, null=True)
     file_type = CharField(max_length=255, null=True, blank=True)
 
+    # extra settings
+    notify_user_on_view = BooleanField(
+        "Receive notifications on file view", default=False
+    )
+
     @property
     def file_name(self):
         return self.file.path.split("/")[-1]

akarpov/files/previews/application/__init__.py

@@ -1 +1,3 @@
 from . import doc, docx, json, odt, pdf, zip  # noqa
+
+# TODO: add gzip, xlsx

akarpov/files/services/text.py

@@ -0,0 +1,7 @@
+import textract
+
+
+def extract_file_text(file: str) -> str:
+    text = textract.process(file)
+
+    return text

akarpov/files/views.py

@@ -34,6 +34,7 @@
 from akarpov.files.services.folders import delete_folder
 from akarpov.files.services.preview import get_base_meta
 from akarpov.files.tables import FileTable
+from akarpov.notifications.services import send_notification
 
 logger = structlog.get_logger(__name__)
 
@@ -148,6 +149,16 @@ def dispatch(self, request, *args, **kwargs):
         if "bot" in useragent:
             if file.file_type and file.file_type.split("/")[0] == "image":
                 return HttpResponseRedirect(file.file.url)
+        else:
+            if file.notify_user_on_view:
+                if self.request.user != file.user:
+                    send_notification(
+                        "File view",
+                        f"File {file.name} was opened",
+                        "site",
+                        user_id=file.user.id,
+                        conformation=True,
+                    )
         return super().dispatch(request, *args, **kwargs)
 
     def get_context_data(self, **kwargs):

akarpov/notifications/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+
+from akarpov.notifications.models import Notification
+
+
+@admin.register(Notification)
+class NotificationAdmin(admin.ModelAdmin):
+    list_filter = ["provider"]

akarpov/notifications/apps.py

@@ -0,0 +1,12 @@
+from django.apps import AppConfig
+
+
+class NotificationsConfig(AppConfig):
+    name = "akarpov.notifications"
+    verbose_name = "Notifications"
+
+    def ready(self):
+        try:
+            import akarpov.notifications.signals  # noqa F401
+        except ImportError:
+            pass

akarpov/notifications/migrations/0001_initial.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.5 on 2023-09-20 10:52
+
+from django.db import migrations, models
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Notification",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("title", models.CharField(max_length=255)),
+                ("body", models.TextField(blank=True, max_length=5000, null=True)),
+                (
+                    "provider",
+                    models.CharField(
+                        choices=[
+                            ("akarpov.notifications.providers.site", "site"),
+                            ("akarpov.notifications.providers.email", "email"),
+                        ]
+                    ),
+                ),
+                ("meta", models.JSONField(null=True)),
+                ("delivered", models.BooleanField(default=False)),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

akarpov/notifications/models.py

@@ -0,0 +1,17 @@
+from django.db import models
+from django_extensions.db.models import TimeStampedModel
+
+
+class Notification(TimeStampedModel):
+    class NotificationProviders(models.TextChoices):
+        site = "akarpov.notifications.providers.site", "site"
+        email = "akarpov.notifications.providers.email", "email"
+
+    title = models.CharField(max_length=255)
+    body = models.TextField(max_length=5000, null=True, blank=True)
+    provider = models.CharField(choices=NotificationProviders.choices)
+    meta = models.JSONField(null=True)
+    delivered = models.BooleanField(default=False)
+
+    def __str__(self):
+        return self.title

akarpov/notifications/providers/email/__init__.py

@@ -0,0 +1 @@
+from .send import send_notification  # noqa

akarpov/notifications/providers/email/send.py

@@ -0,0 +1,34 @@
+from django.conf import settings
+from django.core.mail import send_mail
+from django.template.loader import render_to_string
+
+from akarpov.notifications.models import Notification
+from akarpov.users.models import User
+
+
+def send_notification(notification: Notification) -> bool:
+    if not notification.meta or all(
+        ["email" not in notification.meta, "user_id" not in notification.meta]
+    ):
+        raise KeyError(
+            f"can't send notification {notification.id}, email/user_id is not found"
+        )
+    if "email" in notification.meta:
+        email = notification.meta["email"]
+        username = ""
+    else:
+        user = User.objects.get(id=notification.meta["user_id"])
+        email = user.email
+        username = user.username
+    message = render_to_string(
+        "email/notification.html", {"username": username, "body": notification.body}
+    )
+    send_mail(
+        notification.title,
+        notification.body,
+        settings.EMAIL_FROM,
+        [email],
+        fail_silently=False,
+        html_message=message,
+    )
+    return True

akarpov/notifications/providers/site/__init__.py

@@ -0,0 +1,8 @@
+"""
+Notifications site provider
+meta params:
+    - user_id: bool, required
+    - conformation: bool, optional
+"""
+
+from .send import send_notification  # noqa

akarpov/notifications/providers/site/api/serializers.py

@@ -0,0 +1,9 @@
+from rest_framework import serializers
+
+from akarpov.notifications.models import Notification
+
+
+class SiteNotificationSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Notification
+        fields = ["title", "body", "created", "delivered"]

akarpov/notifications/providers/site/api/views.py

@@ -0,0 +1,19 @@
+from rest_framework import generics, permissions
+
+from akarpov.common.api import StandardResultsSetPagination
+from akarpov.notifications.models import Notification
+from akarpov.notifications.providers.site.api.serializers import (
+    SiteNotificationSerializer,
+)
+
+
+class ListNotificationsAPIView(generics.ListAPIView):
+    permission_classes = [permissions.IsAuthenticated]
+    serializer_class = SiteNotificationSerializer
+    pagination_class = StandardResultsSetPagination
+
+    def get_queryset(self):
+        return Notification.objects.filter(meta__user_id=self.request.user.id)
+
+
+# TODO: add read notification url here

akarpov/notifications/providers/site/consumers.py

@@ -0,0 +1,28 @@
+from akarpov.common.channels import BaseConsumer
+
+
+class NotificationsConsumer(BaseConsumer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(args, kwargs)
+        self.room_group_name = None
+
+    async def connect(self):
+        self.room_group_name = f"notifications_{self.scope['user_id']}"
+
+        await self.accept()
+        await self.channel_layer.group_add(self.room_group_name, self.channel_name)
+        if not self.scope["user_id"]:
+            await self.send_error("Authorization is required")
+            await self.disconnect(close_code=None)
+            return
+
+    async def disconnect(self, close_code):
+        await self.channel_layer.group_discard(self.room_group_name, self.channel_name)
+        await self.close()
+
+    async def receive_json(self, content: dict, **kwargs):
+        return content
+
+    async def notification(self, event):
+        data = event["data"]
+        await self.send_json(data)

akarpov/notifications/providers/site/send.py

@@ -0,0 +1,30 @@
+from asgiref.sync import async_to_sync
+from channels.layers import get_channel_layer
+
+from akarpov.notifications.models import Notification
+from akarpov.notifications.providers.site.api.serializers import (
+    SiteNotificationSerializer,
+)
+
+
+def send_notification(notification: Notification) -> bool:
+    if (
+        not notification.meta
+        or "user_id" not in notification.meta
+        or not notification.meta["user_id"]
+    ):
+        raise KeyError(
+            f"can't send notification {notification.id}, user_id is not found"
+        )
+    channel_layer = get_channel_layer()
+    async_to_sync(channel_layer.group_send)(
+        f"notifications_{notification.meta['user_id']}",
+        {
+            "type": "notification",
+            "data": SiteNotificationSerializer().to_representation(notification),
+        },
+    )
+    if "conformation" in notification.meta and notification.meta["conformation"]:
+        # no view conformation required, only pop up on site
+        return False
+    return True

akarpov/notifications/providers/site/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from akarpov.notifications.providers.site.api.views import ListNotificationsAPIView
+
+app_name = "notifications:site"
+urlpatterns = [
+    path("", ListNotificationsAPIView.as_view(), name="list"),
+]

akarpov/notifications/providers/urls.py

@@ -0,0 +1,8 @@
+from django.urls import include, path
+
+app_name = "notifications"
+urlpatterns = [
+    path(
+        "site/", include("akarpov.notifications.providers.site.urls", namespace="site")
+    ),
+]

akarpov/notifications/services.py

@@ -0,0 +1,7 @@
+from akarpov.notifications.tasks import run_create_send_notification
+
+
+def send_notification(title: str, body: str, provider: str, **kwargs):
+    run_create_send_notification.apply_async(
+        kwargs={"title": title, "body": body, "provider": provider} | kwargs
+    )

akarpov/notifications/signals.py

@@ -0,0 +1,11 @@
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from akarpov.notifications.models import Notification
+from akarpov.notifications.tasks import run_send_notification
+
+
+@receiver(post_save, sender=Notification)
+def notification_create(sender, instance: Notification, created, **kwargs):
+    if created:
+        run_send_notification.apply_async(kwargs={"pk": instance.pk}, countdown=2)

akarpov/notifications/tasks.py

@@ -0,0 +1,32 @@
+from importlib import import_module
+
+from celery import shared_task
+
+from akarpov.notifications.models import Notification
+
+providers = {x[1]: x[0] for x in Notification.NotificationProviders.choices}
+
+
+@shared_task
+def run_send_notification(pk):
+    instance = Notification.objects.get(pk=pk)
+    provider = import_module(instance.provider)
+    instance.delivered = provider.send_notification(instance)
+    instance.save()
+
+
+@shared_task
+def run_create_send_notification(title: str, body: str, provider: str, **kwargs):
+    if provider != "*" and provider not in providers:
+        raise ValueError(f"no such provider: {provider}")
+    if provider == "*":
+        for provider in providers:
+            Notification.objects.create(
+                title=title, body=body, provider=providers[provider], meta=kwargs
+            )
+    else:
+        Notification.objects.create(
+            title=title, body=body, provider=providers[provider], meta=kwargs
+        )
+
+    return

akarpov/templates/base.html

@@ -132,6 +132,10 @@
                 <footer class="row bg-light py-1 mt-auto text-center">
                     <div class="col"> Writen by <a href="/about">sanspie</a>, find source code <a href="https://github.com/Alexander-D-Karpov/akarpov">here</a> </div>
                 </footer>
+            <div id="toastContainer" class="toast-container position-fixed bottom-0 end-0 p-3">
+
+            </div>
+
             </div>
         </div>
     </div>
@@ -140,5 +144,87 @@
 
     {% block inline_javascript %}
     {% endblock inline_javascript %}
+    {% if request.user.is_authenticated %}
+    <script>
+      {% if request.is_secure %}
+      let socket = new WebSocket(`wss://{{ request.get_host }}/ws/notifications/`);
+      {% else %}
+      let socket = new WebSocket(`ws://{{ request.get_host }}/ws/notifications/`);
+      {% endif %}
+
+      function sleep(ms) {
+          return new Promise(resolve => setTimeout(resolve, ms));
+      }
+
+      function timeSince(date) {
+            let seconds = Math.floor((new Date() - date) / 1000);
+            let interval = seconds / 31536000;
+            if (interval > 1) {
+              return Math.floor(interval) + " years";
+            }
+            interval = seconds / 2592000;
+            if (interval > 1) {
+              return Math.floor(interval) + " months";
+            }
+            interval = seconds / 86400;
+            if (interval > 1) {
+              return Math.floor(interval) + " days";
+            }
+            interval = seconds / 3600;
+            if (interval > 1) {
+              return Math.floor(interval) + " hours";
+            }
+            interval = seconds / 60;
+            if (interval > 1) {
+              return Math.floor(interval) + " minutes";
+            }
+            return Math.floor(seconds) + " seconds";
+      }
+
+      const toastContainer = document.getElementById('toastContainer')
+
+
+      let fn = async function(event) {
+        let data = JSON.parse(event.data)
+        const toast = document.createElement("div")
+        toast.id = "liveToast"
+        toast.className = "toast mb-4 ml-2"
+        toast.setAttribute("role", "alert")
+        toast.setAttribute("aria-live", "assertive")
+        toast.setAttribute("aria-atomic", "true")
+        toast.innerHTML = `<div class="toast-header">
+                    <strong class="me-auto">${data.title}</strong>
+                    <small>${timeSince(Date.parse(data.created))} ago</small>
+                    <button type="button" class="btn-close" data-bs-dismiss="toast" aria-label="Close"></button>
+                  </div>
+                  <div class="toast-body">
+                    ${data.body}
+                  </div>`
+        toastContainer.appendChild(toast)
+        const toastBootstrap = bootstrap.Toast.getOrCreateInstance(toast)
+        toastBootstrap.show()
+      }
+
+      socket.onmessage = fn
+      socket.onclose = async function(event) {
+        console.log("Notifications socket disconnected, reconnecting...")
+        let socketClosed = true;
+        await sleep(5000)
+        while (socketClosed) {
+          {# TODO: reconnect socket here #}
+          try {
+            let cl = socket.onclose
+            socket = new WebSocket(`ws://127.0.0.1:8000/ws/notifications/`);
+            socket.onmessage = fn
+            socket.onclose = cl
+            socketClosed = false
+          } catch (e) {
+            console.log("Can't connect to socket, reconnecting...")
+            await sleep(1000)
+          }
+          }
+      }
+    </script>
+    {% endif %}
   </body>
 </html>

akarpov/templates/email/notification.html

@@ -0,0 +1,6 @@
+<h3>Hello, {% if username %}{{ username }}{% endif %}!</h3>
+<p>You are seeing this message, because you received notification from <a href="https://akarpov.ru">akarpov.ru</a></p>
+
+<p>{{ body }}</p>
+
+<p>If you don't want to receive notifications via email, or got it by excitement <a href="{# TODO: add unsubscribe #}">press here</a></p>

akarpov/templates/files/list.html

@@ -56,7 +56,7 @@
     </nav>
     {% endif %}
   {% endif %}
-  {% if request.user.is_authenticated and is_folder_owner %}
+  {% if request.user.is_authenticated and is_folder_owner and not folder_slug %}
   <div class="d-flex justify-content-end me-5 col">
     <a class="me-5" href="{% url 'files:table' %}">table view</a>
   </div>

akarpov/templates/files/upload.html

@@ -26,11 +26,11 @@
 
 {% block inline_javascript %}
   <script type="text/javascript">
-    var md5 = "",
+    let md5 = "",
         csrf = $("input[name='csrfmiddlewaretoken']")[0].value,
         form_data = [{"name": "csrfmiddlewaretoken", "value": csrf}];
     function calculate_md5(file, chunk_size) {
-      var slice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
+      let slice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
           chunks = chunks = Math.ceil(file.size / chunk_size),
           current_chunk = 0,
           spark = new SparkMD5.ArrayBuffer();
@@ -44,9 +44,9 @@
         }
       };
       function read_next_chunk() {
-        var reader = new FileReader();
+        let reader = new FileReader();
         reader.onload = onload;
-        var start = current_chunk * chunk_size,
+        let start = current_chunk * chunk_size,
             end = Math.min(start + chunk_size, file.size);
         reader.readAsArrayBuffer(slice.call(file, start, end));
       };
@@ -71,7 +71,7 @@
             {"name": "upload_id", "value": data.result.upload_id}
           );
         }
-        var progress = parseInt(data.loaded / data.total * 100.0, 10);
+        let progress = parseInt(data.loaded / data.total * 100.0, 10);
         $("#progress").text(Array(progress).join("=") + "> " + progress + "%");
       },
       done: function (e, data) { // Called when the file has completely uploaded

akarpov/users/api/serializers.py

@@ -57,3 +57,24 @@ class Meta:
             "is_staff": {"read_only": True},
             "is_superuser": {"read_only": True},
         }
+
+
+class UserUpdatePassword(serializers.ModelSerializer):
+    old_password = serializers.CharField(write_only=True)
+
+    class Meta:
+        model = User
+        fields = ("old_password", "password")
+        extra_kwargs = {
+            "password": {"write_only": True},
+        }
+
+    def validate_old_password(self, password: str):
+        if not self.instance.check_password(password):
+            raise serializers.ValidationError("Old password is incorrect")
+        return password
+
+    def update(self, instance, validated_data):
+        instance.set_password(validated_data["password"])
+        instance.save(update_fields=["password"])
+        return instance

akarpov/users/api/urls.py

@@ -1,30 +1,36 @@
 from django.urls import path
 
 from .views import (
-    UserListViewSet,
-    UserRetireUpdateSelfViewSet,
-    UserRetrieveIdViewSet,
-    UserRetrieveViewSet,
+    UserListAPIViewSet,
+    UserRetireUpdateSelfAPIViewSet,
+    UserRetrieveAPIViewSet,
+    UserRetrieveIdAPIAPIView,
+    UserUpdatePasswordAPIView,
 )
 
 app_name = "users_api"
 
 
 urlpatterns = [
-    path("", UserListViewSet.as_view(), name="list"),
+    path("", UserListAPIViewSet.as_view(), name="list"),
     path(
         "self/",
-        UserRetireUpdateSelfViewSet.as_view(),
+        UserRetireUpdateSelfAPIViewSet.as_view(),
         name="self",
     ),
+    path(
+        "self/password",
+        UserUpdatePasswordAPIView.as_view(),
+        name="password",
+    ),
     path(
         "id/<int:pk>",
-        UserRetrieveIdViewSet.as_view(),
+        UserRetrieveIdAPIAPIView.as_view(),
         name="get_by_id",
     ),
     path(
         "<str:username>",
-        UserRetrieveViewSet.as_view(),
+        UserRetrieveAPIViewSet.as_view(),
         name="get",
     ),
 ]

akarpov/users/api/views.py

@@ -1,19 +1,22 @@
+from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema
 from rest_framework import generics, permissions, status, views
 from rest_framework.response import Response
 
 from akarpov.common.api import SmallResultsSetPagination
+from akarpov.common.jwt import sign_jwt
 from akarpov.users.api.serializers import (
     UserEmailVerification,
     UserFullPublicInfoSerializer,
     UserFullSerializer,
     UserPublicInfoSerializer,
     UserRegisterSerializer,
+    UserUpdatePassword,
 )
 from akarpov.users.models import User
 
 
-class UserRegisterViewSet(generics.CreateAPIView):
+class UserRegisterAPIViewSet(generics.CreateAPIView):
     """Creates new user and sends verification email"""
 
     serializer_class = UserRegisterSerializer
@@ -26,7 +29,15 @@ def post(self, request, *args, **kwargs):
         return self.create(request, *args, **kwargs)
 
 
-class UserEmailValidationViewSet(views.APIView):
+class GenerateUserJWTTokenAPIView(generics.GenericAPIView):
+    permission_classes = [permissions.IsAuthenticated]
+
+    @extend_schema(responses={200: OpenApiTypes.STR})
+    def get(self, request, *args, **kwargs):
+        return Response(data=sign_jwt(data={"id": self.request.user.id}))
+
+
+class UserEmailValidationAPIViewSet(views.APIView):
     """Receives token from email and activates user"""
 
     permission_classes = [permissions.AllowAny]
@@ -43,7 +54,7 @@ def post(self, request):
         return Response(status=status.HTTP_200_OK)
 
 
-class UserListViewSet(generics.ListAPIView):
+class UserListAPIViewSet(generics.ListAPIView):
     serializer_class = UserPublicInfoSerializer
     pagination_class = SmallResultsSetPagination
 
@@ -54,7 +65,7 @@ def get(self, request, *args, **kwargs):
         return self.list(request, *args, **kwargs)
 
 
-class UserRetrieveViewSet(generics.RetrieveAPIView):
+class UserRetrieveAPIViewSet(generics.RetrieveAPIView):
     """Returns user's instance on username"""
 
     serializer_class = UserFullPublicInfoSerializer
@@ -70,7 +81,7 @@ def get(self, request, *args, **kwargs):
         return super().get(request, *args, **kwargs)
 
 
-class UserRetrieveIdViewSet(UserRetrieveViewSet):
+class UserRetrieveIdAPIAPIView(UserRetrieveAPIViewSet):
     """Returns user's instance on user's id"""
 
     lookup_field = "pk"
@@ -82,8 +93,15 @@ def get(self, request, *args, **kwargs):
         return self.retrieve(request, *args, **kwargs)
 
 
-class UserRetireUpdateSelfViewSet(generics.RetrieveUpdateDestroyAPIView):
+class UserRetireUpdateSelfAPIViewSet(generics.RetrieveUpdateDestroyAPIView):
     serializer_class = UserFullSerializer
 
     def get_object(self):
         return self.request.user
+
+
+class UserUpdatePasswordAPIView(generics.UpdateAPIView):
+    serializer_class = UserUpdatePassword
+
+    def get_object(self):
+        return self.request.user

akarpov/users/tests/test_api_views.py

@@ -0,0 +1,29 @@
+from django.urls import reverse_lazy
+from pytest_lambda import lambda_fixture, static_fixture
+from rest_framework import status
+
+
+class TestChangePassword:
+    url = static_fixture(reverse_lazy("api:users:password"))
+    new_password = static_fixture("P@ssw0rd123")
+    user = lambda_fixture(lambda user_factory: user_factory(password="P@ssw0rd"))
+
+    def test_ok(self, api_user_client, url, new_password, user):
+        response = api_user_client.put(
+            url, {"old_password": "P@ssw0rd", "password": new_password}
+        )
+
+        assert response.status_code == status.HTTP_200_OK
+        user.refresh_from_db()
+        assert user.check_password(new_password)
+
+    def test_return_err_if_data_is_invalid(
+        self, api_user_client, url, new_password, user
+    ):
+        response = api_user_client.put(
+            url, {"old_password": "123456", "password": new_password}
+        )
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        user.refresh_from_db()
+        assert not user.check_password(new_password)

compose/local/django/Dockerfile

@@ -28,6 +28,8 @@ RUN apt-get update && \
     apt-get install -y build-essential libpq-dev gettext libmagic-dev libjpeg-dev zlib1g-dev  && \
     # Dependencies for file preview generation
     apt-get install -y webp libimage-exiftool-perl libmagickwand-dev ffmpeg libgdal-dev && \
+    # ML dependencies \
+    # none for now
     apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
     rm -rf /var/lib/apt/lists/*
 

compose/production/django/Dockerfile

@@ -26,6 +26,7 @@ ARG APP_HOME=/app
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV BUILD_ENV ${BUILD_ENVIRONMENT}
+ENV POETRY_VERSION 1.4.2
 
 WORKDIR ${APP_HOME}
 
@@ -34,14 +35,12 @@ RUN addgroup --system django \
 
 
 # Install required system dependencies
-RUN apt-get update && apt-get install --no-install-recommends -y \
-  # psycopg2 dependencies
-  libpq-dev \
-  # Translations dependencies
-  gettext \
-  # cleaning up unused files
-  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
-  && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    apt-get install -y build-essential libpq-dev gettext libmagic-dev libjpeg-dev zlib1g-dev && \
+    # Dependencies for file preview generation
+    apt-get install -y webp libimage-exiftool-perl libmagickwand-dev ffmpeg libgdal-dev && \
+    apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
+    rm -rf /var/lib/apt/lists/*
 
 
 RUN pip install "poetry==$POETRY_VERSION"

compose/production/postgres/Dockerfile

@@ -1,5 +1,10 @@
 FROM postgres:14
 
+RUN apt-get update && \
+    apt-get install -y postgresql-14-pgvector && \
+    apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
+    rm -rf /var/lib/apt/lists/*
+
 COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance
 RUN chmod +x /usr/local/bin/maintenance/*
 RUN mv /usr/local/bin/maintenance/* /usr/local/bin \

compose/production/traefik/traefik.yml

@@ -63,6 +63,11 @@ http:
         servers:
           - url: http://django:5000
 
+    redirect:
+      loadBalancer:
+        servers:
+          - url: http://redirect:3000
+
     flower:
       loadBalancer:
         servers:

config/api_router.py

@@ -1,7 +1,7 @@
 from django.urls import include, path
 from rest_framework.authtoken.views import obtain_auth_token
 
-from akarpov.users.api.views import UserRegisterViewSet
+from akarpov.users.api.views import GenerateUserJWTTokenAPIView, UserRegisterAPIViewSet
 
 app_name = "api"
 
@@ -11,9 +11,12 @@
         include(
             [
                 path(
-                    "register/", UserRegisterViewSet.as_view(), name="user_register_api"
+                    "register/",
+                    UserRegisterAPIViewSet.as_view(),
+                    name="user_register_api",
                 ),
                 path("token/", obtain_auth_token),
+                path("jwt/", GenerateUserJWTTokenAPIView.as_view()),
             ]
         ),
     ),
@@ -21,6 +24,10 @@
         "users/",
         include("akarpov.users.api.urls", namespace="users"),
     ),
+    path(
+        "notifications/",
+        include("akarpov.notifications.providers.urls", namespace="notifications"),
+    ),
     path(
         "blog/",
         include("akarpov.blog.api.urls", namespace="blog"),

config/routing.py

@@ -1,7 +1,9 @@
 from django.urls import re_path
 
 from akarpov.music.consumers import RadioConsumer
+from akarpov.notifications.providers.site.consumers import NotificationsConsumer
 
 websocket_urlpatterns = [
     re_path(r"ws/radio/", RadioConsumer.as_asgi()),
+    re_path(r"ws/notifications/", NotificationsConsumer.as_asgi()),
 ]

config/settings/base.py

@@ -155,10 +155,11 @@
     "akarpov.files",
     "akarpov.music",
     "akarpov.gallery",
+    "akarpov.tools.qr",
     "akarpov.pipeliner",
+    "akarpov.notifications",
     "akarpov.test_platform",
     "akarpov.tools.shortener",
-    "akarpov.tools.qr",
     "akarpov.tools.promocodes",
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps

config/settings/local.py

@@ -10,6 +10,7 @@
     "DJANGO_SECRET_KEY",
     default="Lm4alUqtub6qQT4MnV4NmtXQP02RCBtmGj1bJhyDho07Bkjk9WFZxGtwpnLNQGJQ",
 )
+TOKEN_EXP = 24 * 60 * 60
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = ["*"]
 CSRF_TRUSTED_ORIGINS = ["http://127.0.0.1", "https://*.akarpov.ru"]
@@ -20,6 +21,7 @@
 EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-port
 EMAIL_PORT = env("EMAIL_PORT", default="1025")
+EMAIL_FROM = env("EMAIL_FROM", default="noreply@akarpov.ru")
 
 # WhiteNoise
 # ------------------------------------------------------------------------------
@@ -59,5 +61,5 @@
 
 # SHORTENER
 # ------------------------------------------------------------------------------
-SHORTENER_REDIRECT_TO = "http://127.0.0.1:8000"
-SHORTENER_HOST = "http://127.0.0.1:3000"
+SHORTENER_REDIRECT_TO = "https://dev2.akarpov.ru"
+SHORTENER_HOST = "https://dev.akarpov.ru"

production.yml

@@ -24,7 +24,6 @@ services:
       - ./.envs/.production/.postgres
     command: /start
 
-
   redirect:
     build:
       context: .
@@ -35,16 +34,30 @@ services:
     depends_on:
       - postgres
       - redis
-      - mailhog
     volumes:
-      - .:/app:z
+      - type: bind
+        source: /var/www/media/
+        target: /app/akarpov/media/
     env_file:
       - ./.envs/.production/.django
       - ./.envs/.production/.postgres
-    ports:
-      - "3000:3000"
     command: /start-redirect
 
+  traefik:
+    build:
+      context: .
+      dockerfile: ./compose/production/traefik/Dockerfile
+    image: akarpov_production_traefik
+    depends_on:
+      - django
+      - redirect
+    volumes:
+      - production_traefik:/etc/traefik/acme
+    ports:
+      - "0.0.0.0:80:80"
+      - "0.0.0.0:443:443"
+      - "0.0.0.0:3000:3000"
+      - "0.0.0.0:5555:5555"
 
   postgres:
     build:

pyproject.toml

@@ -102,11 +102,13 @@ requests = ">=2.25"
 spacy = {extras = ["lookups"], version = "^3.6.1"}
 spacy-transformers = "^1.2.5"
 extract-msg = "0.28.7"
-pytest-factoryboy = "2.3.1"
+pytest-factoryboy = "2.5.1"
 pytest-xdist = "^3.3.1"
 pytest-mock = "^3.11.1"
 pytest-asyncio = "^0.21.1"
 pytest-lambda = "^2.2.0"
+pgvector = "^0.2.2"
+pycld2 = "^0.41"
 
 
 [build-system]