sanspie/akarpov
1
1
Fork 0
mirror of https://github.com/Alexander-D-Karpov/akarpov synced 2024-11-24 02:03:49 +03:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: sanspie:18a9ce319d7c79f19ae4745f2e087a2dce044ac5
Branches Tags
sanspie:main
sanspie:dependabot/pip/pytest-asyncio-0.23.6
sanspie:dependabot/pip/crispy-bootstrap5-2024.2
sanspie:dependabot/docker/compose/production/traefik/traefik-2.11.0
sanspie:dependabot/github_actions/pre-commit/action-3.0.1
sanspie:dependabot/pip/drf-spectacular-0.27.1
sanspie:dependabot/github_actions/actions/cache-4
sanspie:dependabot/pip/fastapi-0.108.0
sanspie:dependabot/pip/werkzeug-3.0.1
..
compare: sanspie:0b20efcd5a13c1178f2a760b935659b14d6a8943
Branches Tags
sanspie:main
sanspie:dependabot/pip/pytest-asyncio-0.23.6
sanspie:dependabot/pip/crispy-bootstrap5-2024.2
sanspie:dependabot/docker/compose/production/traefik/traefik-2.11.0
sanspie:dependabot/github_actions/pre-commit/action-3.0.1
sanspie:dependabot/pip/drf-spectacular-0.27.1
sanspie:dependabot/github_actions/actions/cache-4
sanspie:dependabot/pip/fastapi-0.108.0
sanspie:dependabot/pip/werkzeug-3.0.1

1 Commits
18a9ce319d ... 0b20efcd5a

Author SHA1 Message Date
dependabot[bot]
0b20efcd5a
Bump django-environ from 0.9.0 to 0.11.2 
Bumps [django-environ](https://github.com/joke2k/django-environ) from 0.9.0 to 0.11.2.
- [Release notes](https://github.com/joke2k/django-environ/releases)
- [Changelog](https://github.com/joke2k/django-environ/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/joke2k/django-environ/compare/v0.9.0...v0.11.2)

---
updated-dependencies:
- dependency-name: django-environ
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
 2023-09-08 15:07:25 +00:00
56 changed files with 49 additions and 779 deletions
Show Stats Expand all files Collapse all files

3
README.md
View File

@ -4,9 +4,6 @@ My collection of apps and tools
Writen in Python 3.11 and Django 4.2
Local upstream mirror:
https://git.akarpov.ru/sanspie/akarpov
## Start up
### installation

38
akarpov/common/channels.py
View File

@ -1,46 +1,20 @@
from importlib import import_module
from channels.db import database_sync_to_async
from channels.generic.websocket import AsyncJsonWebsocketConsumer, JsonWebsocketConsumer
from django.conf import settings
from django.contrib.sessions.models import Session
from akarpov.common.jwt import read_jwt
from akarpov.users.models import User
engine = import_module(settings.SESSION_ENGINE)
sessionstore = engine.SessionStore
@database_sync_to_async
def get_user(headers):
# WARNING headers type is bytes
if (b"authorization" not in headers or not headers[b"authorization"]) and (
b"cookie" not in headers or not headers[b"cookie"]
):
return None
if b"authorization" in headers:
jwt = headers[b"authorization"].decode()
data = read_jwt(jwt)
if not data:
return None
payload = data
elif b"cookie" in headers:
cookies = dict([x.split("=") for x in headers[b"cookie"].decode().split("; ")])
if "sessionid" not in cookies:
return None
try:
session = sessionstore(cookies["sessionid"])
user_id = session["_auth_user_id"]
except (Session.DoesNotExist, User.DoesNotExist, KeyError):
return None
if b"authorization" not in headers or not headers[b"authorization"]:
return False
payload = {"id": user_id}
else:
payload = {}
jwt = headers[b"authorization"].decode()
payload = read_jwt(jwt)
if not payload or "id" not in payload:
return None
return False
return payload["id"]
@ -53,7 +27,7 @@ def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
scope["user_id"] = await get_user(dict(scope["headers"]))
scope["user"] = await get_user(dict(scope["headers"]))
try:
return await self.app(scope, receive, send)
except ValueError:

5
akarpov/common/jwt.py
View File

@ -3,7 +3,7 @@
import jwt
import pytz
from django.conf import settings
from jwt import DecodeError, ExpiredSignatureError, InvalidSignatureError
from jwt import ExpiredSignatureError, InvalidSignatureError
TIMEZONE = pytz.timezone("Europe/Moscow")
@ -24,10 +24,7 @@ def sign_jwt(data: dict, t_life: None | int = None) -> str:
def read_jwt(token: str) -> dict | bool:
"""reads jwt, validates it and return payload if correct"""
try:
header_data = jwt.get_unverified_header(token)
except DecodeError:
return False
secret = settings.SECRET_KEY
try:
payload = jwt.decode(token, key=secret, algorithms=[header_data["alg"]])

0
akarpov/common/ml/__init__.py
View File

51
akarpov/common/ml/text.py
View File

@ -1,51 +0,0 @@
import pycld2 as cld2
import spacy
import torch
from transformers import AutoModel, AutoTokenizer
# load ml classes and models on first request
# TODO: move to outer server/service
nlp = None
ru_nlp = None
ru_model = None
ru_tokenizer = None
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def get_text_embedding(text: str):
global nlp, ru_nlp, ru_model, ru_tokenizer
is_reliable, text_bytes_found, details = cld2.detect(text)
if is_reliable:
lang = details[0]
if lang[1] in ["ru", "en"]:
lang = lang[1]
else:
return None
else:
return None
if lang == "ru":
if not ru_nlp:
ru_nlp = spacy.load("ru_core_news_md", disable=["parser", "ner"])
lema = " ".join([token.lemma_ for token in ru_nlp(text)])
if not ru_model:
ru_model = AutoModel.from_pretrained("DeepPavlov/rubert-base-cased")
if not ru_tokenizer:
ru_tokenizer = AutoTokenizer.from_pretrained("DeepPavlov/rubert-base-cased")
encodings = ru_tokenizer(
lema, # the texts to be tokenized
padding=True, # pad the texts to the maximum length (so that all outputs have the same length)
return_tensors="pt", # return the tensors (not lists)
)
with torch.no_grad():
# get the model embeddings
embeds = ru_model(**encodings)
embeds = embeds[0]
elif lang == "en":
embeds = None
else:
embeds = None
return embeds

2
akarpov/files/forms.py
View File

@ -6,7 +6,7 @@
class FileForm(forms.ModelForm):
class Meta:
model = File
fields = ["name", "private", "notify_user_on_view", "description"]
fields = ["name", "private", "description"]
class FolderForm(forms.ModelForm):

10
akarpov/files/migrations/0025_create_vector_ps.py
View File

@ -1,10 +0,0 @@
from django.db import migrations
from pgvector.django import VectorExtension
class Migration(migrations.Migration):
dependencies = [
("files", "0024_alter_file_options_alter_filereport_options_and_more"),
]
operations = [VectorExtension()]

32
akarpov/files/migrations/0025_file_notify_user_on_view_alter_basefileitem_parent.py
View File

@ -1,32 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-24 16:47
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("files", "0024_alter_file_options_alter_filereport_options_and_more"),
]
operations = [
migrations.AddField(
model_name="file",
name="notify_user_on_view",
field=models.BooleanField(
default=False, verbose_name="Receive notifications on file view"
),
),
migrations.AlterField(
model_name="basefileitem",
name="parent",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="children",
to="files.folder",
verbose_name="Folder",
),
),
]

18
akarpov/files/migrations/0026_file_embeddings.py
View File

@ -1,18 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-16 18:33
from django.db import migrations
import pgvector.django
class Migration(migrations.Migration):
dependencies = [
("files", "0025_create_vector_ps"),
]
operations = [
migrations.AddField(
model_name="file",
name="embeddings",
field=pgvector.django.VectorField(dimensions=768, null=True),
),
]

12
akarpov/files/migrations/0027_merge_20230925_2023.py
View File

@ -1,12 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-25 17:23
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("files", "0025_file_notify_user_on_view_alter_basefileitem_parent"),
("files", "0026_file_embeddings"),
]
operations = []

26
akarpov/files/migrations/0028_file_content_file_lang.py
View File

@ -1,26 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-26 09:04
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("files", "0027_merge_20230925_2023"),
]
operations = [
migrations.AddField(
model_name="file",
name="content",
field=models.TextField(default="", max_length=10000),
preserve_default=False,
),
migrations.AddField(
model_name="file",
name="lang",
field=models.CharField(
choices=[("ru", "ru"), ("en", "en")], default="en", max_length=2
),
preserve_default=False,
),
]

10
akarpov/files/models.py
View File

@ -17,7 +17,6 @@
from django.urls import reverse
from model_utils.fields import AutoCreatedField, AutoLastModifiedField
from model_utils.models import TimeStampedModel
from pgvector.django import VectorField
from polymorphic.models import PolymorphicModel
from akarpov.files.services.files import trash_file_upload, user_unique_file_upload
@ -27,7 +26,6 @@
class BaseFileItem(PolymorphicModel):
parent = ForeignKey(
verbose_name="Folder",
to="files.Folder",
null=True,
blank=True,
@ -70,20 +68,12 @@ class File(BaseFileItem, TimeStampedModel, ShortLinkModel, UserHistoryModel):
preview = FileField(blank=True, upload_to="file/previews/")
file_obj = FileField(blank=False, upload_to=user_unique_file_upload)
embeddings = VectorField(dimensions=768, null=True)
content = TextField(max_length=10000)
lang = CharField(max_length=2, choices=[("ru", "ru"), ("en", "en")])
# meta
name = CharField(max_length=255, null=True, blank=True)
description = TextField(blank=True, null=True)
file_type = CharField(max_length=255, null=True, blank=True)
# extra settings
notify_user_on_view = BooleanField(
"Receive notifications on file view", default=False
)
@property
def file_name(self):
return self.file.path.split("/")[-1]

2
akarpov/files/previews/application/__init__.py
View File

@ -1,3 +1 @@
from . import doc, docx, json, odt, pdf, zip # noqa
# TODO: add gzip, xlsx

7
akarpov/files/services/text.py
View File

@ -1,7 +0,0 @@
import textract
def extract_file_text(file: str) -> str:
text = textract.process(file)
return text

11
akarpov/files/views.py
View File

@ -34,7 +34,6 @@
from akarpov.files.services.folders import delete_folder
from akarpov.files.services.preview import get_base_meta
from akarpov.files.tables import FileTable
from akarpov.notifications.services import send_notification
logger = structlog.get_logger(__name__)
@ -149,16 +148,6 @@ def dispatch(self, request, *args, **kwargs):
if "bot" in useragent:
if file.file_type and file.file_type.split("/")[0] == "image":
return HttpResponseRedirect(file.file.url)
else:
if file.notify_user_on_view:
if self.request.user != file.user:
send_notification(
"File view",
f"File {file.name} was opened",
"site",
user_id=file.user.id,
conformation=True,
)
return super().dispatch(request, *args, **kwargs)
def get_context_data(self, **kwargs):

0
akarpov/notifications/__init__.py
View File

8
akarpov/notifications/admin.py
View File

@ -1,8 +0,0 @@
from django.contrib import admin
from akarpov.notifications.models import Notification
@admin.register(Notification)
class NotificationAdmin(admin.ModelAdmin):
list_filter = ["provider"]

12
akarpov/notifications/apps.py
View File

@ -1,12 +0,0 @@
from django.apps import AppConfig
class NotificationsConfig(AppConfig):
name = "akarpov.notifications"
verbose_name = "Notifications"
def ready(self):
try:
import akarpov.notifications.signals # noqa F401
except ImportError:
pass

56
akarpov/notifications/migrations/0001_initial.py
View File

@ -1,56 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-20 10:52
from django.db import migrations, models
import django_extensions.db.fields
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Notification",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
django_extensions.db.fields.CreationDateTimeField(
auto_now_add=True, verbose_name="created"
),
),
(
"modified",
django_extensions.db.fields.ModificationDateTimeField(
auto_now=True, verbose_name="modified"
),
),
("title", models.CharField(max_length=255)),
("body", models.TextField(blank=True, max_length=5000, null=True)),
(
"provider",
models.CharField(
choices=[
("akarpov.notifications.providers.site", "site"),
("akarpov.notifications.providers.email", "email"),
]
),
),
("meta", models.JSONField(null=True)),
("delivered", models.BooleanField(default=False)),
],
options={
"get_latest_by": "modified",
"abstract": False,
},
),
]

0
akarpov/notifications/migrations/__init__.py
View File

17
akarpov/notifications/models.py
View File

@ -1,17 +0,0 @@
from django.db import models
from django_extensions.db.models import TimeStampedModel
class Notification(TimeStampedModel):
class NotificationProviders(models.TextChoices):
site = "akarpov.notifications.providers.site", "site"
email = "akarpov.notifications.providers.email", "email"
title = models.CharField(max_length=255)
body = models.TextField(max_length=5000, null=True, blank=True)
provider = models.CharField(choices=NotificationProviders.choices)
meta = models.JSONField(null=True)
delivered = models.BooleanField(default=False)
def __str__(self):
return self.title

0
akarpov/notifications/providers/__init__.py
View File

1
akarpov/notifications/providers/email/__init__.py
View File

@ -1 +0,0 @@
from .send import send_notification # noqa

34
akarpov/notifications/providers/email/send.py
View File

@ -1,34 +0,0 @@
from django.conf import settings
from django.core.mail import send_mail
from django.template.loader import render_to_string
from akarpov.notifications.models import Notification
from akarpov.users.models import User
def send_notification(notification: Notification) -> bool:
if not notification.meta or all(
["email" not in notification.meta, "user_id" not in notification.meta]
):
raise KeyError(
f"can't send notification {notification.id}, email/user_id is not found"
)
if "email" in notification.meta:
email = notification.meta["email"]
username = ""
else:
user = User.objects.get(id=notification.meta["user_id"])
email = user.email
username = user.username
message = render_to_string(
"email/notification.html", {"username": username, "body": notification.body}
)
send_mail(
notification.title,
notification.body,
settings.EMAIL_FROM,
[email],
fail_silently=False,
html_message=message,
)
return True

8
akarpov/notifications/providers/site/__init__.py
View File

@ -1,8 +0,0 @@
"""
Notifications site provider
meta params:
- user_id: bool, required
- conformation: bool, optional
"""
from .send import send_notification # noqa

0
akarpov/notifications/providers/site/api/__init__.py
View File

9
akarpov/notifications/providers/site/api/serializers.py
View File

@ -1,9 +0,0 @@
from rest_framework import serializers
from akarpov.notifications.models import Notification
class SiteNotificationSerializer(serializers.ModelSerializer):
class Meta:
model = Notification
fields = ["title", "body", "created", "delivered"]

19
akarpov/notifications/providers/site/api/views.py
View File

@ -1,19 +0,0 @@
from rest_framework import generics, permissions
from akarpov.common.api import StandardResultsSetPagination
from akarpov.notifications.models import Notification
from akarpov.notifications.providers.site.api.serializers import (
SiteNotificationSerializer,
)
class ListNotificationsAPIView(generics.ListAPIView):
permission_classes = [permissions.IsAuthenticated]
serializer_class = SiteNotificationSerializer
pagination_class = StandardResultsSetPagination
def get_queryset(self):
return Notification.objects.filter(meta__user_id=self.request.user.id)
# TODO: add read notification url here

28
akarpov/notifications/providers/site/consumers.py
View File

@ -1,28 +0,0 @@
from akarpov.common.channels import BaseConsumer
class NotificationsConsumer(BaseConsumer):
def __init__(self, *args, **kwargs):
super().__init__(args, kwargs)
self.room_group_name = None
async def connect(self):
self.room_group_name = f"notifications_{self.scope['user_id']}"
await self.accept()
await self.channel_layer.group_add(self.room_group_name, self.channel_name)
if not self.scope["user_id"]:
await self.send_error("Authorization is required")
await self.disconnect(close_code=None)
return
async def disconnect(self, close_code):
await self.channel_layer.group_discard(self.room_group_name, self.channel_name)
await self.close()
async def receive_json(self, content: dict, **kwargs):
return content
async def notification(self, event):
data = event["data"]
await self.send_json(data)

30
akarpov/notifications/providers/site/send.py
View File

@ -1,30 +0,0 @@
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from akarpov.notifications.models import Notification
from akarpov.notifications.providers.site.api.serializers import (
SiteNotificationSerializer,
)
def send_notification(notification: Notification) -> bool:
if (
not notification.meta
or "user_id" not in notification.meta
or not notification.meta["user_id"]
):
raise KeyError(
f"can't send notification {notification.id}, user_id is not found"
)
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
f"notifications_{notification.meta['user_id']}",
{
"type": "notification",
"data": SiteNotificationSerializer().to_representation(notification),
},
)
if "conformation" in notification.meta and notification.meta["conformation"]:
# no view conformation required, only pop up on site
return False
return True

8
akarpov/notifications/providers/site/urls.py
View File

@ -1,8 +0,0 @@
from django.urls import path
from akarpov.notifications.providers.site.api.views import ListNotificationsAPIView
app_name = "notifications:site"
urlpatterns = [
path("", ListNotificationsAPIView.as_view(), name="list"),
]

8
akarpov/notifications/providers/urls.py
View File

@ -1,8 +0,0 @@
from django.urls import include, path
app_name = "notifications"
urlpatterns = [
path(
"site/", include("akarpov.notifications.providers.site.urls", namespace="site")
),
]

7
akarpov/notifications/services.py
View File

@ -1,7 +0,0 @@
from akarpov.notifications.tasks import run_create_send_notification
def send_notification(title: str, body: str, provider: str, **kwargs):
run_create_send_notification.apply_async(
kwargs={"title": title, "body": body, "provider": provider} | kwargs
)

11
akarpov/notifications/signals.py
View File

@ -1,11 +0,0 @@
from django.db.models.signals import post_save
from django.dispatch import receiver
from akarpov.notifications.models import Notification
from akarpov.notifications.tasks import run_send_notification
@receiver(post_save, sender=Notification)
def notification_create(sender, instance: Notification, created, **kwargs):
if created:
run_send_notification.apply_async(kwargs={"pk": instance.pk}, countdown=2)

32
akarpov/notifications/tasks.py
View File

@ -1,32 +0,0 @@
from importlib import import_module
from celery import shared_task
from akarpov.notifications.models import Notification
providers = {x[1]: x[0] for x in Notification.NotificationProviders.choices}
@shared_task
def run_send_notification(pk):
instance = Notification.objects.get(pk=pk)
provider = import_module(instance.provider)
instance.delivered = provider.send_notification(instance)
instance.save()
@shared_task
def run_create_send_notification(title: str, body: str, provider: str, **kwargs):
if provider != "*" and provider not in providers:
raise ValueError(f"no such provider: {provider}")
if provider == "*":
for provider in providers:
Notification.objects.create(
title=title, body=body, provider=providers[provider], meta=kwargs
)
else:
Notification.objects.create(
title=title, body=body, provider=providers[provider], meta=kwargs
)
return

0
akarpov/notifications/tests.py
View File

0
akarpov/notifications/views.py
View File

86
akarpov/templates/base.html
View File

@ -132,10 +132,6 @@
<footer class="row bg-light py-1 mt-auto text-center">
<div class="col"> Writen by <a href="/about">sanspie</a>, find source code <a href="https://github.com/Alexander-D-Karpov/akarpov">here</a> </div>
</footer>
<div id="toastContainer" class="toast-container position-fixed bottom-0 end-0 p-3">
</div>
</div>
</div>
</div>
@ -144,87 +140,5 @@
{% block inline_javascript %}
{% endblock inline_javascript %}
{% if request.user.is_authenticated %}
<script>
{% if request.is_secure %}
let socket = new WebSocket(`wss://{{ request.get_host }}/ws/notifications/`);
{% else %}
let socket = new WebSocket(`ws://{{ request.get_host }}/ws/notifications/`);
{% endif %}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function timeSince(date) {
let seconds = Math.floor((new Date() - date) / 1000);
let interval = seconds / 31536000;
if (interval > 1) {
return Math.floor(interval) + " years";
}
interval = seconds / 2592000;
if (interval > 1) {
return Math.floor(interval) + " months";
}
interval = seconds / 86400;
if (interval > 1) {
return Math.floor(interval) + " days";
}
interval = seconds / 3600;
if (interval > 1) {
return Math.floor(interval) + " hours";
}
interval = seconds / 60;
if (interval > 1) {
return Math.floor(interval) + " minutes";
}
return Math.floor(seconds) + " seconds";
}
const toastContainer = document.getElementById('toastContainer')
let fn = async function(event) {
let data = JSON.parse(event.data)
const toast = document.createElement("div")
toast.id = "liveToast"
toast.className = "toast mb-4 ml-2"
toast.setAttribute("role", "alert")
toast.setAttribute("aria-live", "assertive")
toast.setAttribute("aria-atomic", "true")
toast.innerHTML = `<div class="toast-header">
<strong class="me-auto">${data.title}</strong>
<small>${timeSince(Date.parse(data.created))} ago</small>
<button type="button" class="btn-close" data-bs-dismiss="toast" aria-label="Close"></button>
</div>
<div class="toast-body">
${data.body}
</div>`
toastContainer.appendChild(toast)
const toastBootstrap = bootstrap.Toast.getOrCreateInstance(toast)
toastBootstrap.show()
}
socket.onmessage = fn
socket.onclose = async function(event) {
console.log("Notifications socket disconnected, reconnecting...")
let socketClosed = true;
await sleep(5000)
while (socketClosed) {
{# TODO: reconnect socket here #}
try {
let cl = socket.onclose
socket = new WebSocket(`ws://127.0.0.1:8000/ws/notifications/`);
socket.onmessage = fn
socket.onclose = cl
socketClosed = false
} catch (e) {
console.log("Can't connect to socket, reconnecting...")
await sleep(1000)
}
}
}
</script>
{% endif %}
</body>
</html>

6
akarpov/templates/email/notification.html
View File

@ -1,6 +0,0 @@
<h3>Hello, {% if username %}{{ username }}{% endif %}!</h3>
<p>You are seeing this message, because you received notification from <a href="https://akarpov.ru">akarpov.ru</a></p>
<p>{{ body }}</p>
<p>If you don't want to receive notifications via email, or got it by excitement <a href="{# TODO: add unsubscribe #}">press here</a></p>

2
akarpov/templates/files/list.html
View File

@ -56,7 +56,7 @@
</nav>
{% endif %}
{% endif %}
{% if request.user.is_authenticated and is_folder_owner and not folder_slug %}
{% if request.user.is_authenticated and is_folder_owner %}
<div class="d-flex justify-content-end me-5 col">
<a class="me-5" href="{% url 'files:table' %}">table view</a>
</div>

10
akarpov/templates/files/upload.html
View File

@ -26,11 +26,11 @@
{% block inline_javascript %}
<script type="text/javascript">
let md5 = "",
var md5 = "",
csrf = $("input[name='csrfmiddlewaretoken']")[0].value,
form_data = [{"name": "csrfmiddlewaretoken", "value": csrf}];
function calculate_md5(file, chunk_size) {
let slice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
var slice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
chunks = chunks = Math.ceil(file.size / chunk_size),
current_chunk = 0,
spark = new SparkMD5.ArrayBuffer();
@ -44,9 +44,9 @@
}
};
function read_next_chunk() {
let reader = new FileReader();
var reader = new FileReader();
reader.onload = onload;
let start = current_chunk * chunk_size,
var start = current_chunk * chunk_size,
end = Math.min(start + chunk_size, file.size);
reader.readAsArrayBuffer(slice.call(file, start, end));
};
@ -71,7 +71,7 @@
{"name": "upload_id", "value": data.result.upload_id}
);
}
let progress = parseInt(data.loaded / data.total * 100.0, 10);
var progress = parseInt(data.loaded / data.total * 100.0, 10);
$("#progress").text(Array(progress).join("=") + "> " + progress + "%");
},
done: function (e, data) { // Called when the file has completely uploaded

21
akarpov/users/api/serializers.py
View File

@ -57,24 +57,3 @@ class Meta:
"is_staff": {"read_only": True},
"is_superuser": {"read_only": True},
}
class UserUpdatePassword(serializers.ModelSerializer):
old_password = serializers.CharField(write_only=True)
class Meta:
model = User
fields = ("old_password", "password")
extra_kwargs = {
"password": {"write_only": True},
}
def validate_old_password(self, password: str):
if not self.instance.check_password(password):
raise serializers.ValidationError("Old password is incorrect")
return password
def update(self, instance, validated_data):
instance.set_password(validated_data["password"])
instance.save(update_fields=["password"])
return instance

22
akarpov/users/api/urls.py
View File

@ -1,36 +1,30 @@
from django.urls import path
from .views import (
UserListAPIViewSet,
UserRetireUpdateSelfAPIViewSet,
UserRetrieveAPIViewSet,
UserRetrieveIdAPIAPIView,
UserUpdatePasswordAPIView,
UserListViewSet,
UserRetireUpdateSelfViewSet,
UserRetrieveIdViewSet,
UserRetrieveViewSet,
)
app_name = "users_api"
urlpatterns = [
path("", UserListAPIViewSet.as_view(), name="list"),
path("", UserListViewSet.as_view(), name="list"),
path(
"self/",
UserRetireUpdateSelfAPIViewSet.as_view(),
UserRetireUpdateSelfViewSet.as_view(),
name="self",
),
path(
"self/password",
UserUpdatePasswordAPIView.as_view(),
name="password",
),
path(
"id/<int:pk>",
UserRetrieveIdAPIAPIView.as_view(),
UserRetrieveIdViewSet.as_view(),
name="get_by_id",
),
path(
"<str:username>",
UserRetrieveAPIViewSet.as_view(),
UserRetrieveViewSet.as_view(),
name="get",
),
]

30
akarpov/users/api/views.py
View File

@ -1,22 +1,19 @@
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import extend_schema
from rest_framework import generics, permissions, status, views
from rest_framework.response import Response
from akarpov.common.api import SmallResultsSetPagination
from akarpov.common.jwt import sign_jwt
from akarpov.users.api.serializers import (
UserEmailVerification,
UserFullPublicInfoSerializer,
UserFullSerializer,
UserPublicInfoSerializer,
UserRegisterSerializer,
UserUpdatePassword,
)
from akarpov.users.models import User
class UserRegisterAPIViewSet(generics.CreateAPIView):
class UserRegisterViewSet(generics.CreateAPIView):
"""Creates new user and sends verification email"""
serializer_class = UserRegisterSerializer
@ -29,15 +26,7 @@ def post(self, request, *args, **kwargs):
return self.create(request, *args, **kwargs)
class GenerateUserJWTTokenAPIView(generics.GenericAPIView):
permission_classes = [permissions.IsAuthenticated]
@extend_schema(responses={200: OpenApiTypes.STR})
def get(self, request, *args, **kwargs):
return Response(data=sign_jwt(data={"id": self.request.user.id}))
class UserEmailValidationAPIViewSet(views.APIView):
class UserEmailValidationViewSet(views.APIView):
"""Receives token from email and activates user"""
permission_classes = [permissions.AllowAny]
@ -54,7 +43,7 @@ def post(self, request):
return Response(status=status.HTTP_200_OK)
class UserListAPIViewSet(generics.ListAPIView):
class UserListViewSet(generics.ListAPIView):
serializer_class = UserPublicInfoSerializer
pagination_class = SmallResultsSetPagination
@ -65,7 +54,7 @@ def get(self, request, *args, **kwargs):
return self.list(request, *args, **kwargs)
class UserRetrieveAPIViewSet(generics.RetrieveAPIView):
class UserRetrieveViewSet(generics.RetrieveAPIView):
"""Returns user's instance on username"""
serializer_class = UserFullPublicInfoSerializer
@ -81,7 +70,7 @@ def get(self, request, *args, **kwargs):
return super().get(request, *args, **kwargs)
class UserRetrieveIdAPIAPIView(UserRetrieveAPIViewSet):
class UserRetrieveIdViewSet(UserRetrieveViewSet):
"""Returns user's instance on user's id"""
lookup_field = "pk"
@ -93,15 +82,8 @@ def get(self, request, *args, **kwargs):
return self.retrieve(request, *args, **kwargs)
class UserRetireUpdateSelfAPIViewSet(generics.RetrieveUpdateDestroyAPIView):
class UserRetireUpdateSelfViewSet(generics.RetrieveUpdateDestroyAPIView):
serializer_class = UserFullSerializer
def get_object(self):
return self.request.user
class UserUpdatePasswordAPIView(generics.UpdateAPIView):
serializer_class = UserUpdatePassword
def get_object(self):
return self.request.user

29
akarpov/users/tests/test_api_views.py
View File

@ -1,29 +0,0 @@
from django.urls import reverse_lazy
from pytest_lambda import lambda_fixture, static_fixture
from rest_framework import status
class TestChangePassword:
url = static_fixture(reverse_lazy("api:users:password"))
new_password = static_fixture("P@ssw0rd123")
user = lambda_fixture(lambda user_factory: user_factory(password="P@ssw0rd"))
def test_ok(self, api_user_client, url, new_password, user):
response = api_user_client.put(
url, {"old_password": "P@ssw0rd", "password": new_password}
)
assert response.status_code == status.HTTP_200_OK
user.refresh_from_db()
assert user.check_password(new_password)
def test_return_err_if_data_is_invalid(
self, api_user_client, url, new_password, user
):
response = api_user_client.put(
url, {"old_password": "123456", "password": new_password}
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
user.refresh_from_db()
assert not user.check_password(new_password)

2
compose/local/django/Dockerfile
View File

@ -28,8 +28,6 @@ RUN apt-get update && \
apt-get install -y build-essential libpq-dev gettext libmagic-dev libjpeg-dev zlib1g-dev && \
# Dependencies for file preview generation
apt-get install -y webp libimage-exiftool-perl libmagickwand-dev ffmpeg libgdal-dev && \
# ML dependencies \
# none for now
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
rm -rf /var/lib/apt/lists/*

15
compose/production/django/Dockerfile
View File

@ -26,7 +26,6 @@ ARG APP_HOME=/app
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
ENV BUILD_ENV ${BUILD_ENVIRONMENT}
ENV POETRY_VERSION 1.4.2
WORKDIR ${APP_HOME}
@ -35,12 +34,14 @@ RUN addgroup --system django \
# Install required system dependencies
RUN apt-get update && \
apt-get install -y build-essential libpq-dev gettext libmagic-dev libjpeg-dev zlib1g-dev && \
# Dependencies for file preview generation
apt-get install -y webp libimage-exiftool-perl libmagickwand-dev ffmpeg libgdal-dev && \
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install --no-install-recommends -y \
# psycopg2 dependencies
libpq-dev \
# Translations dependencies
gettext \
# cleaning up unused files
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
&& rm -rf /var/lib/apt/lists/*
RUN pip install "poetry==$POETRY_VERSION"

5
compose/production/postgres/Dockerfile
View File

@ -1,10 +1,5 @@
FROM postgres:14
RUN apt-get update && \
apt-get install -y postgresql-14-pgvector && \
apt-get purge -y --auto-remove -o APT:AutoRemove:RecommendsImportant=false && \
rm -rf /var/lib/apt/lists/*
COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance
RUN chmod +x /usr/local/bin/maintenance/*
RUN mv /usr/local/bin/maintenance/* /usr/local/bin \

5
compose/production/traefik/traefik.yml
View File

@ -63,11 +63,6 @@ http:
servers:
- url: http://django:5000
redirect:
loadBalancer:
servers:
- url: http://redirect:3000
flower:
loadBalancer:
servers:

11
config/api_router.py
View File

@ -1,7 +1,7 @@
from django.urls import include, path
from rest_framework.authtoken.views import obtain_auth_token
from akarpov.users.api.views import GenerateUserJWTTokenAPIView, UserRegisterAPIViewSet
from akarpov.users.api.views import UserRegisterViewSet
app_name = "api"
@ -11,12 +11,9 @@
include(
[
path(
"register/",
UserRegisterAPIViewSet.as_view(),
name="user_register_api",
"register/", UserRegisterViewSet.as_view(), name="user_register_api"
),
path("token/", obtain_auth_token),
path("jwt/", GenerateUserJWTTokenAPIView.as_view()),
]
),
),
@ -24,10 +21,6 @@
"users/",
include("akarpov.users.api.urls", namespace="users"),
),
path(
"notifications/",
include("akarpov.notifications.providers.urls", namespace="notifications"),
),
path(
"blog/",
include("akarpov.blog.api.urls", namespace="blog"),

2
config/routing.py
View File

@ -1,9 +1,7 @@
from django.urls import re_path
from akarpov.music.consumers import RadioConsumer
from akarpov.notifications.providers.site.consumers import NotificationsConsumer
websocket_urlpatterns = [
re_path(r"ws/radio/", RadioConsumer.as_asgi()),
re_path(r"ws/notifications/", NotificationsConsumer.as_asgi()),
]

3
config/settings/base.py
View File

@ -155,11 +155,10 @@
"akarpov.files",
"akarpov.music",
"akarpov.gallery",
"akarpov.tools.qr",
"akarpov.pipeliner",
"akarpov.notifications",
"akarpov.test_platform",
"akarpov.tools.shortener",
"akarpov.tools.qr",
"akarpov.tools.promocodes",
]
# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps

6
config/settings/local.py
View File

@ -10,7 +10,6 @@
"DJANGO_SECRET_KEY",
default="Lm4alUqtub6qQT4MnV4NmtXQP02RCBtmGj1bJhyDho07Bkjk9WFZxGtwpnLNQGJQ",
)
TOKEN_EXP = 24 * 60 * 60
# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ["*"]
CSRF_TRUSTED_ORIGINS = ["http://127.0.0.1", "https://*.akarpov.ru"]
@ -21,7 +20,6 @@
EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
# https://docs.djangoproject.com/en/dev/ref/settings/#email-port
EMAIL_PORT = env("EMAIL_PORT", default="1025")
EMAIL_FROM = env("EMAIL_FROM", default="noreply@akarpov.ru")
# WhiteNoise
# ------------------------------------------------------------------------------
@ -61,5 +59,5 @@
# SHORTENER
# ------------------------------------------------------------------------------
SHORTENER_REDIRECT_TO = "https://dev2.akarpov.ru"
SHORTENER_HOST = "https://dev.akarpov.ru"
SHORTENER_REDIRECT_TO = "http://127.0.0.1:8000"
SHORTENER_HOST = "http://127.0.0.1:3000"

29
poetry.lock generated
View File

@ -1154,7 +1154,6 @@ files = [
{file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"},
{file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"},
{file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"},
{file = "contourpy-1.1.0-cp310-cp310-win32.whl", hash = "sha256:9b2dd2ca3ac561aceef4c7c13ba654aaa404cf885b187427760d7f7d4c57cff8"},
{file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"},
{file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"},
{file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"},
@ -1163,7 +1162,6 @@ files = [
{file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"},
{file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"},
{file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"},
{file = "contourpy-1.1.0-cp311-cp311-win32.whl", hash = "sha256:edb989d31065b1acef3828a3688f88b2abb799a7db891c9e282df5ec7e46221b"},
{file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"},
{file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"},
{file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"},
@ -1172,7 +1170,6 @@ files = [
{file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"},
{file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"},
{file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"},
{file = "contourpy-1.1.0-cp38-cp38-win32.whl", hash = "sha256:108dfb5b3e731046a96c60bdc46a1a0ebee0760418951abecbe0fc07b5b93b27"},
{file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"},
{file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"},
{file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"},
@ -1181,7 +1178,6 @@ files = [
{file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"},
{file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"},
{file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"},
{file = "contourpy-1.1.0-cp39-cp39-win32.whl", hash = "sha256:71551f9520f008b2950bef5f16b0e3587506ef4f23c734b71ffb7b89f8721999"},
{file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"},
{file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"},
{file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"},
@ -4197,19 +4193,6 @@ files = [
[package.dependencies]
ptyprocess = ">=0.5"
[[package]]
name = "pgvector"
version = "0.2.2"
description = "pgvector support for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "pgvector-0.2.2-py2.py3-none-any.whl", hash = "sha256:4c2b138ef2023364b93795f2c4ff2a2c8ba54f24979a82995f6b9ce9c86da271"},
]
[package.dependencies]
numpy = "*"
[[package]]
name = "pickleshare"
version = "0.7.5"
@ -4579,16 +4562,6 @@ files = [
[package.dependencies]
pyasn1 = ">=0.4.6,<0.6.0"
[[package]]
name = "pycld2"
version = "0.41"
description = "Python bindings around Google Chromium's embedded compact language detection library (CLD2)"
optional = false
python-versions = "*"
files = [
{file = "pycld2-0.41.tar.gz", hash = "sha256:a42f6e974df8fdd70685c2baa8a9f523069a260e1140ce604fb9f1fb6c3064df"},
]
[[package]]
name = "pycodestyle"
version = "2.11.0"
@ -7927,4 +7900,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "6b7d29ed26a7085574af891d034bf745144f580ac9659f9163ffc707d9fc33a6"
content-hash = "2f0c0bc8aaadd5efee5341ffd7a87b0deac7e7076762bf57cec4751af5c6fda7"

23
production.yml
View File

@ -24,6 +24,7 @@ services:
- ./.envs/.production/.postgres
command: /start
redirect:
build:
context: .
@ -34,30 +35,16 @@ services:
depends_on:
- postgres
- redis
- mailhog
volumes:
- type: bind
source: /var/www/media/
target: /app/akarpov/media/
- .:/app:z
env_file:
- ./.envs/.production/.django
- ./.envs/.production/.postgres
ports:
- "3000:3000"
command: /start-redirect
traefik:
build:
context: .
dockerfile: ./compose/production/traefik/Dockerfile
image: akarpov_production_traefik
depends_on:
- django
- redirect
volumes:
- production_traefik:/etc/traefik/acme
ports:
- "0.0.0.0:80:80"
- "0.0.0.0:443:443"
- "0.0.0.0:3000:3000"
- "0.0.0.0:5555:5555"
postgres:
build:

2
pyproject.toml
View File

@ -107,8 +107,6 @@ pytest-xdist = "^3.3.1"
pytest-mock = "^3.11.1"
pytest-asyncio = "^0.21.1"
pytest-lambda = "^2.2.0"
pgvector = "^0.2.2"
pycld2 = "^0.41"
[build-system]