From 60ceb2d2ae912804c665960d95afa5ff787fedcc Mon Sep 17 00:00:00 2001 From: Alexandr Karpov Date: Tue, 13 Dec 2022 13:21:50 +0300 Subject: [PATCH] created base pipeliner, added structlog --- akarpov/pipeliner/__init__.py | 0 akarpov/pipeliner/apps.py | 6 ++ akarpov/pipeliner/migrations/0001_initial.py | 39 +++++++++++ ...aseblock_name_baseblock_parent_and_more.py | 36 ++++++++++ ...constantnumberblock_constantstringblock.py | 35 ++++++++++ ...04_multiplicationblock_storage_and_more.py | 52 ++++++++++++++ akarpov/pipeliner/migrations/__init__.py | 0 akarpov/pipeliner/models/__init__.py | 3 + akarpov/pipeliner/models/base.py | 48 +++++++++++++ akarpov/pipeliner/models/basic/__init__.py | 3 + akarpov/pipeliner/models/basic/modifiers.py | 9 +++ akarpov/pipeliner/models/basic/output.py | 9 +++ akarpov/pipeliner/models/basic/provides.py | 15 +++++ akarpov/pipeliner/models/manage.py | 9 +++ akarpov/pipeliner/models/web/__init__.py | 0 akarpov/pipeliner/models/web/data.py | 0 akarpov/pipeliner/models/web/input.py | 0 akarpov/pipeliner/models/web/output.py | 0 akarpov/pipeliner/services/__init__.py | 0 akarpov/pipeliner/services/run.py | 33 +++++++++ config/settings/base.py | 67 ++++++++++++++++--- requirements/base.txt | 1 + 22 files changed, 357 insertions(+), 8 deletions(-) create mode 100644 akarpov/pipeliner/__init__.py create mode 100644 akarpov/pipeliner/apps.py create mode 100644 akarpov/pipeliner/migrations/0001_initial.py create mode 100644 akarpov/pipeliner/migrations/0002_baseblock_created_baseblock_name_baseblock_parent_and_more.py create mode 100644 akarpov/pipeliner/migrations/0003_constantnumberblock_constantstringblock.py create mode 100644 akarpov/pipeliner/migrations/0004_multiplicationblock_storage_and_more.py create mode 100644 akarpov/pipeliner/migrations/__init__.py create mode 100644 akarpov/pipeliner/models/__init__.py create mode 100644 akarpov/pipeliner/models/base.py create mode 100644 akarpov/pipeliner/models/basic/__init__.py create mode 100644 akarpov/pipeliner/models/basic/modifiers.py create mode 100644 akarpov/pipeliner/models/basic/output.py create mode 100644 akarpov/pipeliner/models/basic/provides.py create mode 100644 akarpov/pipeliner/models/manage.py create mode 100644 akarpov/pipeliner/models/web/__init__.py create mode 100644 akarpov/pipeliner/models/web/data.py create mode 100644 akarpov/pipeliner/models/web/input.py create mode 100644 akarpov/pipeliner/models/web/output.py create mode 100644 akarpov/pipeliner/services/__init__.py create mode 100644 akarpov/pipeliner/services/run.py diff --git a/akarpov/pipeliner/__init__.py b/akarpov/pipeliner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/apps.py b/akarpov/pipeliner/apps.py new file mode 100644 index 0000000..40d8076 --- /dev/null +++ b/akarpov/pipeliner/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class PipelinerConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "akarpov.pipeliner" diff --git a/akarpov/pipeliner/migrations/0001_initial.py b/akarpov/pipeliner/migrations/0001_initial.py new file mode 100644 index 0000000..4d66d3a --- /dev/null +++ b/akarpov/pipeliner/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# Generated by Django 4.0.8 on 2022-12-06 10:13 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='Workspace', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=50)), + ('slug', models.SlugField(max_length=8)), + ], + ), + migrations.CreateModel( + name='BaseBlock', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pipeline_blocks', to=settings.AUTH_USER_MODEL)), + ('polymorphic_ctype', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_%(app_label)s.%(class)s_set+', to='contenttypes.contenttype')), + ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='blocks', to='pipeliner.workspace')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + ), + ] diff --git a/akarpov/pipeliner/migrations/0002_baseblock_created_baseblock_name_baseblock_parent_and_more.py b/akarpov/pipeliner/migrations/0002_baseblock_created_baseblock_name_baseblock_parent_and_more.py new file mode 100644 index 0000000..16d5a2e --- /dev/null +++ b/akarpov/pipeliner/migrations/0002_baseblock_created_baseblock_name_baseblock_parent_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 4.0.8 on 2022-12-06 10:25 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('pipeliner', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='baseblock', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='baseblock', + name='name', + field=models.CharField(blank=True, max_length=100), + ), + migrations.AddField( + model_name='baseblock', + name='parent', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='pipeliner.baseblock'), + ), + migrations.AddField( + model_name='baseblock', + name='updated', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/akarpov/pipeliner/migrations/0003_constantnumberblock_constantstringblock.py b/akarpov/pipeliner/migrations/0003_constantnumberblock_constantstringblock.py new file mode 100644 index 0000000..1479166 --- /dev/null +++ b/akarpov/pipeliner/migrations/0003_constantnumberblock_constantstringblock.py @@ -0,0 +1,35 @@ +# Generated by Django 4.0.8 on 2022-12-07 10:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('pipeliner', '0002_baseblock_created_baseblock_name_baseblock_parent_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ConstantNumberBlock', + fields=[ + ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')), + ('number', models.DecimalField(decimal_places=2, max_digits=5)), + ], + options={ + 'abstract': False, + }, + bases=('pipeliner.baseblock',), + ), + migrations.CreateModel( + name='ConstantStringBlock', + fields=[ + ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')), + ], + options={ + 'abstract': False, + }, + bases=('pipeliner.baseblock',), + ), + ] diff --git a/akarpov/pipeliner/migrations/0004_multiplicationblock_storage_and_more.py b/akarpov/pipeliner/migrations/0004_multiplicationblock_storage_and_more.py new file mode 100644 index 0000000..565a4aa --- /dev/null +++ b/akarpov/pipeliner/migrations/0004_multiplicationblock_storage_and_more.py @@ -0,0 +1,52 @@ +# Generated by Django 4.0.8 on 2022-12-07 11:08 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('pipeliner', '0003_constantnumberblock_constantstringblock'), + ] + + operations = [ + migrations.CreateModel( + name='MultiplicationBlock', + fields=[ + ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')), + ('by', models.DecimalField(decimal_places=2, max_digits=5)), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + bases=('pipeliner.baseblock',), + ), + migrations.CreateModel( + name='Storage', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('data', models.JSONField(default=dict)), + ], + ), + migrations.AddField( + model_name='constantstringblock', + name='string', + field=models.TextField(default='wf'), + preserve_default=False, + ), + migrations.CreateModel( + name='TrashBlock', + fields=[ + ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')), + ('storage', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pipeliner.storage')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + bases=('pipeliner.baseblock',), + ), + ] diff --git a/akarpov/pipeliner/migrations/__init__.py b/akarpov/pipeliner/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/models/__init__.py b/akarpov/pipeliner/models/__init__.py new file mode 100644 index 0000000..f547bd1 --- /dev/null +++ b/akarpov/pipeliner/models/__init__.py @@ -0,0 +1,3 @@ +from .base import * # noqa +from .basic import * # noqa +from .manage import * # noqa diff --git a/akarpov/pipeliner/models/base.py b/akarpov/pipeliner/models/base.py new file mode 100644 index 0000000..4d89d82 --- /dev/null +++ b/akarpov/pipeliner/models/base.py @@ -0,0 +1,48 @@ +import uuid + +from django.db import models +from polymorphic.models import PolymorphicModel + + +class BaseBlock(PolymorphicModel): + """Base block for pipelines for further explanation check examples""" + + TYPE = "Base" + + name: models.CharField + created: models.DateTimeField + updated: models.DateTimeField + + creator: models.ForeignKey + workspace: models.ForeignKey + parent: models.ForeignKey + + name = models.CharField(max_length=100, blank=True) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + creator = models.ForeignKey( + "users.User", related_name="pipeline_blocks", on_delete=models.CASCADE + ) + workspace = models.ForeignKey( + "Workspace", related_name="blocks", on_delete=models.CASCADE + ) + parent = models.ForeignKey( + "self", null=True, related_name="children", on_delete=models.SET_NULL + ) + + def __str__(self): + return f"{self.TYPE} block" + + +class ProviderBlock(BaseBlock): + TYPE = "Provider" + parent = None + + class Meta: + abstract = True + + +class Storage(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + data = models.JSONField(default=dict) diff --git a/akarpov/pipeliner/models/basic/__init__.py b/akarpov/pipeliner/models/basic/__init__.py new file mode 100644 index 0000000..3e591b2 --- /dev/null +++ b/akarpov/pipeliner/models/basic/__init__.py @@ -0,0 +1,3 @@ +from .modifiers import * # noqa +from .output import * # noqa +from .provides import * # noqa diff --git a/akarpov/pipeliner/models/basic/modifiers.py b/akarpov/pipeliner/models/basic/modifiers.py new file mode 100644 index 0000000..653c7f1 --- /dev/null +++ b/akarpov/pipeliner/models/basic/modifiers.py @@ -0,0 +1,9 @@ +from django.db import models + +from akarpov.pipeliner.models import BaseBlock + + +class MultiplicationBlock(BaseBlock): + TYPE = "Multiply" + + by = models.DecimalField(max_digits=5, decimal_places=2) diff --git a/akarpov/pipeliner/models/basic/output.py b/akarpov/pipeliner/models/basic/output.py new file mode 100644 index 0000000..6be6969 --- /dev/null +++ b/akarpov/pipeliner/models/basic/output.py @@ -0,0 +1,9 @@ +from django.db import models + +from akarpov.pipeliner.models import BaseBlock + + +class TrashBlock(BaseBlock): + TYPE = "Trash" + + storage = models.ForeignKey("Storage", on_delete=models.CASCADE) diff --git a/akarpov/pipeliner/models/basic/provides.py b/akarpov/pipeliner/models/basic/provides.py new file mode 100644 index 0000000..776e46d --- /dev/null +++ b/akarpov/pipeliner/models/basic/provides.py @@ -0,0 +1,15 @@ +from django.db import models + +from akarpov.pipeliner.models import ProviderBlock + + +class ConstantNumberBlock(ProviderBlock): + TYPE = "Number" + + number = models.DecimalField(max_digits=5, decimal_places=2) + + +class ConstantStringBlock(ProviderBlock): + TYPE = "String" + + string = models.TextField() diff --git a/akarpov/pipeliner/models/manage.py b/akarpov/pipeliner/models/manage.py new file mode 100644 index 0000000..ed8c291 --- /dev/null +++ b/akarpov/pipeliner/models/manage.py @@ -0,0 +1,9 @@ +from django.db import models + + +class Workspace(models.Model): + name = models.CharField(max_length=50, blank=True) + slug = models.SlugField(max_length=8) + + def __str__(self): + return self.name diff --git a/akarpov/pipeliner/models/web/__init__.py b/akarpov/pipeliner/models/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/models/web/data.py b/akarpov/pipeliner/models/web/data.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/models/web/input.py b/akarpov/pipeliner/models/web/input.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/models/web/output.py b/akarpov/pipeliner/models/web/output.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/services/__init__.py b/akarpov/pipeliner/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/akarpov/pipeliner/services/run.py b/akarpov/pipeliner/services/run.py new file mode 100644 index 0000000..5651d4f --- /dev/null +++ b/akarpov/pipeliner/services/run.py @@ -0,0 +1,33 @@ +from akarpov.pipeliner.models import BaseBlock + + +class BlockRunner: + """iterates over block in tree order""" + + def __init__(self, parent_block: BaseBlock): + self.parent_block = parent_block + self.root = self.get_root() + self.order = self.get_order(self.root) + + def __iter__(self): + self.block = self.parent_block.get_root() + return self + + def __str__(self): + return f"block runner for {self.root}, currently running {self.block}" + + def get_order(self, block: BaseBlock) -> list[BaseBlock]: + order = [] + for children in block.children.all(): + order.extend(self.get_order(children)) + return order + + def get_root(self): + root = self.parent_block + if root.parent: + while root.parent: + root = root.parent + return root + + def __next__(self): + yield from self.order diff --git a/config/settings/base.py b/config/settings/base.py index 1dd96eb..e112bea 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -4,6 +4,7 @@ from pathlib import Path import environ +import structlog ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent # akarpov/ @@ -81,6 +82,7 @@ "ckeditor", "ckeditor_uploader", "colorfield", + "polymorphic", ] HEALTH_CHECKS = [ @@ -105,6 +107,7 @@ LOCAL_APPS = [ "akarpov.users", "akarpov.blog", + "akarpov.pipeliner" # Your stuff: custom apps go here ] # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps @@ -158,6 +161,7 @@ "django.middleware.security.SecurityMiddleware", "corsheaders.middleware.CorsMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", + "django_structlog.middlewares.RequestMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.locale.LocaleMiddleware", "django.middleware.common.CommonMiddleware", @@ -264,25 +268,72 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#logging # See https://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. + LOGGING = { "version": 1, "disable_existing_loggers": False, "formatters": { - "verbose": { - "format": "%(levelname)s %(asctime)s %(module)s " - "%(process)d %(thread)d %(message)s" - } + "json_formatter": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.processors.JSONRenderer(), + }, + "plain_console": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.dev.ConsoleRenderer(), + }, + "key_value": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.processors.KeyValueRenderer( + key_order=["timestamp", "level", "event", "logger"] + ), + }, }, "handlers": { "console": { - "level": "DEBUG", "class": "logging.StreamHandler", - "formatter": "verbose", - } + "formatter": "plain_console", + }, + "json_file": { + "class": "logging.handlers.WatchedFileHandler", + "filename": "logs/json.log", + "formatter": "json_formatter", + }, + "flat_line_file": { + "class": "logging.handlers.WatchedFileHandler", + "filename": "logs/flat_line.log", + "formatter": "key_value", + }, + }, + "loggers": { + "django_structlog": { + "handlers": ["console", "flat_line_file", "json_file"], + "level": "INFO", + }, + # Make sure to replace the following logger's name for yours + "django_structlog_demo_project": { + "handlers": ["console", "flat_line_file", "json_file"], + "level": "INFO", + }, }, - "root": {"level": "INFO", "handlers": ["console"]}, } +structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + structlog.stdlib.filter_by_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, +) + # Celery # ------------------------------------------------------------------------------ if USE_TZ: diff --git a/requirements/base.txt b/requirements/base.txt index f75a298..b1420ab 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,6 +14,7 @@ flower==1.2.0 # https://github.com/mher/flower # ------------------------------------------------------------------------------ django==4.0.8 # pyup: < 4.1 # https://www.djangoproject.com/ django-health-check==3.17.0 +django-structlog==4.0.1 django-environ==0.9.0 # https://github.com/joke2k/django-environ django-model-utils==4.2.0 # https://github.com/jazzband/django-model-utils django-allauth==0.51.0 # https://github.com/pennersr/django-allauth