created base pipeliner, added structlog

akarpov/pipeliner/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class PipelinerConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "akarpov.pipeliner"

akarpov/pipeliner/migrations/0001_initial.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.0.8 on 2022-12-06 10:13
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('contenttypes', '0002_remove_content_type_name'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Workspace',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(blank=True, max_length=50)),
+                ('slug', models.SlugField(max_length=8)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='BaseBlock',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pipeline_blocks', to=settings.AUTH_USER_MODEL)),
+                ('polymorphic_ctype', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_%(app_label)s.%(class)s_set+', to='contenttypes.contenttype')),
+                ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='blocks', to='pipeliner.workspace')),
+            ],
+            options={
+                'abstract': False,
+                'base_manager_name': 'objects',
+            },
+        ),
+    ]

akarpov/pipeliner/migrations/0002_baseblock_created_baseblock_name_baseblock_parent_and_more.py

@@ -0,0 +1,36 @@
+# Generated by Django 4.0.8 on 2022-12-06 10:25
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeliner', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='baseblock',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='baseblock',
+            name='name',
+            field=models.CharField(blank=True, max_length=100),
+        ),
+        migrations.AddField(
+            model_name='baseblock',
+            name='parent',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='pipeliner.baseblock'),
+        ),
+        migrations.AddField(
+            model_name='baseblock',
+            name='updated',
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

akarpov/pipeliner/migrations/0003_constantnumberblock_constantstringblock.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.0.8 on 2022-12-07 10:52
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeliner', '0002_baseblock_created_baseblock_name_baseblock_parent_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ConstantNumberBlock',
+            fields=[
+                ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')),
+                ('number', models.DecimalField(decimal_places=2, max_digits=5)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=('pipeliner.baseblock',),
+        ),
+        migrations.CreateModel(
+            name='ConstantStringBlock',
+            fields=[
+                ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=('pipeliner.baseblock',),
+        ),
+    ]

akarpov/pipeliner/migrations/0004_multiplicationblock_storage_and_more.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.0.8 on 2022-12-07 11:08
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeliner', '0003_constantnumberblock_constantstringblock'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MultiplicationBlock',
+            fields=[
+                ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')),
+                ('by', models.DecimalField(decimal_places=2, max_digits=5)),
+            ],
+            options={
+                'abstract': False,
+                'base_manager_name': 'objects',
+            },
+            bases=('pipeliner.baseblock',),
+        ),
+        migrations.CreateModel(
+            name='Storage',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('data', models.JSONField(default=dict)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='constantstringblock',
+            name='string',
+            field=models.TextField(default='wf'),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name='TrashBlock',
+            fields=[
+                ('baseblock_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='pipeliner.baseblock')),
+                ('storage', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pipeliner.storage')),
+            ],
+            options={
+                'abstract': False,
+                'base_manager_name': 'objects',
+            },
+            bases=('pipeliner.baseblock',),
+        ),
+    ]

akarpov/pipeliner/models/__init__.py

@@ -0,0 +1,3 @@
+from .base import *  # noqa
+from .basic import *  # noqa
+from .manage import *  # noqa

akarpov/pipeliner/models/base.py

@@ -0,0 +1,48 @@
+import uuid
+
+from django.db import models
+from polymorphic.models import PolymorphicModel
+
+
+class BaseBlock(PolymorphicModel):
+    """Base block for pipelines for further explanation check examples"""
+
+    TYPE = "Base"
+
+    name: models.CharField
+    created: models.DateTimeField
+    updated: models.DateTimeField
+
+    creator: models.ForeignKey
+    workspace: models.ForeignKey
+    parent: models.ForeignKey
+
+    name = models.CharField(max_length=100, blank=True)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+
+    creator = models.ForeignKey(
+        "users.User", related_name="pipeline_blocks", on_delete=models.CASCADE
+    )
+    workspace = models.ForeignKey(
+        "Workspace", related_name="blocks", on_delete=models.CASCADE
+    )
+    parent = models.ForeignKey(
+        "self", null=True, related_name="children", on_delete=models.SET_NULL
+    )
+
+    def __str__(self):
+        return f"{self.TYPE} block"
+
+
+class ProviderBlock(BaseBlock):
+    TYPE = "Provider"
+    parent = None
+
+    class Meta:
+        abstract = True
+
+
+class Storage(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    data = models.JSONField(default=dict)

akarpov/pipeliner/models/basic/__init__.py

@@ -0,0 +1,3 @@
+from .modifiers import *  # noqa
+from .output import *  # noqa
+from .provides import *  # noqa

akarpov/pipeliner/models/basic/modifiers.py

@@ -0,0 +1,9 @@
+from django.db import models
+
+from akarpov.pipeliner.models import BaseBlock
+
+
+class MultiplicationBlock(BaseBlock):
+    TYPE = "Multiply"
+
+    by = models.DecimalField(max_digits=5, decimal_places=2)

akarpov/pipeliner/models/basic/output.py

@@ -0,0 +1,9 @@
+from django.db import models
+
+from akarpov.pipeliner.models import BaseBlock
+
+
+class TrashBlock(BaseBlock):
+    TYPE = "Trash"
+
+    storage = models.ForeignKey("Storage", on_delete=models.CASCADE)

akarpov/pipeliner/models/basic/provides.py

@@ -0,0 +1,15 @@
+from django.db import models
+
+from akarpov.pipeliner.models import ProviderBlock
+
+
+class ConstantNumberBlock(ProviderBlock):
+    TYPE = "Number"
+
+    number = models.DecimalField(max_digits=5, decimal_places=2)
+
+
+class ConstantStringBlock(ProviderBlock):
+    TYPE = "String"
+
+    string = models.TextField()

akarpov/pipeliner/models/manage.py

@@ -0,0 +1,9 @@
+from django.db import models
+
+
+class Workspace(models.Model):
+    name = models.CharField(max_length=50, blank=True)
+    slug = models.SlugField(max_length=8)
+
+    def __str__(self):
+        return self.name

akarpov/pipeliner/services/run.py

@@ -0,0 +1,33 @@
+from akarpov.pipeliner.models import BaseBlock
+
+
+class BlockRunner:
+    """iterates over block in tree order"""
+
+    def __init__(self, parent_block: BaseBlock):
+        self.parent_block = parent_block
+        self.root = self.get_root()
+        self.order = self.get_order(self.root)
+
+    def __iter__(self):
+        self.block = self.parent_block.get_root()
+        return self
+
+    def __str__(self):
+        return f"block runner for {self.root}, currently running {self.block}"
+
+    def get_order(self, block: BaseBlock) -> list[BaseBlock]:
+        order = []
+        for children in block.children.all():
+            order.extend(self.get_order(children))
+        return order
+
+    def get_root(self):
+        root = self.parent_block
+        if root.parent:
+            while root.parent:
+                root = root.parent
+        return root
+
+    def __next__(self):
+        yield from self.order

config/settings/base.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 
 import environ
+import structlog
 
 ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent
 # akarpov/
@@ -81,6 +82,7 @@
     "ckeditor",
     "ckeditor_uploader",
     "colorfield",
+    "polymorphic",
 ]
 
 HEALTH_CHECKS = [
@@ -105,6 +107,7 @@
 LOCAL_APPS = [
     "akarpov.users",
     "akarpov.blog",
+    "akarpov.pipeliner"
     # Your stuff: custom apps go here
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -158,6 +161,7 @@
     "django.middleware.security.SecurityMiddleware",
     "corsheaders.middleware.CorsMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
+    "django_structlog.middlewares.RequestMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.locale.LocaleMiddleware",
     "django.middleware.common.CommonMiddleware",
@@ -264,25 +268,72 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#logging
 # See https://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
+
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "verbose": {
+        "json_formatter": {
-            "format": "%(levelname)s %(asctime)s %(module)s "
+            "()": structlog.stdlib.ProcessorFormatter,
-            "%(process)d %(thread)d %(message)s"
+            "processor": structlog.processors.JSONRenderer(),
-        }
+        },
+        "plain_console": {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.dev.ConsoleRenderer(),
+        },
+        "key_value": {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.processors.KeyValueRenderer(
+                key_order=["timestamp", "level", "event", "logger"]
+            ),
+        },
     },
     "handlers": {
         "console": {
-            "level": "DEBUG",
             "class": "logging.StreamHandler",
-            "formatter": "verbose",
+            "formatter": "plain_console",
-        }
+        },
+        "json_file": {
+            "class": "logging.handlers.WatchedFileHandler",
+            "filename": "logs/json.log",
+            "formatter": "json_formatter",
+        },
+        "flat_line_file": {
+            "class": "logging.handlers.WatchedFileHandler",
+            "filename": "logs/flat_line.log",
+            "formatter": "key_value",
+        },
+    },
+    "loggers": {
+        "django_structlog": {
+            "handlers": ["console", "flat_line_file", "json_file"],
+            "level": "INFO",
+        },
+        # Make sure to replace the following logger's name for yours
+        "django_structlog_demo_project": {
+            "handlers": ["console", "flat_line_file", "json_file"],
+            "level": "INFO",
+        },
     },
-    "root": {"level": "INFO", "handlers": ["console"]},
 }
 
+structlog.configure(
+    processors=[
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.filter_by_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+        structlog.processors.UnicodeDecoder(),
+        structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+    ],
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    cache_logger_on_first_use=True,
+)
+
 # Celery
 # ------------------------------------------------------------------------------
 if USE_TZ:

requirements/base.txt

@@ -14,6 +14,7 @@ flower==1.2.0  # https://github.com/mher/flower
 # ------------------------------------------------------------------------------
 django==4.0.8  # pyup: < 4.1  # https://www.djangoproject.com/
 django-health-check==3.17.0
+django-structlog==4.0.1
 django-environ==0.9.0  # https://github.com/joke2k/django-environ
 django-model-utils==4.2.0  # https://github.com/jazzband/django-model-utils
 django-allauth==0.51.0  # https://github.com/pennersr/django-allauth