Merge remote-tracking branch 'origin/main'

.env.example

@@ -1,5 +1,6 @@
 DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/akarpov
 CELERY_BROKER_URL=redis://localhost:6379/0
 REDIS_URL=redis://localhost:6379/1
+REDIS_CACHE=rediscache://localhost:6379/1
 USE_DOCKER=no
 EMAIL_HOST=127.0.0.1:8025

.envs/.local/.django

@@ -5,8 +5,9 @@ IPYTHONDIR=/app/.ipython
 DJANGO_READ_DOT_ENV_FILE=no
 # Redis
 # ------------------------------------------------------------------------------
-REDIS_URL=redis://redis:6379/0
+REDIS_URL=redis://redis:6379/1
-CELERY_BROKER_URL=redis://localhost:6379/0
+REDIS_CACHE=rediscache://redis:6379/1
+CELERY_BROKER_URL=redis://redis:6379/0
 
 # Celery
 # ------------------------------------------------------------------------------

.envs/.production/.django

@@ -0,0 +1,16 @@
+# General
+# CHANGE ON REAL SERVER
+# ------------------------------------------------------------------------------
+USE_DOCKER=yes
+DJANGO_READ_DOT_ENV_FILE=no
+# Redis
+# ------------------------------------------------------------------------------
+REDIS_URL=redis://redis:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+
+# Celery
+# ------------------------------------------------------------------------------
+
+# Flower
+CELERY_FLOWER_USER=debug
+CELERY_FLOWER_PASSWORD=debug

.envs/.production/.postgres

@@ -0,0 +1,7 @@
+# PostgreSQL
+# ------------------------------------------------------------------------------
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_DB=akarpov
+POSTGRES_USER=debug
+POSTGRES_PASSWORD=debug

.github/workflows/ci.yml

@@ -26,11 +26,15 @@ jobs:
       - name: Checkout Code Repository
         uses: actions/checkout@v3
 
-      - name: Set up Python
+      - uses: actions/checkout@v3
-        uses: actions/setup-python@v3
+      - name: Install poetry
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: '3.11'
-          cache: poetry
+          cache: 'poetry'
+      - run: poetry install
 
       - name: Run pre-commit
         uses: pre-commit/action@v2.0.3
@@ -46,24 +50,8 @@ jobs:
       - name: Build the Stack
         run:  docker-compose -f local.yml build
 
-      - name: Run DB Migrations
-        run:  docker-compose -f local.yml run --rm django python manage.py migrate
-
       - name: Run Django Tests
         run:  docker-compose -f local.yml run django pytest
 
       - name: Tear down the Stack
         run:  docker-compose -f local.yml down
-
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout Code Repository
-        uses: actions/checkout@v3
-
-      - name: Build the Stack
-        run:  docker-compose -f production.yml build
-
-      - name: Tear down the Stack
-        run:  docker-compose -f production.yml down

akarpov/pipeliner/migrations/0005_basestorage_alter_storage_options_remove_storage_id_and_more.py

@@ -0,0 +1,88 @@
+# Generated by Django 4.1.4 on 2023-01-01 21:26
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("pipeliner", "0004_multiplicationblock_storage_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BaseStorage",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                (
+                    "polymorphic_ctype",
+                    models.ForeignKey(
+                        editable=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="polymorphic_%(app_label)s.%(class)s_set+",
+                        to="contenttypes.contenttype",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+                "base_manager_name": "objects",
+            },
+        ),
+        migrations.AlterModelOptions(
+            name="storage",
+            options={"base_manager_name": "objects"},
+        ),
+        migrations.RemoveField(
+            model_name="storage",
+            name="id",
+        ),
+        migrations.CreateModel(
+            name="RunnerStorage",
+            fields=[
+                (
+                    "basestorage_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="pipeliner.basestorage",
+                    ),
+                ),
+                ("data", models.JSONField(default=dict)),
+            ],
+            options={
+                "abstract": False,
+                "base_manager_name": "objects",
+            },
+            bases=("pipeliner.basestorage",),
+        ),
+        migrations.AddField(
+            model_name="storage",
+            name="basestorage_ptr",
+            field=models.OneToOneField(
+                auto_created=True,
+                default=123,
+                on_delete=django.db.models.deletion.CASCADE,
+                parent_link=True,
+                primary_key=True,
+                serialize=False,
+                to="pipeliner.basestorage",
+            ),
+            preserve_default=False,
+        ),
+    ]

compose/local/django/celery/beat/start

@@ -4,7 +4,6 @@ set -o errexit
 set -o nounset
 
 python manage.py makemigrations
-python manage.py migrate auth
 python manage.py migrate
 rm -f './celerybeat.pid'
 celery -A config.celery_app beat -l INFO

compose/local/django/start

@@ -5,7 +5,6 @@ set -o pipefail
 set -o nounset
 
 
-python manage.py makemigrations
 python manage.py migrate auth
 python manage.py migrate
 python manage.py runserver_plus 0.0.0.0:8000

compose/production/django/Dockerfile

@@ -1,5 +1,4 @@
-ARG PYTHON_VERSION=3.10-slim-bullseye
+ARG PYTHON_VERSION=3.11-slim
-
 
 
 # define an alias for the specfic python version used in this file.
@@ -17,13 +16,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # psycopg2 dependencies
   libpq-dev
 
-# Requirements are installed here to ensure they will be cached.
-COPY ./requirements .
-
-# Create Python Dependency and Sub-Dependency Wheels.
-RUN pip wheel --wheel-dir /usr/src/app/wheels  \
-  -r ${BUILD_ENVIRONMENT}.txt
-
 
 # Python 'run' stage
 FROM python as python-run-stage
@@ -51,13 +43,15 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
-# copy python dependency wheels from python-build-stage
-COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
 
-# use wheels to install python dependencies
+RUN pip install poetry
-RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
+
-  && rm -rf /wheels/
+# Configuring poetry
+RUN poetry config virtualenvs.create false
+COPY pyproject.toml poetry.lock /
+
+# Installing requirements
+RUN poetry install
 
 COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r$//g' /entrypoint

config/settings/base.py

@@ -56,14 +56,7 @@
 # ------------------------------------------------------------------------------
 CACHE_TTL = 60 * 1500
 
-CACHES = {
+CACHES = {"default": env.cache("REDIS_CACHE")}
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": env.db("REDIS_URL"),
-        "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
-        "KEY_PREFIX": "example",
-    }
-}
 SESSION_ENGINE = "django.contrib.sessions.backends.cache"
 SESSION_CACHE_ALIAS = "default"
 CACHEOPS_DEFAULTS = {"timeout": 60 * 60}
@@ -73,6 +66,7 @@
     "auth.permission": {"ops": "all"},
     "*.*": {},
 }
+CACHEOPS_REDIS = env.str("REDIS_URL")
 
 # URLS
 # ------------------------------------------------------------------------------
@@ -196,6 +190,7 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#middleware
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",
+    "django.middleware.cache.UpdateCacheMiddleware",
     "corsheaders.middleware.CorsMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
     "django_structlog.middlewares.RequestMiddleware",
@@ -207,6 +202,11 @@
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.common.BrokenLinkEmailsMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "cms.middleware.language.LanguageCookieMiddleware",
+    "cms.middleware.user.CurrentUserMiddleware",
+    "cms.middleware.page.CurrentPageMiddleware",
+    "cms.middleware.toolbar.ToolbarMiddleware",
+    "django.middleware.cache.FetchFromCacheMiddleware",
 ]
 
 # STATIC

pyproject.toml

@@ -12,13 +12,13 @@ psutil = "^5.9.4"
 python-slugify = "^7.0.0"
 pillow = "^9.3.0"
 argon2-cffi = "^21.3.0"
-whitenoise = "^6.2.0"
+whitenoise = "^6.3.0"
 redis = "^4.4.0"
 hiredis = "^2.1.0"
-celery = "^5.2.7"
+celery = {extras = ["redis"], version = "^5.2.7"}
 django-celery-beat = "^2.4.0"
 flower = "^1.2.0"
-django = "^4.1.4"
+django = "^4.1.5"
 django-health-check = "^3.17.0"
 django-structlog = "^4.0.1"
 django-environ = "^0.9.0"
@@ -41,11 +41,11 @@ django-stubs = "^1.13.1"
 pytest = "^7.2.0"
 pytest-sugar = "^0.9.6"
 djangorestframework-stubs = "^1.8.0"
-sphinx = "^5.3.0"
+sphinx = "^6.1.1"
 sphinx-autobuild = "^2021.3.14"
 flake8 = "^6.0.0"
 flake8-isort = "^6.0.0"
-coverage = "^7.0.1"
+coverage = "^7.0.3"
 black = "^22.12.0"
 pylint-django = "^2.5.3"
 pylint-celery = "^0.3"