added poetry(docker is not working for now), minor changes for pipeliner

.env.example

@@ -1,4 +1,5 @@
 DATABASE_URL=
 CELERY_BROKER_URL=
+REDIS_URL=
 USE_DOCKER=
 EMAIL_HOST=

akarpov/pipeliner/models/base.py

@@ -43,6 +43,15 @@ class Meta:
         abstract = True
 
 
-class Storage(models.Model):
+class BaseStorage(PolymorphicModel):
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    id: uuid.uuid4 = models.UUIDField(
+        primary_key=True, default=uuid.uuid4, editable=False
+    )
+
+
+class Storage(BaseStorage):
+    data = models.JSONField(default=dict)
+
+
+class RunnerStorage(BaseStorage):
     data = models.JSONField(default=dict)

akarpov/pipeliner/tasks/runer.py

@@ -0,0 +1,6 @@
+from celery import shared_task
+
+
+@shared_task()
+def run_pipe_thread(pk: int):
+    return pk

compose/local/django/Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.10-slim-bullseye
+ARG PYTHON_VERSION=3.11-slim
 
 # define an alias for the specfic python version used in this file.
 FROM python:${PYTHON_VERSION} as python
@@ -15,12 +15,18 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # psycopg2 dependencies
   libpq-dev
 
-# Requirements are installed here to ensure they will be cached.
+RUN pip install poetry==1.2.2
-COPY ./requirements .
 
-# Create Python Dependency and Sub-Dependency Wheels.
+# Configuring poetry
-RUN pip wheel --wheel-dir /usr/src/app/wheels  \
+RUN poetry config virtualenvs.create false
-  -r ${BUILD_ENVIRONMENT}.txt
+COPY pyproject.toml poetry.lock /
+
+# Installing requirements
+RUN poetry install
+# Removing gcc
+RUN apt-get purge -y \
+  gcc \
+  && rm -rf /var/lib/apt/lists/*
 
 
 # Python 'run' stage
@@ -45,13 +51,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
-# copy python dependency wheels from python-build-stage
-COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
-
-# use wheels to install python dependencies
-RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
-	&& rm -rf /wheels/
 
 COPY ./compose/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r$//g' /entrypoint

compose/local/docs/Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.10-slim-bullseye
+ARG PYTHON_VERSION=3.11-slim
 
 # define an alias for the specfic python version used in this file.
 FROM python:${PYTHON_VERSION} as python
@@ -18,13 +18,19 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# Requirements are installed here to ensure they will be cached.
-COPY ./requirements /requirements
 
-# create python dependency wheels
+RUN pip install poetry==1.2.2
-RUN pip wheel --no-cache-dir --wheel-dir /usr/src/app/wheels  \
+
-  -r /requirements/local.txt -r /requirements/production.txt \
+# Configuring poetry
-  && rm -rf /requirements
+RUN poetry config virtualenvs.create false
+COPY pyproject.toml poetry.lock /
+
+# Installing requirements
+RUN poetry install
+# Removing gcc
+RUN apt-get purge -y \
+  gcc \
+  && rm -rf /var/lib/apt/lists/*
 
 
 # Python 'run' stage
@@ -50,12 +56,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# copy python dependency wheels from python-build-stage
-COPY --from=python-build-stage /usr/src/app/wheels /wheels
-
-# use wheels to install python dependencies
-RUN pip install --no-cache /wheels/* \
-  && rm -rf /wheels
 
 COPY ./compose/local/docs/start /start-docs
 RUN sed -i 's/\r$//g' /start-docs

config/settings/base.py

@@ -42,7 +42,7 @@
 # DATABASES
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#databases
-DATABASES = {"default": env.db("DATABASE_URL")}
+DATABASES = {"default": env.db("DATABASE_URL"), "memory": env.db("REDIS_URL")}
 DATABASES["default"]["ATOMIC_REQUESTS"] = True
 # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-DEFAULT_AUTO_FIELD
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

pyproject.toml

@@ -0,0 +1,67 @@
+[tool.poetry]
+name = "akarpov"
+version = "0.1.0"
+description = ""
+authors = ["Alexandr Karpov <alexandr.d.karpov@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+pytz = "^2022.7"
+psutil = "^5.9.4"
+python-slugify = "^7.0.0"
+pillow = "^9.3.0"
+argon2-cffi = "^21.3.0"
+whitenoise = "^6.2.0"
+redis = "^4.4.0"
+hiredis = "^2.1.0"
+celery = "^5.2.7"
+django-celery-beat = "^2.4.0"
+flower = "^1.2.0"
+django = "^4.1.4"
+django-health-check = "^3.17.0"
+django-structlog = "^4.0.1"
+django-environ = "^0.9.0"
+django-model-utils = "^4.3.1"
+django-allauth = "^0.51.0"
+django-crispy-forms = "^1.14.0"
+crispy-bootstrap5 = "^0.7"
+django-redis = "^5.2.0"
+django-ckeditor = "^6.5.1"
+django-colorfield = "^0.8.0"
+djangorestframework = "^3.14.0"
+django-rest-auth = "^0.9.5"
+django-cors-headers = "^3.13.0"
+drf-spectacular = "^0.25.1"
+werkzeug = {extras = ["watchdog"], version = "^2.2.2"}
+ipdb = "^0.13.11"
+watchfiles = "^0.18.1"
+mypy = "^0.991"
+django-stubs = "^1.13.1"
+pytest = "^7.2.0"
+pytest-sugar = "^0.9.6"
+djangorestframework-stubs = "^1.8.0"
+sphinx = "^5.3.0"
+sphinx-autobuild = "^2021.3.14"
+flake8 = "^6.0.0"
+flake8-isort = "^5.0.3"
+coverage = "^6.5.0"
+black = "^22.12.0"
+pylint-django = "^2.5.3"
+pylint-celery = "^0.3"
+pre-commit = "^2.20.0"
+factory-boy = "^3.2.1"
+django-debug-toolbar = "^3.8.1"
+django-extensions = "^3.2.1"
+django-coverage-plugin = "^3.0.0"
+pytest-django = "^4.5.2"
+gunicorn = "^20.1.0"
+sentry-sdk = "^1.12.0"
+django-anymail = {extras = ["mailgun"], version = "^8.6"}
+psycopg2-binary = "^2.9.5"
+django-polymorphic = "^3.1.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements/base.txt

@@ -1,31 +0,0 @@
-pytz==2022.6  # https://github.com/stub42/pytz
-psutil==5.9.4
-python-slugify==7.0.0  # https://github.com/un33k/python-slugify
-Pillow==9.3.0  # https://github.com/python-pillow/Pillow
-argon2-cffi==21.3.0  # https://github.com/hynek/argon2_cffi
-whitenoise==6.2.0  # https://github.com/evansd/whitenoise
-redis==4.4.0  # https://github.com/redis/redis-py
-hiredis==2.0.0  # https://github.com/redis/hiredis-py
-celery==5.2.7  # pyup: < 6.0  # https://github.com/celery/celery
-django-celery-beat==2.4.0  # https://github.com/celery/django-celery-beat
-flower==1.2.0  # https://github.com/mher/flower
-
-# Django
-# ------------------------------------------------------------------------------
-django==4.1.4  # pyup: < 4.1  # https://www.djangoproject.com/
-django-health-check==3.17.0
-django-structlog==4.0.1
-django-environ==0.9.0  # https://github.com/joke2k/django-environ
-django-model-utils==4.2.0  # https://github.com/jazzband/django-model-utils
-django-allauth==0.51.0  # https://github.com/pennersr/django-allauth
-django-crispy-forms==1.14.0  # https://github.com/django-crispy-forms/django-crispy-forms
-crispy-bootstrap5==0.7  # https://github.com/django-crispy-forms/crispy-bootstrap5
-django-redis==5.2.0  # https://github.com/jazzband/django-redis
-django-ckeditor==6.5.1
-django-colorfield==0.8.0
-# Django REST Framework
-djangorestframework==3.14.0  # https://github.com/encode/django-rest-framework
-django-rest-auth==0.9.5
-django-cors-headers==3.13.0 # https://github.com/adamchainz/django-cors-headers
-# DRF-spectacular for api documentation
-drf-spectacular==0.24.2  # https://github.com/tfranzel/drf-spectacular

requirements/local.txt

@@ -1,38 +0,0 @@
--r base.txt
-
-Werkzeug[watchdog]==2.2.2 # https://github.com/pallets/werkzeug
-ipdb==0.13.9  # https://github.com/gotcha/ipdb
-psycopg2==2.9.5  # https://github.com/psycopg/psycopg2
-watchfiles==0.18.1  # https://github.com/samuelcolvin/watchfiles
-
-# Testing
-# ------------------------------------------------------------------------------
-mypy==0.982  # https://github.com/python/mypy
-django-stubs==1.13.0  # https://github.com/typeddjango/django-stubs
-pytest==7.2.0  # https://github.com/pytest-dev/pytest
-pytest-sugar==0.9.6  # https://github.com/Frozenball/pytest-sugar
-djangorestframework-stubs==1.7.0  # https://github.com/typeddjango/djangorestframework-stubs
-
-# Documentation
-# ------------------------------------------------------------------------------
-sphinx==5.3.0  # https://github.com/sphinx-doc/sphinx
-sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild
-
-# Code quality
-# ------------------------------------------------------------------------------
-flake8==6.0.0  # https://github.com/PyCQA/flake8
-flake8-isort==5.0.3  # https://github.com/gforcada/flake8-isort
-coverage==6.5.0  # https://github.com/nedbat/coveragepy
-black==22.10.0  # https://github.com/psf/black
-pylint-django==2.5.3  # https://github.com/PyCQA/pylint-django
-pylint-celery==0.3  # https://github.com/PyCQA/pylint-celery
-pre-commit==2.20.0  # https://github.com/pre-commit/pre-commit
-
-# Django
-# ------------------------------------------------------------------------------
-factory-boy==3.2.1  # https://github.com/FactoryBoy/factory_boy
-
-django-debug-toolbar==3.8.1  # https://github.com/jazzband/django-debug-toolbar
-django-extensions==3.2.1  # https://github.com/django-extensions/django-extensions
-django-coverage-plugin==3.0.0  # https://github.com/nedbat/django_coverage_plugin
-pytest-django==4.5.2  # https://github.com/pytest-dev/pytest-django

requirements/production.txt

@@ -1,11 +0,0 @@
-# PRECAUTION: avoid production dependencies that aren't in development
-
--r base.txt
-
-gunicorn==20.1.0  # https://github.com/benoitc/gunicorn
-psycopg2==2.9.5  # https://github.com/psycopg/psycopg2
-sentry-sdk==1.11.1  # https://github.com/getsentry/sentry-python
-
-# Django
-# ------------------------------------------------------------------------------
-django-anymail[mailgun]==8.6  # https://github.com/anymail/django-anymail