added celery, postgres, whitenoise to docker

.dockerignore

@@ -0,0 +1,10 @@
+.editorconfig
+.gitattributes
+.github
+.gitignore
+.gitlab-ci.yml
+.idea
+.pre-commit-config.yaml
+.readthedocs.yml
+.travis.yml
+venv

.editorconfig

@@ -0,0 +1,27 @@
+# http://editorconfig.org
+
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{py,rst,ini}]
+indent_style = space
+indent_size = 4
+
+[*.{html,css,scss,json,yml,xml}]
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab
+
+[nginx.conf]
+indent_style = space
+indent_size = 2

Dockerfile

@@ -4,5 +4,5 @@ ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 WORKDIR /code
 COPY requirements/* /code/
-RUN pip install -r prod.txt
+RUN pip install -r production.txt
-COPY . /code/
+COPY app/* /code/

README.md

@@ -5,6 +5,7 @@
 Стек технологий:
 - Django, DRF, Channels
 - Celery
+- Celery Beat
 - Postgresql
 - Swagger
 
@@ -33,4 +34,4 @@ $ python3 manage.py runserver
 #### Запуск celery
 ```shell
 $ celery -A conf worker --loglevel=INFO
-```
+```

app/conf/settings/deploy.py

@@ -1 +1,84 @@
-from .base import *
+from .base import *  # noqa
+from .base import env
+
+# GENERAL
+# ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
+SECRET_KEY = env("DJANGO_SECRET_KEY")
+# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["example.com"])
+
+# SECURITY
+# ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+# https://docs.djangoproject.com/en/dev/ref/settings/#secure-ssl-redirect
+SECURE_SSL_REDIRECT = env.bool("DJANGO_SECURE_SSL_REDIRECT", default=True)
+# https://docs.djangoproject.com/en/dev/ref/settings/#session-cookie-secure
+SESSION_COOKIE_SECURE = True
+# https://docs.djangoproject.com/en/dev/ref/settings/#csrf-cookie-secure
+CSRF_COOKIE_SECURE = True
+# https://docs.djangoproject.com/en/dev/topics/security/#ssl-https
+# https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-seconds
+# https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-include-subdomains
+SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
+    "DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True
+)
+# https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-preload
+SECURE_HSTS_PRELOAD = env.bool("DJANGO_SECURE_HSTS_PRELOAD", default=True)
+# https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff
+SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
+    "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True
+)
+
+# STATIC
+# ------------------------
+STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+# MEDIA
+# ------------------------------------------------------------------------------
+
+
+# LOGGING
+# ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#logging
+# See https://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
+    "formatters": {
+        "verbose": {
+            "format": "%(levelname)s %(asctime)s %(module)s "
+            "%(process)d %(thread)d %(message)s"
+        }
+    },
+    "handlers": {
+        "mail_admins": {
+            "level": "ERROR",
+            "filters": ["require_debug_false"],
+            "class": "django.utils.log.AdminEmailHandler",
+        },
+        "console": {
+            "level": "DEBUG",
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
+        },
+    },
+    "root": {"level": "INFO", "handlers": ["console"]},
+    "loggers": {
+        "django.request": {
+            "handlers": ["mail_admins"],
+            "level": "ERROR",
+            "propagate": True,
+        },
+        "django.security.DisallowedHost": {
+            "level": "ERROR",
+            "handlers": ["console", "mail_admins"],
+            "propagate": True,
+        },
+    },
+}

app/manage.py

@@ -6,7 +6,7 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings.local")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

compose/production/django/Dockerfile

@@ -0,0 +1,88 @@
+ARG PYTHON_VERSION=3.9-slim-bullseye
+
+
+
+# define an alias for the specfic python version used in this file.
+FROM python:${PYTHON_VERSION} as python
+
+# Python build stage
+FROM python as python-build-stage
+
+ARG BUILD_ENVIRONMENT=production
+
+# Install apt packages
+RUN apt-get update && apt-get install --no-install-recommends -y \
+  # dependencies for building Python packages
+  build-essential \
+  # psycopg2 dependencies
+  libpq-dev
+
+# Requirements are installed here to ensure they will be cached.
+COPY ./requirements .
+
+# Create Python Dependency and Sub-Dependency Wheels.
+RUN pip wheel --wheel-dir /usr/src/app/wheels  \
+  -r ${BUILD_ENVIRONMENT}.txt
+
+
+# Python 'run' stage
+FROM python as python-run-stage
+
+ARG BUILD_ENVIRONMENT=production
+ARG APP_HOME=/app
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV BUILD_ENV ${BUILD_ENVIRONMENT}
+
+WORKDIR ${APP_HOME}
+
+RUN addgroup --system django \
+    && adduser --system --ingroup django django
+
+
+# Install required system dependencies
+RUN apt-get update && apt-get install --no-install-recommends -y \
+  # psycopg2 dependencies
+  libpq-dev \
+  # Translations dependencies
+  gettext \
+  # cleaning up unused files
+  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+  && rm -rf /var/lib/apt/lists/*
+
+# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
+# copy python dependency wheels from python-build-stage
+COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
+
+# use wheels to install python dependencies
+RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
+  && rm -rf /wheels/
+
+
+COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint
+RUN sed -i 's/\r$//g' /entrypoint
+RUN chmod +x /entrypoint
+
+
+COPY --chown=django:django ./compose/production/django/start /start
+RUN sed -i 's/\r$//g' /start
+RUN chmod +x /start
+COPY --chown=django:django ./compose/production/django/celery/worker/start /start-celeryworker
+RUN sed -i 's/\r$//g' /start-celeryworker
+RUN chmod +x /start-celeryworker
+
+
+COPY --chown=django:django ./compose/production/django/celery/beat/start /start-celerybeat
+RUN sed -i 's/\r$//g' /start-celerybeat
+RUN chmod +x /start-celerybeat
+
+# copy application code to WORKDIR
+COPY --chown=django:django . ${APP_HOME}
+
+# make django owner of the WORKDIR directory as well.
+RUN chown django:django ${APP_HOME}
+
+USER django
+
+ENTRYPOINT ["/entrypoint"]

compose/production/django/celery/beat/start

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+exec celery -A conf.celery beat -l INFO

compose/production/django/celery/worker/start

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+exec celery -A conf.celery worker -l INFO

compose/production/django/entrypoint

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+
+# N.B. If only .env files supported variable expansion...
+export CELERY_BROKER_URL="${REDIS_URL}"
+
+python << END
+import sys
+import time
+
+import psycopg2
+
+suggest_unrecoverable_after = 30
+start = time.time()
+
+while True:
+    try:
+        psycopg2.connect(
+            dbname="${POSTGRES_DB}",
+            user="${POSTGRES_USER}",
+            password="${POSTGRES_PASSWORD}",
+            host="${POSTGRES_HOST}",
+            port="${POSTGRES_PORT}",
+        )
+        break
+    except psycopg2.OperationalError as error:
+        sys.stderr.write("Waiting for PostgreSQL to become available...\n")
+
+        if time.time() - start > suggest_unrecoverable_after:
+            sys.stderr.write("  This is taking longer than expected. The following exception may be indicative of an unrecoverable error: '{}'\n".format(error))
+
+    time.sleep(1)
+END
+
+>&2 echo 'PostgreSQL is available'
+
+exec "$@"

compose/production/django/start

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+python /app/manage.py collectstatic --noinput
+
+/usr/local/bin/gunicorn config.wsgi --bind 0.0.0.0:5000 --chdir=/app

compose/production/postgres/Dockerfile

@@ -0,0 +1,6 @@
+FROM postgres:14
+
+COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance
+RUN chmod +x /usr/local/bin/maintenance/*
+RUN mv /usr/local/bin/maintenance/* /usr/local/bin \
+    && rmdir /usr/local/bin/maintenance

compose/production/postgres/maintenance/_sourced/constants.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+
+BACKUP_DIR_PATH='/backups'
+BACKUP_FILE_PREFIX='backup'

compose/production/postgres/maintenance/_sourced/countdown.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+
+countdown() {
+    declare desc="A simple countdown. Source: https://superuser.com/a/611582"
+    local seconds="${1}"
+    local d=$(($(date +%s) + "${seconds}"))
+    while [ "$d" -ge `date +%s` ]; do
+        echo -ne "$(date -u --date @$(($d - `date +%s`)) +%H:%M:%S)\r";
+        sleep 0.1
+    done
+}

compose/production/postgres/maintenance/_sourced/messages.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+
+message_newline() {
+    echo
+}
+
+message_debug()
+{
+    echo -e "DEBUG: ${@}"
+}
+
+message_welcome()
+{
+    echo -e "\e[1m${@}\e[0m"
+}
+
+message_warning()
+{
+    echo -e "\e[33mWARNING\e[0m: ${@}"
+}
+
+message_error()
+{
+    echo -e "\e[31mERROR\e[0m: ${@}"
+}
+
+message_info()
+{
+    echo -e "\e[37mINFO\e[0m: ${@}"
+}
+
+message_suggestion()
+{
+    echo -e "\e[33mSUGGESTION\e[0m: ${@}"
+}
+
+message_success()
+{
+    echo -e "\e[32mSUCCESS\e[0m: ${@}"
+}

compose/production/postgres/maintenance/_sourced/yes_no.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+
+yes_no() {
+    declare desc="Prompt for confirmation. \$\"\{1\}\": confirmation message."
+    local arg1="${1}"
+
+    local response=
+    read -r -p "${arg1} (y/[n])? " response
+    if [[ "${response}" =~ ^[Yy]$ ]]
+    then
+        exit 0
+    else
+        exit 1
+    fi
+}

compose/production/postgres/maintenance/backup

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+
+### Create a database backup.
+###
+### Usage:
+###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres backup
+
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+working_dir="$(dirname ${0})"
+source "${working_dir}/_sourced/constants.sh"
+source "${working_dir}/_sourced/messages.sh"
+
+
+message_welcome "Backing up the '${POSTGRES_DB}' database..."
+
+
+if [[ "${POSTGRES_USER}" == "postgres" ]]; then
+    message_error "Backing up as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again."
+    exit 1
+fi
+
+export PGHOST="${POSTGRES_HOST}"
+export PGPORT="${POSTGRES_PORT}"
+export PGUSER="${POSTGRES_USER}"
+export PGPASSWORD="${POSTGRES_PASSWORD}"
+export PGDATABASE="${POSTGRES_DB}"
+
+backup_filename="${BACKUP_FILE_PREFIX}_$(date +'%Y_%m_%dT%H_%M_%S').sql.gz"
+pg_dump | gzip > "${BACKUP_DIR_PATH}/${backup_filename}"
+
+
+message_success "'${POSTGRES_DB}' database backup '${backup_filename}' has been created and placed in '${BACKUP_DIR_PATH}'."

compose/production/postgres/maintenance/backups

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+
+### View backups.
+###
+### Usage:
+###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres backups
+
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+working_dir="$(dirname ${0})"
+source "${working_dir}/_sourced/constants.sh"
+source "${working_dir}/_sourced/messages.sh"
+
+
+message_welcome "These are the backups you have got:"
+
+ls -lht "${BACKUP_DIR_PATH}"

compose/production/postgres/maintenance/restore

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+
+### Restore database from a backup.
+###
+### Parameters:
+###     <1> filename of an existing backup.
+###
+### Usage:
+###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres restore <1>
+
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+working_dir="$(dirname ${0})"
+source "${working_dir}/_sourced/constants.sh"
+source "${working_dir}/_sourced/messages.sh"
+
+
+if [[ -z ${1+x} ]]; then
+    message_error "Backup filename is not specified yet it is a required parameter. Make sure you provide one and try again."
+    exit 1
+fi
+backup_filename="${BACKUP_DIR_PATH}/${1}"
+if [[ ! -f "${backup_filename}" ]]; then
+    message_error "No backup with the specified filename found. Check out the 'backups' maintenance script output to see if there is one and try again."
+    exit 1
+fi
+
+message_welcome "Restoring the '${POSTGRES_DB}' database from the '${backup_filename}' backup..."
+
+if [[ "${POSTGRES_USER}" == "postgres" ]]; then
+    message_error "Restoring as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again."
+    exit 1
+fi
+
+export PGHOST="${POSTGRES_HOST}"
+export PGPORT="${POSTGRES_PORT}"
+export PGUSER="${POSTGRES_USER}"
+export PGPASSWORD="${POSTGRES_PASSWORD}"
+export PGDATABASE="${POSTGRES_DB}"
+
+message_info "Dropping the database..."
+dropdb "${PGDATABASE}"
+
+message_info "Creating a new database..."
+createdb --owner="${POSTGRES_USER}"
+
+message_info "Applying the backup to the new database..."
+gunzip -c "${backup_filename}" | psql "${POSTGRES_DB}"
+
+message_success "The '${POSTGRES_DB}' database has been restored from the '${backup_filename}' backup."

docker-compose.yml

@@ -1,25 +0,0 @@
-version: "3.9"
-
-services:
-  db:
-    image: postgres
-    volumes:
-      - ./data/db:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_DB=moretech
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=debug
-  web:
-    build: .
-    command: python manage.py runserver 0.0.0.0:8000
-    volumes:
-      - .:/code
-    ports:
-      - "8000:8000"
-    environment:
-      - POSTGRES_NAME=postgres
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-    depends_on:
-      - db
-

production.yml

@@ -0,0 +1,43 @@
+version: '3'
+
+volumes:
+  production_postgres_data: {}
+  production_postgres_data_backups: {}
+
+services:
+  django: &django
+    build:
+      context: .
+      dockerfile: ./compose/production/django/Dockerfile
+    image: backend_production_django
+    platform: linux/x86_64
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - ./.env
+    command: /start
+
+  postgres:
+    build:
+      context: .
+      dockerfile: ./compose/production/postgres/Dockerfile
+    image: backend_production_postgres
+    volumes:
+      - production_postgres_data:/var/lib/postgresql/data:Z
+      - production_postgres_data_backups:/backups:z
+    env_file:
+      - ./.env
+  redis:
+    image: redis:6
+
+  celeryworker:
+    <<: *django
+    image: backend_production_celeryworker
+    command: /start-celeryworker
+
+  celerybeat:
+    <<: *django
+    image: backend_production_celerybeat
+    command: /start-celerybeat
+

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = --ds=config.settings.test --reuse-db
+python_files = tests.py test_*.py

requirements/prod.txt

@@ -1,3 +0,0 @@
--r base.txt
-
-gunicorn

requirements/production.txt

@@ -0,0 +1,4 @@
+-r base.txt
+
+gunicorn
+whitenoise==6.2.0

setup.cfg

@@ -0,0 +1,40 @@
+[flake8]
+max-line-length = 120
+exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,venv
+
+[pycodestyle]
+max-line-length = 120
+exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,venv
+
+[isort]
+line_length = 88
+known_first_party = backend_infr,config
+multi_line_output = 3
+default_section = THIRDPARTY
+skip = venv/
+skip_glob = **/migrations/*.py
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+
+[mypy]
+python_version = 3.9
+check_untyped_defs = True
+ignore_missing_imports = True
+warn_unused_ignores = True
+warn_redundant_casts = True
+warn_unused_configs = True
+plugins = mypy_django_plugin.main, mypy_drf_plugin.main
+
+[mypy.plugins.django-stubs]
+django_settings_module = config.settings.test
+
+[mypy-*.migrations.*]
+# Django migrations should not produce any errors:
+ignore_errors = True
+
+[coverage:run]
+include = backend_infr/*
+omit = *migrations*, *tests*
+plugins =
+    django_coverage_plugin