diff --git a/README.rst b/README.rst index 0eaea519..a69712d0 100644 --- a/README.rst +++ b/README.rst @@ -65,7 +65,7 @@ Optional Integrations *These features can be enabled during initial project setup.* * Serve static files from Amazon S3 or Whitenoise_ -* Configuration for Celery_ +* Configuration for Celery_ and Flower_ (the latter in Docker setup only) * Integration with MailHog_ for local email testing * Integration with Sentry_ for error logging @@ -78,6 +78,7 @@ Optional Integrations .. _Mailgun: http://www.mailgun.com/ .. _Whitenoise: https://whitenoise.readthedocs.io/ .. _Celery: http://www.celeryproject.org/ +.. _Flower: https://github.com/mher/flower .. _Anymail: https://github.com/anymail/django-anymail .. _MailHog: https://github.com/mailhog/MailHog .. _Sentry: https://sentry.io/welcome/ diff --git a/docs/deployment-with-docker.rst b/docs/deployment-with-docker.rst index 42798aae..0115e3a4 100644 --- a/docs/deployment-with-docker.rst +++ b/docs/deployment-with-docker.rst @@ -21,10 +21,13 @@ Before you begin, check out the ``production.yml`` file in the root of this proj * ``redis``: Redis instance for caching; * ``caddy``: Caddy web server with HTTPS on by default. -Provided you have opted for Celery (via setting ``use_celery`` to ``y``) there are two more services: +Provided you have opted for Celery (via setting ``use_celery`` to ``y``) there are three more services: * ``celeryworker`` running a Celery worker process; -* ``celerybeat`` running a Celery beat process. +* ``celerybeat`` running a Celery beat process; +* ``flower`` running Flower_ (for more info, check out :ref:`CeleryFlower` instructions for local environment). + +.. _`Flower`: https://github.com/mher/flower Configuring the Stack diff --git a/docs/developing-locally-docker.rst b/docs/developing-locally-docker.rst index 207f0ea2..c6af1d96 100644 --- a/docs/developing-locally-docker.rst +++ b/docs/developing-locally-docker.rst @@ -170,3 +170,20 @@ When developing locally you can go with MailHog_ for email testing provided ``us #. open up ``http://127.0.0.1:8025``. .. _Mailhog: https://github.com/mailhog/MailHog/ + + +.. _`CeleryFlower`: + +Celery Flower +~~~~~~~~~~~~~ + +`Flower`_ is a "real-time monitor and web admin for Celery distributed task queue". + +Prerequisites: + +* ``use_docker`` was set to ``y`` on project initialization; +* ``use_celery`` was set to ``y`` on project initialization. + +By default, it's enabled both in local and production environments (``local.yml`` and ``production.yml`` Docker Compose configs, respectively) through a ``flower`` service. For added security, ``flower`` requires its clients to provide authentication credentials specified as the corresponding environments' ``.envs/.local/.django`` and ``.envs/.production/.django`` ``CELERY_FLOWER_USER`` and ``CELERY_FLOWER_PASSWORD`` environment variables. Check out ``localhost:5555`` and see for yourself. + +.. _`Flower`: https://github.com/mher/flower diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 6b48471e..9118f6c9 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -162,8 +162,12 @@ def set_django_admin_url(file_path): return django_admin_url +def generate_random_user(): + return generate_random_string(length=32, using_ascii_letters=True) + + def generate_postgres_user(debug=False): - return DEBUG_VALUE if debug else generate_random_string(length=32, using_ascii_letters=True) + return DEBUG_VALUE if debug else generate_random_user() def set_postgres_user(file_path, value): @@ -187,25 +191,56 @@ def set_postgres_password(file_path, value=None): return postgres_password +def set_celery_flower_user(file_path, value): + celery_flower_user = set_flag( + file_path, + "!!!SET CELERY_FLOWER_USER!!!", + value=value, + ) + return celery_flower_user + + +def set_celery_flower_password(file_path, value=None): + celery_flower_password = set_flag( + file_path, + "!!!SET CELERY_FLOWER_PASSWORD!!!", + value=value, + length=64, + using_digits=True, + using_ascii_letters=True, + ) + return celery_flower_password + + def append_to_gitignore_file(s): with open(".gitignore", "a") as gitignore_file: gitignore_file.write(s) gitignore_file.write(os.linesep) -def set_flags_in_envs(postgres_user, debug=False): - local_postgres_envs_path = os.path.join(".envs", ".local", ".postgres") - set_postgres_user(local_postgres_envs_path, value=postgres_user) - set_postgres_password(local_postgres_envs_path, value=DEBUG_VALUE if debug else None) - +def set_flags_in_envs( + postgres_user, + celery_flower_user, + debug=False, +): + local_django_envs_path = os.path.join(".envs", ".local", ".django") production_django_envs_path = os.path.join(".envs", ".production", ".django") + local_postgres_envs_path = os.path.join(".envs", ".local", ".postgres") + production_postgres_envs_path = os.path.join(".envs", ".production", ".postgres") + set_django_secret_key(production_django_envs_path) set_django_admin_url(production_django_envs_path) - production_postgres_envs_path = os.path.join(".envs", ".production", ".postgres") + set_postgres_user(local_postgres_envs_path, value=postgres_user) + set_postgres_password(local_postgres_envs_path, value=DEBUG_VALUE if debug else None) set_postgres_user(production_postgres_envs_path, value=postgres_user) set_postgres_password(production_postgres_envs_path, value=DEBUG_VALUE if debug else None) + set_celery_flower_user(local_django_envs_path, value=celery_flower_user) + set_celery_flower_password(local_django_envs_path, value=DEBUG_VALUE if debug else None) + set_celery_flower_user(production_django_envs_path, value=celery_flower_user) + set_celery_flower_password(production_django_envs_path, value=DEBUG_VALUE if debug else None) + def set_flags_in_settings_files(): set_django_secret_key(os.path.join("config", "settings", "local.py")) @@ -223,8 +258,13 @@ def remove_celery_compose_dirs(): def main(): - postgres_user = generate_postgres_user(debug="{{ cookiecutter.debug }}".lower() == "y") - set_flags_in_envs(postgres_user, debug="{{ cookiecutter.debug }}".lower() == "y") + debug = "{{ cookiecutter.debug }}".lower() == "y" + + set_flags_in_envs( + DEBUG_VALUE if debug else generate_random_user(), + DEBUG_VALUE if debug else generate_random_user(), + debug=debug, + ) set_flags_in_settings_files() if "{{ cookiecutter.open_source_license }}" == "Not open source": diff --git a/{{cookiecutter.project_slug}}/.envs/.local/.django b/{{cookiecutter.project_slug}}/.envs/.local/.django index 8aa9a994..d94a17e5 100644 --- a/{{cookiecutter.project_slug}}/.envs/.local/.django +++ b/{{cookiecutter.project_slug}}/.envs/.local/.django @@ -5,3 +5,11 @@ USE_DOCKER=yes # Redis # ------------------------------------------------------------------------------ REDIS_URL=redis://redis:6379/0 +{% if cookiecutter.use_celery == 'y' %} +# Celery +# ------------------------------------------------------------------------------ + +# Flower +CELERY_FLOWER_USER=!!!SET CELERY_FLOWER_USER!!! +CELERY_FLOWER_PASSWORD=!!!SET CELERY_FLOWER_PASSWORD!!! +{% endif %} diff --git a/{{cookiecutter.project_slug}}/.envs/.production/.django b/{{cookiecutter.project_slug}}/.envs/.production/.django index 5cb90897..4175f894 100644 --- a/{{cookiecutter.project_slug}}/.envs/.production/.django +++ b/{{cookiecutter.project_slug}}/.envs/.production/.django @@ -43,3 +43,11 @@ SENTRY_DSN= # Redis # ------------------------------------------------------------------------------ REDIS_URL=redis://redis:6379/0 +{% if cookiecutter.use_celery == 'y' %} +# Celery +# ------------------------------------------------------------------------------ + +# Flower +CELERY_FLOWER_USER=!!!SET CELERY_FLOWER_USER!!! +CELERY_FLOWER_PASSWORD=!!!SET CELERY_FLOWER_PASSWORD!!! +{% endif %} diff --git a/{{cookiecutter.project_slug}}/compose/local/django/Dockerfile b/{{cookiecutter.project_slug}}/compose/local/django/Dockerfile index 27424954..e2e9e5f3 100644 --- a/{{cookiecutter.project_slug}}/compose/local/django/Dockerfile +++ b/{{cookiecutter.project_slug}}/compose/local/django/Dockerfile @@ -34,6 +34,10 @@ RUN chmod +x /start-celeryworker COPY ./compose/local/django/celery/beat/start /start-celerybeat RUN sed -i 's/\r//' /start-celerybeat RUN chmod +x /start-celerybeat + +COPY ./compose/local/django/celery/flower/start /start-flower +RUN sed -i 's/\r//' /start-flower +RUN chmod +x /start-flower {% endif %} WORKDIR /app diff --git a/{{cookiecutter.project_slug}}/compose/local/django/celery/flower/start b/{{cookiecutter.project_slug}}/compose/local/django/celery/flower/start new file mode 100644 index 00000000..f0abae7e --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/local/django/celery/flower/start @@ -0,0 +1,10 @@ +#!/bin/sh + +set -o errexit +set -o nounset + + +celery flower \ + --app={{cookiecutter.project_slug}}.taskapp \ + --broker="${CELERY_BROKER_URL}" \ + --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" diff --git a/{{cookiecutter.project_slug}}/compose/production/django/Dockerfile b/{{cookiecutter.project_slug}}/compose/production/django/Dockerfile index b204a481..68d72327 100644 --- a/{{cookiecutter.project_slug}}/compose/production/django/Dockerfile +++ b/{{cookiecutter.project_slug}}/compose/production/django/Dockerfile @@ -38,6 +38,10 @@ COPY ./compose/production/django/celery/beat/start /start-celerybeat RUN sed -i 's/\r//' /start-celerybeat RUN chmod +x /start-celerybeat RUN chown django /start-celerybeat + +COPY ./compose/production/django/celery/flower/start /start-flower +RUN sed -i 's/\r//' /start-flower +RUN chmod +x /start-flower {% endif %} COPY . /app diff --git a/{{cookiecutter.project_slug}}/compose/production/django/celery/flower/start b/{{cookiecutter.project_slug}}/compose/production/django/celery/flower/start new file mode 100644 index 00000000..f0abae7e --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/production/django/celery/flower/start @@ -0,0 +1,10 @@ +#!/bin/sh + +set -o errexit +set -o nounset + + +celery flower \ + --app={{cookiecutter.project_slug}}.taskapp \ + --broker="${CELERY_BROKER_URL}" \ + --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" diff --git a/{{cookiecutter.project_slug}}/local.yml b/{{cookiecutter.project_slug}}/local.yml index 5ad26dfd..ac13d185 100644 --- a/{{cookiecutter.project_slug}}/local.yml +++ b/{{cookiecutter.project_slug}}/local.yml @@ -71,4 +71,11 @@ services: ports: [] command: /start-celerybeat + flower: + <<: *django + image: {{ cookiecutter.project_slug }}_local_flower + ports: + - "5555:5555" + command: /start-flower + {%- endif %} diff --git a/{{cookiecutter.project_slug}}/production.yml b/{{cookiecutter.project_slug}}/production.yml index 4ee178d8..9415db61 100644 --- a/{{cookiecutter.project_slug}}/production.yml +++ b/{{cookiecutter.project_slug}}/production.yml @@ -59,4 +59,11 @@ services: image: {{ cookiecutter.project_slug }}_production_celerybeat command: /start-celerybeat + flower: + <<: *django + image: {{ cookiecutter.project_slug }}_production_flower + ports: + - "5555:5555" + command: /start-flower + {%- endif %} diff --git a/{{cookiecutter.project_slug}}/requirements/base.txt b/{{cookiecutter.project_slug}}/requirements/base.txt index b553bb83..5dd5d99e 100644 --- a/{{cookiecutter.project_slug}}/requirements/base.txt +++ b/{{cookiecutter.project_slug}}/requirements/base.txt @@ -11,6 +11,9 @@ whitenoise==3.3.1 # https://github.com/evansd/whitenoise redis>=2.10.5 # https://github.com/antirez/redis {%- if cookiecutter.use_celery == "y" %} celery==4.2.0 # pyup: <5.0 # https://github.com/celery/celery +{%- if cookiecutter.use_docker == 'y' %} +flower==0.9.2 # https://github.com/mher/flower +{%- endif %} {%- endif %} # Django