Refactor Celery integration according to current best practices (#2016)

Refactor Celery integration according to current best practices

hooks/post_gen_project.py

@@ -89,8 +89,16 @@ def remove_packagejson_file():
         os.remove(file_name)
 
 
-def remove_celery_app():
-    shutil.rmtree(os.path.join("{{ cookiecutter.project_slug }}", "taskapp"))
+def remove_celery_files():
+    file_names = [
+        os.path.join("config", "celery_app.py"),
+        os.path.join("{{ cookiecutter.project_slug }}", "users", "tasks.py"),
+        os.path.join(
+            "{{ cookiecutter.project_slug }}", "users", "tests", "test_tasks.py"
+        ),
+    ]
+    for file_name in file_names:
+        os.remove(file_name)
 
 
 def remove_dottravisyml_file():
@@ -321,7 +329,7 @@ def main():
             remove_node_dockerfile()
 
     if "{{ cookiecutter.use_celery }}".lower() == "n":
-        remove_celery_app()
+        remove_celery_files()
         if "{{ cookiecutter.use_docker }}".lower() == "y":
             remove_celery_compose_dirs()
 

{{cookiecutter.project_slug}}/Procfile

@@ -1,5 +1,5 @@
 release: python manage.py migrate
 web: gunicorn config.wsgi:application
 {% if cookiecutter.use_celery == "y" -%}
-worker: celery worker --app={{cookiecutter.project_slug}}.taskapp --loglevel=info
+worker: celery worker --app=config.celery_app --loglevel=info
 {%- endif %}

{{cookiecutter.project_slug}}/README.rst

@@ -79,7 +79,7 @@ To run a celery worker:
 .. code-block:: bash
 
     cd {{cookiecutter.project_slug}}
-    celery -A {{cookiecutter.project_slug}}.taskapp worker -l info
+    celery -A config.celery_app worker -l info
 
 Please note: For Celery's import magic to work, it is important *where* the celery commands are run. If you are in the same folder with *manage.py*, you should be right.
 

{{cookiecutter.project_slug}}/compose/local/django/celery/beat/start

@@ -5,4 +5,4 @@ set -o nounset
 
 
 rm -f './celerybeat.pid'
-celery -A {{cookiecutter.project_slug}}.taskapp beat -l INFO
+celery -A config.celery_app beat -l INFO

{{cookiecutter.project_slug}}/compose/local/django/celery/flower/start

@@ -5,6 +5,6 @@ set -o nounset
 
 
 celery flower \
-    --app={{cookiecutter.project_slug}}.taskapp \
+    --app=config.celery_app \
     --broker="${CELERY_BROKER_URL}" \
     --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}"

{{cookiecutter.project_slug}}/compose/local/django/celery/worker/start

@@ -4,4 +4,4 @@ set -o errexit
 set -o nounset
 
 
-celery -A {{cookiecutter.project_slug}}.taskapp worker -l INFO
+celery -A config.celery_app worker -l INFO

{{cookiecutter.project_slug}}/compose/production/django/celery/beat/start

@@ -5,4 +5,4 @@ set -o pipefail
 set -o nounset
 
 
-celery -A {{cookiecutter.project_slug}}.taskapp beat -l INFO
+celery -A config.celery_app beat -l INFO

{{cookiecutter.project_slug}}/compose/production/django/celery/flower/start

@@ -5,6 +5,6 @@ set -o nounset
 
 
 celery flower \
-    --app={{cookiecutter.project_slug}}.taskapp \
+    --app=config.celery_app \
     --broker="${CELERY_BROKER_URL}" \
     --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}"

{{cookiecutter.project_slug}}/compose/production/django/celery/worker/start

@@ -5,4 +5,4 @@ set -o pipefail
 set -o nounset
 
 
-celery -A {{cookiecutter.project_slug}}.taskapp worker -l INFO
+celery -A config.celery_app worker -l INFO

{{cookiecutter.project_slug}}/config/__init__.py

@@ -0,0 +1,7 @@
+{% if cookiecutter.use_celery == 'y' -%}
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery_app import app as celery_app
+
+__all__ = ("celery_app",)
+{% endif -%}

{{cookiecutter.project_slug}}/config/celery_app.py

@@ -0,0 +1,16 @@
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
+
+app = Celery("{{cookiecutter.project_slug}}")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

{{cookiecutter.project_slug}}/config/settings/base.py

@@ -251,7 +251,6 @@ LOGGING = {
 {% if cookiecutter.use_celery == 'y' -%}
 # Celery
 # ------------------------------------------------------------------------------
-INSTALLED_APPS += ["{{cookiecutter.project_slug}}.taskapp.celery.CeleryAppConfig"]
 if USE_TZ:
     # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-timezone
     CELERY_TIMEZONE = TIME_ZONE

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/taskapp/celery.py

@@ -1,38 +0,0 @@
-{% if cookiecutter.use_celery == 'y' -%}
-import os
-from celery import Celery
-from django.apps import apps, AppConfig
-from django.conf import settings
-
-
-if not settings.configured:
-    # set the default Django settings module for the 'celery' program.
-    os.environ.setdefault(
-        "DJANGO_SETTINGS_MODULE", "config.settings.local"
-    )  # pragma: no cover
-
-
-app = Celery("{{cookiecutter.project_slug}}")
-# Using a string here means the worker will not have to
-# pickle the object when using Windows.
-# - namespace='CELERY' means all celery-related configuration keys
-#   should have a `CELERY_` prefix.
-app.config_from_object("django.conf:settings", namespace="CELERY")
-
-
-class CeleryAppConfig(AppConfig):
-    name = "{{cookiecutter.project_slug}}.taskapp"
-    verbose_name = "Celery Config"
-
-    def ready(self):
-        installed_apps = [app_config.name for app_config in apps.get_app_configs()]
-        app.autodiscover_tasks(lambda: installed_apps, force=True)
-
-
-@app.task(bind=True)
-def debug_task(self):
-    print(f"Request: {self.request!r}")  # pragma: no cover
-{% else %}
-# Use this as a starting point for your project with celery.
-# If you are not using celery, you can remove this app
-{% endif -%}

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tasks.py

@@ -0,0 +1,11 @@
+from django.contrib.auth import get_user_model
+
+from config import celery_app
+
+User = get_user_model()
+
+
+@celery_app.task()
+def get_users_count():
+    """A pointless Celery task to demonstrate usage."""
+    return User.objects.count()

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/test_tasks.py

@@ -0,0 +1,16 @@
+import pytest
+from celery.result import EagerResult
+
+
+from {{ cookiecutter.project_slug }}.users.tasks import get_users_count
+from {{ cookiecutter.project_slug }}.users.tests.factories import UserFactory
+
+
+@pytest.mark.django_db
+def test_user_count(settings):
+    """A basic test to execute the get_users_count Celery task."""
+    UserFactory.create_batch(3)
+    settings.CELERY_TASK_ALWAYS_EAGER = True
+    task_result = get_users_count.delay()
+    assert isinstance(task_result, EagerResult)
+    assert task_result.result == 3