Test all possible template combinations (#1993)

## Description

Fixes #591.

## Rationale

We are currently not testing many combinations, we run Flake8 on the generated project with default options, but that rarely catch any issues.

## Use case(s) / visualization(s)

Catch problems with invalid combinations, for instance, it fails due to Whitenoise breaking flake8 with `django-compressor` because `STATIC_URL` was undefined here:

b91c70d755/%7B%7Bcookiecutter.project_slug%7D%7D/config/settings/production.py (L185)

requirements.txt

@@ -4,11 +4,13 @@ binaryornot==0.4.4
 
 # Code quality
 # ------------------------------------------------------------------------------
+black==19.3b0
 flake8==3.7.6
 
 # Testing
 # ------------------------------------------------------------------------------
 tox==3.8.3
 pytest==4.3.1
+pytest_cases==1.5.1
 pytest-cookies==0.3.0
 pyyaml==5.1

tests/test_cookiecutter_generation.py

@@ -1,14 +1,17 @@
 import os
 import re
-import sh
-import yaml
 
 import pytest
+from pytest_cases import pytest_fixture_plus
+import sh
+import yaml
 from binaryornot.check import is_binary
 
 PATTERN = "{{(\s?cookiecutter)[.](.*?)}}"
 RE_OBJ = re.compile(PATTERN)
 
+YN_CHOICES = ["y", "n"]
+
 
 @pytest.fixture
 def context():
@@ -24,6 +27,35 @@ def context():
     }
 
 
+@pytest_fixture_plus
+@pytest.mark.parametrize("windows", YN_CHOICES, ids=lambda yn: f"win:{yn}")
+@pytest.mark.parametrize("use_docker", YN_CHOICES, ids=lambda yn: f"docker:{yn}")
+@pytest.mark.parametrize("use_celery", YN_CHOICES, ids=lambda yn: f"celery:{yn}")
+@pytest.mark.parametrize("use_mailhog", YN_CHOICES, ids=lambda yn: f"mailhog:{yn}")
+@pytest.mark.parametrize("use_sentry", YN_CHOICES, ids=lambda yn: f"sentry:{yn}")
+@pytest.mark.parametrize("use_compressor", YN_CHOICES, ids=lambda yn: f"cmpr:{yn}")
+@pytest.mark.parametrize("use_whitenoise", YN_CHOICES, ids=lambda yn: f"wnoise:{yn}")
+def context_combination(
+    windows,
+    use_docker,
+    use_celery,
+    use_mailhog,
+    use_sentry,
+    use_compressor,
+    use_whitenoise,
+):
+    """Fixture that parametrize the function where it's used."""
+    return {
+        "windows": windows,
+        "use_docker": use_docker,
+        "use_compressor": use_compressor,
+        "use_celery": use_celery,
+        "use_mailhog": use_mailhog,
+        "use_sentry": use_sentry,
+        "use_whitenoise": use_whitenoise,
+    }
+
+
 def build_files_list(root_dir):
     """Build a list containing absolute paths to the generated files."""
     return [
@@ -48,8 +80,13 @@ def check_paths(paths):
             assert match is None, msg.format(path)
 
 
-def test_default_configuration(cookies, context):
-    result = cookies.bake(extra_context=context)
+def test_project_generation(cookies, context, context_combination):
+    """
+    Test that project is generated and fully rendered.
+
+    This is parametrized for each combination from ``context_combination`` fixture
+    """
+    result = cookies.bake(extra_context={**context, **context_combination})
     assert result.exit_code == 0
     assert result.exception is None
     assert result.project.basename == context["project_slug"]
@@ -60,33 +97,24 @@ def test_default_configuration(cookies, context):
     check_paths(paths)
 
 
-@pytest.fixture(params=["use_mailhog", "use_celery", "windows"])
-def feature_context(request, context):
-    context.update({request.param: "y"})
-    return context
+def test_linting_passes(cookies, context_combination):
+    """
+    Generated project should pass flake8 & black.
 
-
-def test_enabled_features(cookies, feature_context):
-    result = cookies.bake(extra_context=feature_context)
-    assert result.exit_code == 0
-    assert result.exception is None
-    assert result.project.basename == feature_context["project_slug"]
-    assert result.project.isdir()
-
-    paths = build_files_list(str(result.project))
-    assert paths
-    check_paths(paths)
-
-
-def test_flake8_compliance(cookies):
-    """generated project should pass flake8"""
-    result = cookies.bake()
+    This is parametrized for each combination from ``context_combination`` fixture
+    """
+    result = cookies.bake(extra_context=context_combination)
 
     try:
         sh.flake8(str(result.project))
     except sh.ErrorReturnCode as e:
         pytest.fail(e)
 
+    try:
+        sh.black("--check", "--diff", "--exclude", "migrations", f"{result.project}/")
+    except sh.ErrorReturnCode as e:
+        pytest.fail(e)
+
 
 def test_travis_invokes_pytest(cookies, context):
     context.update({"use_travisci": "y"})

{{cookiecutter.project_slug}}/config/settings/base.py

@@ -43,9 +43,7 @@ USE_TZ = True
 DATABASES = {"default": env.db("DATABASE_URL")}
 {%- else %}
 DATABASES = {
-    "default": env.db(
-        "DATABASE_URL", default="postgres://{% if cookiecutter.windows == 'y' %}localhost{% endif %}/{{cookiecutter.project_slug}}"
-    ),
+    "default": env.db("DATABASE_URL", default="postgres://{% if cookiecutter.windows == 'y' %}localhost{% endif %}/{{cookiecutter.project_slug}}")
 }
 {%- endif %}
 DATABASES["default"]["ATOMIC_REQUESTS"] = True

{{cookiecutter.project_slug}}/config/settings/production.py

@@ -182,7 +182,7 @@ COMPRESS_ENABLED = env.bool("COMPRESS_ENABLED", default=True)
 # https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_STORAGE
 COMPRESS_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 # https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL
-COMPRESS_URL = STATIC_URL
+COMPRESS_URL = STATIC_URL{% if cookiecutter.use_whitenoise == 'y' %}  # noqa F405{% endif %}
 {% endif %}
 {%- if cookiecutter.use_whitenoise == 'n' -%}
 # Collectfast

{{cookiecutter.project_slug}}/requirements/base.txt

@@ -2,7 +2,7 @@ pytz==2018.9  # https://github.com/stub42/pytz
 python-slugify==3.0.1  # https://github.com/un33k/python-slugify
 Pillow==5.4.1  # https://github.com/python-pillow/Pillow
 {%- if cookiecutter.use_compressor == "y" %}
-rcssmin==1.0.6{% if cookiecutter.windows == 'y' and use_docker == 'n' %} --install-option="--without-c-extensions"{% endif %}  # https://github.com/ndparker/rcssmin
+rcssmin==1.0.6{% if cookiecutter.windows == 'y' and cookiecutter.use_docker == 'n' %} --install-option="--without-c-extensions"{% endif %}  # https://github.com/ndparker/rcssmin
 {%- endif %}
 argon2-cffi==19.1.0  # https://github.com/hynek/argon2_cffi
 {%- if cookiecutter.use_whitenoise == 'y' %}