From 8a9eecea0e9014028848363ef503bb5b054d1be9 Mon Sep 17 00:00:00 2001 From: Gregg Sangster Date: Sat, 18 Jun 2016 00:14:05 -0400 Subject: [PATCH 1/3] Add options for testing and continuous deployment with GitLab CI. --- cookiecutter.json | 4 + docs/ci-gitlab.rst | 80 +++++++++++++++++++ docs/project-generation-options.rst | 14 ++++ hooks/post_gen_project.py | 34 +++++++- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 39 +++++++++ .../compose/django/Dockerfile-test | 22 +++++ {{cookiecutter.project_slug}}/test.yml | 36 +++++++++ 7 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 docs/ci-gitlab.rst create mode 100644 {{cookiecutter.project_slug}}/.gitlab-ci.yml create mode 100644 {{cookiecutter.project_slug}}/compose/django/Dockerfile-test create mode 100644 {{cookiecutter.project_slug}}/test.yml diff --git a/cookiecutter.json b/cookiecutter.json index d3017023e..e91b3c1f5 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -20,5 +20,9 @@ "use_compressor": "n", "js_task_runner": ["Gulp", "Grunt", "None"], "use_lets_encrypt": "n", + "use_gitlab_ci": "n", + "staging_branch": "", + "staging_domain_name": "beta.{{ cookiecutter.domain_name }}", + "production_branch": "", "open_source_license": ["MIT", "BSD", "Apache Software License 2.0", "Not open source"] } diff --git a/docs/ci-gitlab.rst b/docs/ci-gitlab.rst new file mode 100644 index 000000000..c3bc45f8a --- /dev/null +++ b/docs/ci-gitlab.rst @@ -0,0 +1,80 @@ +Testing and Contiinuous Deployment with GitLab CI +================================================= + +Prerequisites +------------- + +* Docker and Docker Compose (minimum versions in Deployment with Docker) +* Docker Machine for continuous deployment (tested with 0.5.5) + +Configuring GitLab CI to Run the Tests +-------------------------------------- + +1) Set up an Ubuntu machine with Docker. Digital Ocean has a droplet configuration +that makes this very convenient but any machine will do. + +2) Install docker-compose according to the docs: + +:: + + curl -L https://github.com/docker/compose/releases/download/1.5.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + +2) Install docker-machine according to the docs: + +:: + + curl -L https://github.com/docker/machine/releases/download/v0.5.5/docker-machine_linux-amd64 > /usr/local/bin/docker-machine + chmod +x /usr/local/bin/docker-machine + +3) Install and register gitlab-ci-multi-runner according to the docs. When registering, +use the shell executor: + +:: + + curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-ci-multi-runner/script.deb.sh | sudo bash + apt-get install gitlab-ci-multi-runner + gitlab-runner register # Use shell executor. + +4) Add the gitlab-runner user account to the docker group: + +:: + + usermod -aG docker gitlab-runner + +At ths point, GitLab CI is capable of running the tests. Continue on below to +set up continuous deployment. + +Configuring GitLab CI for Continuous Deployment +----------------------------------------------- + +1) Set up Ubuntu machines with Docker for the staging (if desired) and production +sites. Digital Ocean has a droplet configuration that makes this very convenient +but any machines will do. + +2) On the test runner, create an ssh key. Add it as an authorized key on the +staging and production machines and as a deploy key in the GitLab project: + +:: + + sudo -u gitlab-runner -H ssh-keygen -t rsa -C "gitlab-runner@DOMAIN" + +3) On the test runner, create a docker machine for the staging site: + +:: + + sudo -u gitlab-runner -H docker-machine create -d generic --generic-ip-address {{cookiecutter.staging_domain_name}} + +4) On the test runner, create a docker machine for the production site: + +:: + + sudo -u gitlab-runner -H docker-machine create -d generic --generic-ip-address {{cookiecutter.domain_name}} + +All Done +-------- + +Congratulations! You now have a GitLab CI environment to run the tests for +every commit (all branches including feature branches) and automatically deploy +staging and production environments. + diff --git a/docs/project-generation-options.rst b/docs/project-generation-options.rst index 2ea27c53b..32fefb63b 100644 --- a/docs/project-generation-options.rst +++ b/docs/project-generation-options.rst @@ -49,6 +49,20 @@ use_python2 [n] By default, the Python code generated will be for Python 3.x. But if you answer `y` here, it will be legacy Python 2.7 code. +use_gitlab_ci [n] + Whether to use GitLab CI for testing. + +staging_branch [] + If using GitLab for continuous deployment, the git branch to deploy in the + staging environment. + +staging_domain_name [beta.domain_name] + If deploying to a staging environment, the domain name of the staging site. + +production_branch [] + If using GitLab for continuous deployment, the git branch to deploy in the + production environment. + .. _WhiteNoise: https://github.com/evansd/whitenoise .. _Celery: https://github.com/celery/celery .. _MailHog: https://github.com/mailhog/MailHog diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index d83d92287..65c7ed614 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -170,6 +170,20 @@ def remove_certbot_files(): file_name = os.path.join(nginx_dir_location, filename) remove_file(file_name) +def remove_gitlab_ci_files(): + """ + Removes files needed for GitLab CI if it isn't going to be used + """ + for filename in [".gitlab-ci.yml, test.yml"]: + os.remove(os.path.join( + PROJECT_DIRECTORY, filename + )) + + django_dir_location = os.path.join(PROJECT_DIRECTORY, 'compose/django') + for filename in ["Dockerfile-test"]: + file_name = os.path.join(django_dir_location, filename) + remove_file(file_name) + # IN PROGRESS # def copy_doc_files(project_directory): # cookiecutters_dir = DEFAULT_CONFIG['cookiecutters_dir'] @@ -245,5 +259,23 @@ if '{{ cookiecutter.use_lets_encrypt }}'.lower() == 'y' and '{{ cookiecutter.use "You must generate a dhparams.pem file before running docker-compose in a production environment." ) -# 4. Copy files from /docs/ to {{ cookiecutter.project_slug }}/docs/ +# 11. Removes all GitLab CI files if it isn't going to be used +if '{{ cookiecutter.use_gitlab_ci }}'.lower() != 'y': + remove_gitlab_ci_files() + +# 12. Removes the GitLab CI files and display a warning if use_gitlab_ci is selected and use_docker isn't. +if '{{ cookiecutter.use_gitlab_ci }}'.lower() == 'y' and '{{ cookiecutter.use_docker }}'.lower() != 'y': + remove_gitlab_ci_files() + print( + "You selected to use GitLab CI and didn't select to use docker. This is NOT supported out of the box for now. You " + "can continue to use the project like you normally would, but GitLab CI files have not been included." + ) + +# 13. Directs the user to the documentation if certbot and docker are selected. +if '{{ cookiecutter.use_gitlab_ci }}'.lower() == 'y' and '{{ cookiecutter.use_docker }}'.lower() == 'y': + print( + "You selected to use GitLab CI. Please see the documentation for instructions on how to set up the GitLab CI environment" + ) + +# 14. Copy files from /docs/ to {{ cookiecutter.project_slug }}/docs/ # copy_doc_files(PROJECT_DIRECTORY) diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml new file mode 100644 index 000000000..fbbf47afb --- /dev/null +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -0,0 +1,39 @@ +stages: + - test + - deploy + +test: + script: + - docker-compose -f test.yml build + - docker-compose -f test.yml run --rm django bash -c "flake8" + - docker-compose -f test.yml run --rm django bash -c "coverage run --branch manage.py test && coverage report -m" + +{% if cookiecutter.staging_branch != '' %} +staging: + type: deploy + script: + - eval "$(docker-machine env {{cookiecutter.staging_domain_name}})" + - docker-compose build + - docker-compose run --rm django python manage.py migrate + - docker-compose up -d + - docker rm -v `docker ps -a -q -f status=exited` || true + - docker rmi `docker images -f "dangling=true" -q` || true + - docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes + only: + - {{cookiecutter.staging_branch}} +{% endif %} + +{% if cookiecutter.production_branch != '' %} +production: + type: deploy + script: + - eval "$(docker-machine env {{cookiecutter.domain_name}})" + - docker-compose build + - docker-compose run --rm django python manage.py migrate + - docker-compose up -d + - docker rm -v `docker ps -a -q -f status=exited` || true + - docker rmi `docker images -f "dangling=true" -q` || true + - docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes + only: + - {{cookiecutter.production_branch}} +{% endif %} diff --git a/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test b/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test new file mode 100644 index 000000000..2a6a189ee --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test @@ -0,0 +1,22 @@ +{% if cookiecutter.use_python2 == 'n' -%} +FROM python:3.5 +{% else %} +FROM python:2.7 +{%- endif %} +ENV PYTHONUNBUFFERED 1 + +# Requirements have to be pulled and installed here, otherwise caching won't work +COPY ./requirements /requirements +RUN pip install -r /requirements/local.txt + +RUN groupadd -r django && useradd -r -g django django +COPY . /app +RUN chown -R django /app + +COPY ./compose/django/entrypoint.sh /entrypoint.sh +RUN sed -i 's/\r//' /entrypoint.sh +RUN chmod +x /entrypoint.sh + +WORKDIR /app + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/{{cookiecutter.project_slug}}/test.yml b/{{cookiecutter.project_slug}}/test.yml new file mode 100644 index 000000000..60b7c248c --- /dev/null +++ b/{{cookiecutter.project_slug}}/test.yml @@ -0,0 +1,36 @@ +version: '2' + +volumes: + postgres_data_dev: {} + postgres_backup_dev: {} + +services: + postgres: + build: ./compose/postgres + volumes: + - postgres_data_dev:/var/lib/postgresql/data + - postgres_backup_dev:/backups + environment: + - POSTGRES_USER={{cookiecutter.project_slug}} + + django: + build: + context: . + dockerfile: ./compose/django/Dockerfile-test + command: python /app/manage.py runserver_plus 0.0.0.0:8000 + depends_on: + - postgres + ports: + - "8000:8000" + links: + - postgres +{% if cookiecutter.use_mailhog == 'y' %} + - mailhog +{% endif %} + +{% if cookiecutter.use_mailhog == 'y' %} + mailhog: + image: mailhog/mailhog + ports: + - "8025:8025" +{% endif %} From e561907c505683d5dab3ca9e278103b73b621abf Mon Sep 17 00:00:00 2001 From: Gregg Sangster Date: Sat, 18 Jun 2016 22:03:18 -0400 Subject: [PATCH 2/3] Use single question to configure GitLab CI. Switch to a single question with three options: "None", "Test Only" and "Test and Deploy". If "Test and Deploy" is chosen then the master branch is automatically deployed to {{ cookiecutter.domain_name }}. --- cookiecutter.json | 5 +---- docs/ci-gitlab.rst | 15 ++++--------- docs/project-generation-options.rst | 15 ++----------- hooks/post_gen_project.py | 8 +++---- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 22 +++++--------------- 5 files changed, 16 insertions(+), 49 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index e91b3c1f5..8dfbc09ea 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -20,9 +20,6 @@ "use_compressor": "n", "js_task_runner": ["Gulp", "Grunt", "None"], "use_lets_encrypt": "n", - "use_gitlab_ci": "n", - "staging_branch": "", - "staging_domain_name": "beta.{{ cookiecutter.domain_name }}", - "production_branch": "", + "use_gitlab_ci": ["No", "Test Only", "Test and Deploy"], "open_source_license": ["MIT", "BSD", "Apache Software License 2.0", "Not open source"] } diff --git a/docs/ci-gitlab.rst b/docs/ci-gitlab.rst index c3bc45f8a..b594834c4 100644 --- a/docs/ci-gitlab.rst +++ b/docs/ci-gitlab.rst @@ -48,23 +48,16 @@ set up continuous deployment. Configuring GitLab CI for Continuous Deployment ----------------------------------------------- -1) Set up Ubuntu machines with Docker for the staging (if desired) and production -sites. Digital Ocean has a droplet configuration that makes this very convenient -but any machines will do. +1) Set up an Ubuntu machine with Docker for the production site. Digital Ocean +has a droplet configuration that makes this very convenient but any machine will do. 2) On the test runner, create an ssh key. Add it as an authorized key on the -staging and production machines and as a deploy key in the GitLab project: +production machine and as a deploy key in the GitLab project: :: sudo -u gitlab-runner -H ssh-keygen -t rsa -C "gitlab-runner@DOMAIN" -3) On the test runner, create a docker machine for the staging site: - -:: - - sudo -u gitlab-runner -H docker-machine create -d generic --generic-ip-address {{cookiecutter.staging_domain_name}} - 4) On the test runner, create a docker machine for the production site: :: @@ -76,5 +69,5 @@ All Done Congratulations! You now have a GitLab CI environment to run the tests for every commit (all branches including feature branches) and automatically deploy -staging and production environments. +the master branch to the production site. diff --git a/docs/project-generation-options.rst b/docs/project-generation-options.rst index 32fefb63b..98994d9fe 100644 --- a/docs/project-generation-options.rst +++ b/docs/project-generation-options.rst @@ -49,19 +49,8 @@ use_python2 [n] By default, the Python code generated will be for Python 3.x. But if you answer `y` here, it will be legacy Python 2.7 code. -use_gitlab_ci [n] - Whether to use GitLab CI for testing. - -staging_branch [] - If using GitLab for continuous deployment, the git branch to deploy in the - staging environment. - -staging_domain_name [beta.domain_name] - If deploying to a staging environment, the domain name of the staging site. - -production_branch [] - If using GitLab for continuous deployment, the git branch to deploy in the - production environment. +use_gitlab_ci ["No"] + Whether to use GitLab CI for testing and continuous deployment. .. _WhiteNoise: https://github.com/evansd/whitenoise .. _Celery: https://github.com/celery/celery diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 65c7ed614..372c126ca 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -174,7 +174,7 @@ def remove_gitlab_ci_files(): """ Removes files needed for GitLab CI if it isn't going to be used """ - for filename in [".gitlab-ci.yml, test.yml"]: + for filename in [".gitlab-ci.yml", "test.yml"]: os.remove(os.path.join( PROJECT_DIRECTORY, filename )) @@ -260,11 +260,11 @@ if '{{ cookiecutter.use_lets_encrypt }}'.lower() == 'y' and '{{ cookiecutter.use ) # 11. Removes all GitLab CI files if it isn't going to be used -if '{{ cookiecutter.use_gitlab_ci }}'.lower() != 'y': +if '{{ cookiecutter.use_gitlab_ci }}'.lower() == 'no': remove_gitlab_ci_files() # 12. Removes the GitLab CI files and display a warning if use_gitlab_ci is selected and use_docker isn't. -if '{{ cookiecutter.use_gitlab_ci }}'.lower() == 'y' and '{{ cookiecutter.use_docker }}'.lower() != 'y': +if '{{ cookiecutter.use_gitlab_ci }}'.lower() != 'no' and '{{ cookiecutter.use_docker }}'.lower() != 'y': remove_gitlab_ci_files() print( "You selected to use GitLab CI and didn't select to use docker. This is NOT supported out of the box for now. You " @@ -272,7 +272,7 @@ if '{{ cookiecutter.use_gitlab_ci }}'.lower() == 'y' and '{{ cookiecutter.use_do ) # 13. Directs the user to the documentation if certbot and docker are selected. -if '{{ cookiecutter.use_gitlab_ci }}'.lower() == 'y' and '{{ cookiecutter.use_docker }}'.lower() == 'y': +if '{{ cookiecutter.use_gitlab_ci }}'.lower() != 'no' and '{{ cookiecutter.use_docker }}'.lower() == 'y': print( "You selected to use GitLab CI. Please see the documentation for instructions on how to set up the GitLab CI environment" ) diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index fbbf47afb..3064d522f 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -1,6 +1,8 @@ stages: - test +{% if cookiecutter.use_gitlab_ci == 'Test and Deploy' %} - deploy +{% endif %} test: script: @@ -8,22 +10,7 @@ test: - docker-compose -f test.yml run --rm django bash -c "flake8" - docker-compose -f test.yml run --rm django bash -c "coverage run --branch manage.py test && coverage report -m" -{% if cookiecutter.staging_branch != '' %} -staging: - type: deploy - script: - - eval "$(docker-machine env {{cookiecutter.staging_domain_name}})" - - docker-compose build - - docker-compose run --rm django python manage.py migrate - - docker-compose up -d - - docker rm -v `docker ps -a -q -f status=exited` || true - - docker rmi `docker images -f "dangling=true" -q` || true - - docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes - only: - - {{cookiecutter.staging_branch}} -{% endif %} - -{% if cookiecutter.production_branch != '' %} +{% if cookiecutter.use_gitlab_ci == 'Test and Deploy' %} production: type: deploy script: @@ -35,5 +22,6 @@ production: - docker rmi `docker images -f "dangling=true" -q` || true - docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes only: - - {{cookiecutter.production_branch}} + - master {% endif %} + From d1b5ace7f40e880bd23ee85b2691701534bd8075 Mon Sep 17 00:00:00 2001 From: Gregg Sangster Date: Mon, 12 Sep 2016 16:23:28 -0400 Subject: [PATCH 3/3] Change use_python2 to use_python3 in test Dockerfile. --- {{cookiecutter.project_slug}}/compose/django/Dockerfile-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test b/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test index 2a6a189ee..0cfe73812 100644 --- a/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test +++ b/{{cookiecutter.project_slug}}/compose/django/Dockerfile-test @@ -1,4 +1,4 @@ -{% if cookiecutter.use_python2 == 'n' -%} +{% if cookiecutter.use_python3 == 'y' -%} FROM python:3.5 {% else %} FROM python:2.7