Merge eb89cb56a5 into 8725809f5e

hooks/post_gen_project.py

@@ -193,6 +193,27 @@ def remove_elasticbeanstalk():
             PROJECT_DIRECTORY, filename
         ))
 
+def remove_elasticbeanstalk_config():
+    """
+    Removes elastic beanstalk configuration components,
+    but keeps the envvars config.
+    """
+    eb_config_dir_location = os.path.join(PROJECT_DIRECTORY, '.ebextensions')
+    if os.path.exists(eb_config_dir_location):
+        for filename in os.listdir(eb_config_dir_location):
+            if filename != '01_envvars.config.example':
+                os.remove(os.path.join(
+                    eb_config_dir_location, filename
+                ))
+
+def remove_nginx():
+    """
+    Removes NGINX components
+    """
+    nginx_dir_location = os.path.join(PROJECT_DIRECTORY, 'compose/production/nginx')
+    if os.path.exists(nginx_dir_location):
+        shutil.rmtree(nginx_dir_location)
+
 def remove_open_source_files():
     """
     Removes files conventional to opensource projects only.
@@ -268,6 +289,14 @@ if '{{ cookiecutter.open_source_license}}' != 'GPLv3':
 if '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower() != 'y':
     remove_elasticbeanstalk()
 
+# Remove Elastic Beanstalk files, except envvars
+if '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower() == 'y' and '{{ cookiecutter.use_docker }}'.lower() == 'y':
+    remove_elasticbeanstalk_config()
+
+# Remove NGINX files
+if '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower() != 'y' and '{{ cookiecutter.use_docker }}'.lower() != 'y':
+    remove_nginx()
+
 # Remove files conventional to opensource projects only.
 if '{{ cookiecutter.open_source_license }}' == 'Not open source':
     remove_open_source_files()

hooks/pre_gen_project.py

@@ -7,8 +7,8 @@ elasticbeanstalk = '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower(
 heroku = '{{ cookiecutter.use_heroku }}'.lower()
 docker = '{{ cookiecutter.use_docker }}'.lower()
 
-if elasticbeanstalk == 'y' and (heroku == 'y' or docker == 'y'):
-    raise Exception("Cookiecutter Django's EXPERIMENTAL Elastic Beanstalk support is incompatible with Heroku and Docker setups.")
+if elasticbeanstalk == 'y' and (heroku == 'y'):
+    raise Exception("Cookiecutter Django's EXPERIMENTAL Elastic Beanstalk support is incompatible with Heroku setup.")
 
 if docker == 'n':
 	import sys
@@ -26,6 +26,6 @@ if docker == 'n':
 		elif choice in yes_options:
 			pass
 		else:
-			sys.stdout.write("Please respond with %s or %s" 
+			sys.stdout.write("Please respond with %s or %s"
 				% (', '.join([o for o in yes_options if not o == ''])
 					, ', '.join([o for o in no_options if not o == ''])))

{{cookiecutter.project_slug}}/.ebextensions/01_envvars.config.example

@@ -0,0 +1,27 @@
+# Elastic Beanstalk configuration
+
+option_settings:
+  "aws:elasticbeanstalk:application:environment":
+    "DJANGO_SETTINGS_MODULE": "config.settings.production"
+    "DJANGO_SECRET_KEY": ""
+    "DJANGO_MAILGUN_API_KEY": ""
+    "DJANGO_SERVER_EMAIL": ""
+    "MAILGUN_SENDER_DOMAIN": ""
+    "DJANGO_DEBUG": "false"
+    "DJANGO_ALLOWED_HOSTS": ".elasticbeanstalk.com"
+    "DJANGO_ADMIN_URL": ""
+    "DJANGO_SECURE_SSL_REDIRECT": "true"
+    "DJANGO_ACCOUNT_ALLOW_REGISTRATION": "true"
+    "DJANGO_SENTRY_DSN": ""
+    "DJANGO_OPBEAT_ORGANIZATION_ID": ""
+    "DJANGO_OPBEAT_APP_ID": ""
+    "DJANGO_OPBEAT_SECRET_TOKEN": ""
+    "RDS_PORT": "5432"
+    "DJANGO_AWS_STORAGE_BUCKET_NAME": ""
+    "RDS_DB_NAME": ""
+    "DJANGO_AWS_SECRET_ACCESS_KEY": ""
+    "RDS_PASSWORD": ""
+    "REDIS_URL": ""
+    "RDS_HOSTNAME": ""
+    "RDS_USERNAME": ""
+    "DJANGO_AWS_ACCESS_KEY_ID": ""

{{cookiecutter.project_slug}}/.gitignore

@@ -367,3 +367,8 @@ mailhog
 # See issue https://github.com/pydanny/cookiecutter-django/issues/1321
 !/compose/local/
 {% endif %}
+
+{% if cookiecutter.use_elasticbeanstalk_experimental == 'y' and cookiecutter.use_docker == 'y' -%}
+# Environment variables for your Beanstalk deployment
+01_envvars.config
+{% endif %}

{{cookiecutter.project_slug}}/Dockerrun.aws.json

@@ -0,0 +1,65 @@
+{
+  "AWSEBDockerrunVersion": 2,
+  "volumes": [
+      {
+          "name": "django",
+          "host": {
+              "sourcePath": "/var/app/current/app"
+          }
+      },
+      {
+        "name": "nginx-proxy-conf",
+        "host": {
+            "sourcePath": "/var/app/current/compose/production/nginx/nginx.conf"
+        }
+      }
+  ],
+  "containerDefinitions": [
+    {
+      "name": "django",
+      "image": "",
+      "essential": true,
+      "memory": 256,
+      "command": ["/gunicorn.sh"],
+      "mountPoints": [
+        {
+          "sourceVolume": "django",
+          "containerPath": "/django",
+          "readOnly": false
+        }
+      ]
+    },
+    {
+      "name": "nginx-proxy",
+      "image": "nginx:latest",
+      "essential": true,
+      "memory": 128,
+      "portMappings": [
+        {
+          "hostPort": 80,
+          "containerPort": 80
+        }
+      ],
+      "depends_on": ["django"],
+      "links": [
+        "django"
+      ],
+      "mountPoints": [
+        {
+          "sourceVolume": "django",
+          "containerPath": "/django",
+          "readOnly": true
+        },
+        {
+          "sourceVolume": "awseb-logs-nginx-proxy",
+          "containerPath": "/var/log/nginx"
+        },
+        {
+          "sourceVolume": "nginx-proxy-conf",
+          "containerPath": "/etc/nginx/nginx.conf",
+          "readOnly": true
+        }
+      ]
+    }
+  ]
+}

{{cookiecutter.project_slug}}/compose/production/django/entrypoint.sh

@@ -9,6 +9,7 @@ set -o pipefail
 
 cmd="$@"
 
+{% if cookiecutter.use_docker != 'y' or cookiecutter.use_elasticbeanstalk_experimental != 'y' %}
 # This entrypoint is used to play nicely with the current cookiecutter configuration.
 # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
 # environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
@@ -43,4 +44,6 @@ until postgres_ready; do
 done
 
 >&2 echo "Postgres is up - continuing..."
+{% endif %}
+
 exec $cmd

{{cookiecutter.project_slug}}/compose/production/django/gunicorn.sh

@@ -4,6 +4,6 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-
+python /app/manage.py migrate --noinput
 python /app/manage.py collectstatic --noinput
 /usr/local/bin/gunicorn config.wsgi -w 4 -b 0.0.0.0:5000 --chdir=/app

{{cookiecutter.project_slug}}/compose/production/nginx/nginx.conf

@@ -0,0 +1,50 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  include       /etc/nginx/mime.types;
+  default_type  application/octet-stream;
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+  '$status $body_bytes_sent "$http_referer" '
+  '"$http_user_agent" "$http_x_forwarded_for"';
+
+  access_log  /var/log/nginx/access.log  main;
+
+  sendfile        on;
+  #tcp_nopush     on;
+
+  keepalive_timeout  65;
+
+  #gzip  on;
+
+  upstream app {
+    server django:5000;
+  }
+
+  server {
+    listen 80;
+    charset     utf-8;
+
+    location / {
+      # checks for static file, if not found proxy to app
+      try_files $uri @proxy_to_app;
+    }
+
+    # cookiecutter-django app
+    location @proxy_to_app {
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header Host $http_host;
+      proxy_redirect off;
+      proxy_pass   http://app;
+
+    }
+  }
+}

{{cookiecutter.project_slug}}/config/settings/production.py

@@ -82,6 +82,8 @@ X_FRAME_OPTIONS = 'DENY'
 # Hosts/domain names that are valid for this site
 # See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['{{cookiecutter.domain_name}}', ])
+
+
 # END SITE CONFIGURATION
 
 INSTALLED_APPS += ['gunicorn', ]

{{cookiecutter.project_slug}}/docs/docker_ebs.rst

@@ -0,0 +1,206 @@
+Deployment on EBS
+=================
+
+.. index:: Amazon Elastic Beanstalk
+
+This is still very much work in progress. Testing is needed and appreciated.
+
+Instructions on how to get django-cookiecutter to work on Amazon Elastic Beanstalk (EBS) with Docker Multicontainer setup.
+
+It is supposed that the developer is using AWS services:
+- RDS for database
+- Route 53 for DNS
+- Certificate manager for HTTPS
+- Elasticache for Redis caching
+- IAM for access management
+- S3 for file storage
+
+
+CLI
+-----
+Install awsebcli (included in local requirements file).
+
+Instructions can be found on
+- http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3.html
+- http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-getting-started.html
+
+
+Create docker images
+--------------------
+.. code-block:: bash
+    docker build -f compose/production/django/Dockerfile -t <user>/<repo> .
+
+Note: the dot "." in the end of the command is important!
+
+
+Upload docker images
+--------------------
+At the time of writing, multicontainer EBS does not support uploading containers straight to AWS. You need to add the containers to a container hub.
+
+.. code-block:: bash
+    docker push <user>/<repo>
+
+
+You can host your containers on the standard Docker hub, which will give you unlimited public containers and 1 private container.
+You can also host your containers on a private container hub, e.g. (free tier options)
+- ECR
+- https://cloud.google.com/container-engine/
+- https://arukas.io/en/
+
+Docs at http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker.container.console.html
+You can upload a .cfg file in the S3 bucket that the Beanstalk deploy created, that way you won't have any trouble with permissions.
+
+Update Dockerrun file
+---------------------
+After uploading your Docker image, you need to update your Dockerrun.aws.json file to point to the correct repository <user>/<repo>
+If you are using a private repository, you need to add a configuration file to a secure S3 bucket. Info on http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker.container.console.html#docker-images-private
+
+
+Setup EBS
+---------
+Follow normal instructions for setting up an EBS instance:
+
+.. code-block:: bash
+    eb init
+    eb create <repo>
+    eb deploy
+
+
+Set environment variables
+-------------------------
+Environment variables can be set in multiple ways:
+- Through the CLI http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environment-configuration-methods-after.html#configuration-options-after-ebcli-ebsetenv
+- Through the console http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environment-configuration-methods-after.html#configuration-options-after-console-configpage
+- Through .ebextensions http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environment-configuration-methods-after.html#configuration-options-after-console-ebextensions
+
+
+Local run
+---------
+You can test out your setup locally by adding "local" after the eb command
+.. code-block:: bash
+    eb local run
+
+http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb3-local.html
+http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker-eblocal.html
+
+RDS
+------
+
+You can setup RDS for your production and development usage.
+
+* Production
+It is possible to create an RDS instance through your EBS console. http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features.managing.db.html
+However, it is recommended to create a RDS DB instance seperately and then link this to you EBS setup. This way both lifecycles are seperate and you can delete your EBS without losing your RDS.
+http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/AWSHowTo.RDS.html
+
+* Development
+It is adviced to create a seperate development RDS instance for your local development.
+- Go to RDS console
+- Go to Instances
+- Launch DB instance
+- Choose PostgreSQL
+- Choose Dev/Test option
+- Fill out all the fields
+- Wait for your database to be created
+- Set all the environment variables locally. You can find the RDS_HOSTNAME as "endpoint". You do need to remove the port, as this is a separate environment variable.
+
+Route 53
+--------
+- Add a hosted zone with your domain name
+- Find the NS record in your hosted zone, these are nameservers
+- Copy the nameservers to your DNS host
+
+Certificate manager
+-------------------
+AWS provide you with free SSL certificates. Request a certificate through the Certificate Manager.
+
+- You can add you domain to "Domain name" (e.g. example.com) and add every subdomain to "Additional names" (e.g. *.example.com).
+- An email will be sent to admin@example.com to verify if you are the owner of the domain, if it is not registered through AWS.
+
+
+ElastiCache
+-----------
+Launch a Redis instance in ElastiCache and copy the Port and Endpoint to your environment variables.
+
+* Note: The author has only tested that "it doesn't crash". Please open a ticket if it turns out that nothing is caching.
+
+
+IAM
+-----
+Using your root account for all AWS is a bad idea. Follow the recommendations in your "Security Status" section in the IAM dashboard.
+
+You need following Policies attached to your user/group:
+- AWSElasticBeanstalkReadOnlyAccess
+- AWSElasticBeanstalkFullAccess
+- AWSElasticBeanstalkService
+
+S3
+-----
+As S3 is already the default for django-cookiecutter, nothing extra needs to be done here.
+
+
+Useful commands
+---------------
+.. code-block:: bash
+    eb terminate
+    eb <env> setenv VAR=value
+
+
+Running commands
+----------------
+
+It is possible to run django commands, such as createsuperuser.
+
+Note: there might be better ways of doing this, PRs welcome!
+
+You can ssh into your EBS instance:
+.. code-block:: bash
+    eb ssh <project_name>
+
+Then there you can go in the correct Docker instance.
+1. Find the name of the Docker instance
+.. code-block:: bash
+    sudo docker ps
+
+2. Log onto the Docker instance
+.. code-block:: bash
+    sudo docker exec -it <docker_instance_name> bash
+
+3. Navigate to correct folder
+.. code-block:: bash
+    cd /var/app/current
+
+4. Run commands
+.. code-block:: bash
+    python manage.py createsuperuser
+
+
+
+Documentation
+-------------
+http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker_ecs.html
+
+TODO
+----
+
+* Celery
+Should Celery have it's own container? How does Celery behave when EBS boots up multiple containers, each with running Celery workers?
+
+CELERY
+https://github.com/Maxbey/socialaggregator/blob/232690ef14ffbd7735297262ab6c26717bd53f05/aws/Dockerrun.aws.json
+https://github.com/pogorelov-ss/django-elastic-beanstalk-docker-stack/blob/fb1e717ec3be0b7fef99497d4e27626386da100f/Dockerrun.aws.json
+
+* Do we need something like Supervisor on EBS?
+
+Troubleshooting
+---------------
+
+* Package version mismatch
+There are issues that come from a mismatch between docker, compose and awsebcli packages.
+For awsebcli to function, you need to install docker-py outside your virtual environment.
+.. code-block:: bash
+    sudo pip install docker-py==1.7.2
+
+
+* SECURE_SSL_REDIRECT
+The author didn't get it to run on production without setting up HTTPS certificates correctly, even with SECURE_SSL_REDIRECT set to False.

{{cookiecutter.project_slug}}/requirements/local.txt

@@ -17,3 +17,9 @@ ipdb==0.10.3
 
 pytest-django==3.1.2
 pytest-sugar==0.9.0
+
+{% if cookiecutter.use_elasticbeanstalk_experimental == "y" -%}
+# AWS Beanstalk CLI
+# -----------------------------------------
+awsebcli==3.10.6
+{%- endif %}