diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index b383e4190..8546f9374 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -193,6 +193,27 @@ def remove_elasticbeanstalk(): PROJECT_DIRECTORY, filename )) +def remove_elasticbeanstalk_config(): + """ + Removes elastic beanstalk configuration components, + but keeps the envvars config. + """ + eb_config_dir_location = os.path.join(PROJECT_DIRECTORY, '.ebextensions') + if os.path.exists(eb_config_dir_location): + for filename in os.listdir(eb_config_dir_location): + if filename != '01_envvars.config.example': + os.remove(os.path.join( + eb_config_dir_location, filename + )) + +def remove_nginx(): + """ + Removes NGINX components + """ + nginx_dir_location = os.path.join(PROJECT_DIRECTORY, 'compose/production/nginx') + if os.path.exists(nginx_dir_location): + shutil.rmtree(nginx_dir_location) + def remove_open_source_files(): """ Removes files conventional to opensource projects only. @@ -268,6 +289,14 @@ if '{{ cookiecutter.open_source_license}}' != 'GPLv3': if '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower() != 'y': remove_elasticbeanstalk() +# Remove Elastic Beanstalk files, except envvars +if '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower() == 'y' and '{{ cookiecutter.use_docker }}'.lower() == 'y': + remove_elasticbeanstalk_config() + +# Remove NGINX files +if '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower() != 'y' and '{{ cookiecutter.use_docker }}'.lower() != 'y': + remove_nginx() + # Remove files conventional to opensource projects only. if '{{ cookiecutter.open_source_license }}' == 'Not open source': remove_open_source_files() diff --git a/hooks/pre_gen_project.py b/hooks/pre_gen_project.py index c7a68450b..dd016d97a 100644 --- a/hooks/pre_gen_project.py +++ b/hooks/pre_gen_project.py @@ -7,8 +7,8 @@ elasticbeanstalk = '{{ cookiecutter.use_elasticbeanstalk_experimental }}'.lower( heroku = '{{ cookiecutter.use_heroku }}'.lower() docker = '{{ cookiecutter.use_docker }}'.lower() -if elasticbeanstalk == 'y' and (heroku == 'y' or docker == 'y'): - raise Exception("Cookiecutter Django's EXPERIMENTAL Elastic Beanstalk support is incompatible with Heroku and Docker setups.") +if elasticbeanstalk == 'y' and (heroku == 'y'): + raise Exception("Cookiecutter Django's EXPERIMENTAL Elastic Beanstalk support is incompatible with Heroku setup.") if docker == 'n': import sys @@ -26,6 +26,6 @@ if docker == 'n': elif choice in yes_options: pass else: - sys.stdout.write("Please respond with %s or %s" + sys.stdout.write("Please respond with %s or %s" % (', '.join([o for o in yes_options if not o == '']) , ', '.join([o for o in no_options if not o == '']))) diff --git a/{{cookiecutter.project_slug}}/.ebextensions/01_envvars.config.example b/{{cookiecutter.project_slug}}/.ebextensions/01_envvars.config.example new file mode 100644 index 000000000..d465f5db3 --- /dev/null +++ b/{{cookiecutter.project_slug}}/.ebextensions/01_envvars.config.example @@ -0,0 +1,27 @@ +# Elastic Beanstalk configuration + +option_settings: + "aws:elasticbeanstalk:application:environment": + "DJANGO_SETTINGS_MODULE": "config.settings.production" + "DJANGO_SECRET_KEY": "" + "DJANGO_MAILGUN_API_KEY": "" + "DJANGO_SERVER_EMAIL": "" + "MAILGUN_SENDER_DOMAIN": "" + "DJANGO_DEBUG": "false" + "DJANGO_ALLOWED_HOSTS": ".elasticbeanstalk.com" + "DJANGO_ADMIN_URL": "" + "DJANGO_SECURE_SSL_REDIRECT": "true" + "DJANGO_ACCOUNT_ALLOW_REGISTRATION": "true" + "DJANGO_SENTRY_DSN": "" + "DJANGO_OPBEAT_ORGANIZATION_ID": "" + "DJANGO_OPBEAT_APP_ID": "" + "DJANGO_OPBEAT_SECRET_TOKEN": "" + "RDS_PORT": "5432" + "DJANGO_AWS_STORAGE_BUCKET_NAME": "" + "RDS_DB_NAME": "" + "DJANGO_AWS_SECRET_ACCESS_KEY": "" + "RDS_PASSWORD": "" + "REDIS_URL": "" + "RDS_HOSTNAME": "" + "RDS_USERNAME": "" + "DJANGO_AWS_ACCESS_KEY_ID": "" diff --git a/{{cookiecutter.project_slug}}/.gitignore b/{{cookiecutter.project_slug}}/.gitignore index 65feea000..440b4c524 100644 --- a/{{cookiecutter.project_slug}}/.gitignore +++ b/{{cookiecutter.project_slug}}/.gitignore @@ -367,3 +367,8 @@ mailhog # See issue https://github.com/pydanny/cookiecutter-django/issues/1321 !/compose/local/ {% endif %} + +{% if cookiecutter.use_elasticbeanstalk_experimental == 'y' and cookiecutter.use_docker == 'y' -%} +# Environment variables for your Beanstalk deployment +01_envvars.config +{% endif %} diff --git a/{{cookiecutter.project_slug}}/Dockerrun.aws.json b/{{cookiecutter.project_slug}}/Dockerrun.aws.json new file mode 100644 index 000000000..3ab5b504d --- /dev/null +++ b/{{cookiecutter.project_slug}}/Dockerrun.aws.json @@ -0,0 +1,65 @@ +{ + "AWSEBDockerrunVersion": 2, + "volumes": [ + { + "name": "django", + "host": { + "sourcePath": "/var/app/current/app" + } + }, + { + "name": "nginx-proxy-conf", + "host": { + "sourcePath": "/var/app/current/compose/production/nginx/nginx.conf" + } + } + ], + "containerDefinitions": [ + { + "name": "django", + "image": "", + "essential": true, + "memory": 256, + "command": ["/gunicorn.sh"], + "mountPoints": [ + { + "sourceVolume": "django", + "containerPath": "/django", + "readOnly": false + } + ] + }, + { + "name": "nginx-proxy", + "image": "nginx:latest", + "essential": true, + "memory": 128, + "portMappings": [ + { + "hostPort": 80, + "containerPort": 80 + } + ], + "depends_on": ["django"], + "links": [ + "django" + ], + "mountPoints": [ + { + "sourceVolume": "django", + "containerPath": "/django", + "readOnly": true + }, + { + "sourceVolume": "awseb-logs-nginx-proxy", + "containerPath": "/var/log/nginx" + }, + { + "sourceVolume": "nginx-proxy-conf", + "containerPath": "/etc/nginx/nginx.conf", + "readOnly": true + } + ] + } + ] +} diff --git a/{{cookiecutter.project_slug}}/compose/production/django/entrypoint.sh b/{{cookiecutter.project_slug}}/compose/production/django/entrypoint.sh index 3b83c7bb6..08cf04d23 100644 --- a/{{cookiecutter.project_slug}}/compose/production/django/entrypoint.sh +++ b/{{cookiecutter.project_slug}}/compose/production/django/entrypoint.sh @@ -9,6 +9,7 @@ set -o pipefail cmd="$@" +{% if cookiecutter.use_docker != 'y' or cookiecutter.use_elasticbeanstalk_experimental != 'y' %} # This entrypoint is used to play nicely with the current cookiecutter configuration. # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple # environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint @@ -43,4 +44,6 @@ until postgres_ready; do done >&2 echo "Postgres is up - continuing..." +{% endif %} + exec $cmd diff --git a/{{cookiecutter.project_slug}}/compose/production/django/gunicorn.sh b/{{cookiecutter.project_slug}}/compose/production/django/gunicorn.sh index 25da06496..9fd2cbc4e 100644 --- a/{{cookiecutter.project_slug}}/compose/production/django/gunicorn.sh +++ b/{{cookiecutter.project_slug}}/compose/production/django/gunicorn.sh @@ -4,6 +4,6 @@ set -o errexit set -o pipefail set -o nounset - +python /app/manage.py migrate --noinput python /app/manage.py collectstatic --noinput /usr/local/bin/gunicorn config.wsgi -w 4 -b 0.0.0.0:5000 --chdir=/app diff --git a/{{cookiecutter.project_slug}}/compose/production/nginx/nginx.conf b/{{cookiecutter.project_slug}}/compose/production/nginx/nginx.conf new file mode 100644 index 000000000..d677a281c --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/production/nginx/nginx.conf @@ -0,0 +1,50 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + upstream app { + server django:5000; + } + + server { + listen 80; + charset utf-8; + + location / { + # checks for static file, if not found proxy to app + try_files $uri @proxy_to_app; + } + + # cookiecutter-django app + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app; + + } + } +} diff --git a/{{cookiecutter.project_slug}}/config/settings/production.py b/{{cookiecutter.project_slug}}/config/settings/production.py index db5813061..2a5d1ea83 100644 --- a/{{cookiecutter.project_slug}}/config/settings/production.py +++ b/{{cookiecutter.project_slug}}/config/settings/production.py @@ -82,6 +82,8 @@ X_FRAME_OPTIONS = 'DENY' # Hosts/domain names that are valid for this site # See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['{{cookiecutter.domain_name}}', ]) + + # END SITE CONFIGURATION INSTALLED_APPS += ['gunicorn', ] diff --git a/{{cookiecutter.project_slug}}/docs/docker_ebs.rst b/{{cookiecutter.project_slug}}/docs/docker_ebs.rst new file mode 100644 index 000000000..e962a2180 --- /dev/null +++ b/{{cookiecutter.project_slug}}/docs/docker_ebs.rst @@ -0,0 +1,206 @@ +Deployment on EBS +================= + +.. index:: Amazon Elastic Beanstalk + +This is still very much work in progress. Testing is needed and appreciated. + +Instructions on how to get django-cookiecutter to work on Amazon Elastic Beanstalk (EBS) with Docker Multicontainer setup. + +It is supposed that the developer is using AWS services: +- RDS for database +- Route 53 for DNS +- Certificate manager for HTTPS +- Elasticache for Redis caching +- IAM for access management +- S3 for file storage + + +CLI +----- +Install awsebcli (included in local requirements file). + +Instructions can be found on +- http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3.html +- http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-getting-started.html + + +Create docker images +-------------------- +.. code-block:: bash + docker build -f compose/production/django/Dockerfile -t / . + +Note: the dot "." in the end of the command is important! + + +Upload docker images +-------------------- +At the time of writing, multicontainer EBS does not support uploading containers straight to AWS. You need to add the containers to a container hub. + +.. code-block:: bash + docker push / + + +You can host your containers on the standard Docker hub, which will give you unlimited public containers and 1 private container. +You can also host your containers on a private container hub, e.g. (free tier options) +- ECR +- https://cloud.google.com/container-engine/ +- https://arukas.io/en/ + +Docs at http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker.container.console.html +You can upload a .cfg file in the S3 bucket that the Beanstalk deploy created, that way you won't have any trouble with permissions. + +Update Dockerrun file +--------------------- +After uploading your Docker image, you need to update your Dockerrun.aws.json file to point to the correct repository / +If you are using a private repository, you need to add a configuration file to a secure S3 bucket. Info on http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker.container.console.html#docker-images-private + + +Setup EBS +--------- +Follow normal instructions for setting up an EBS instance: + +.. code-block:: bash + eb init + eb create + eb deploy + + +Set environment variables +------------------------- +Environment variables can be set in multiple ways: +- Through the CLI http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environment-configuration-methods-after.html#configuration-options-after-ebcli-ebsetenv +- Through the console http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environment-configuration-methods-after.html#configuration-options-after-console-configpage +- Through .ebextensions http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/environment-configuration-methods-after.html#configuration-options-after-console-ebextensions + + +Local run +--------- +You can test out your setup locally by adding "local" after the eb command +.. code-block:: bash + eb local run + +http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb3-local.html +http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker-eblocal.html + +RDS +------ + +You can setup RDS for your production and development usage. + +* Production +It is possible to create an RDS instance through your EBS console. http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features.managing.db.html +However, it is recommended to create a RDS DB instance seperately and then link this to you EBS setup. This way both lifecycles are seperate and you can delete your EBS without losing your RDS. +http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/AWSHowTo.RDS.html + +* Development +It is adviced to create a seperate development RDS instance for your local development. +- Go to RDS console +- Go to Instances +- Launch DB instance +- Choose PostgreSQL +- Choose Dev/Test option +- Fill out all the fields +- Wait for your database to be created +- Set all the environment variables locally. You can find the RDS_HOSTNAME as "endpoint". You do need to remove the port, as this is a separate environment variable. + +Route 53 +-------- +- Add a hosted zone with your domain name +- Find the NS record in your hosted zone, these are nameservers +- Copy the nameservers to your DNS host + +Certificate manager +------------------- +AWS provide you with free SSL certificates. Request a certificate through the Certificate Manager. + +- You can add you domain to "Domain name" (e.g. example.com) and add every subdomain to "Additional names" (e.g. *.example.com). +- An email will be sent to admin@example.com to verify if you are the owner of the domain, if it is not registered through AWS. + + +ElastiCache +----------- +Launch a Redis instance in ElastiCache and copy the Port and Endpoint to your environment variables. + +* Note: The author has only tested that "it doesn't crash". Please open a ticket if it turns out that nothing is caching. + + +IAM +----- +Using your root account for all AWS is a bad idea. Follow the recommendations in your "Security Status" section in the IAM dashboard. + +You need following Policies attached to your user/group: +- AWSElasticBeanstalkReadOnlyAccess +- AWSElasticBeanstalkFullAccess +- AWSElasticBeanstalkService + +S3 +----- +As S3 is already the default for django-cookiecutter, nothing extra needs to be done here. + + +Useful commands +--------------- +.. code-block:: bash + eb terminate + eb setenv VAR=value + + +Running commands +---------------- + +It is possible to run django commands, such as createsuperuser. + +Note: there might be better ways of doing this, PRs welcome! + +You can ssh into your EBS instance: +.. code-block:: bash + eb ssh + +Then there you can go in the correct Docker instance. +1. Find the name of the Docker instance +.. code-block:: bash + sudo docker ps + +2. Log onto the Docker instance +.. code-block:: bash + sudo docker exec -it bash + +3. Navigate to correct folder +.. code-block:: bash + cd /var/app/current + +4. Run commands +.. code-block:: bash + python manage.py createsuperuser + + + +Documentation +------------- +http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_docker_ecs.html + +TODO +---- + +* Celery +Should Celery have it's own container? How does Celery behave when EBS boots up multiple containers, each with running Celery workers? + +CELERY +https://github.com/Maxbey/socialaggregator/blob/232690ef14ffbd7735297262ab6c26717bd53f05/aws/Dockerrun.aws.json +https://github.com/pogorelov-ss/django-elastic-beanstalk-docker-stack/blob/fb1e717ec3be0b7fef99497d4e27626386da100f/Dockerrun.aws.json + +* Do we need something like Supervisor on EBS? + +Troubleshooting +--------------- + +* Package version mismatch +There are issues that come from a mismatch between docker, compose and awsebcli packages. +For awsebcli to function, you need to install docker-py outside your virtual environment. +.. code-block:: bash + sudo pip install docker-py==1.7.2 + + +* SECURE_SSL_REDIRECT +The author didn't get it to run on production without setting up HTTPS certificates correctly, even with SECURE_SSL_REDIRECT set to False. diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index f86cae8cc..4fcb32e6b 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -17,3 +17,9 @@ ipdb==0.10.3 pytest-django==3.1.2 pytest-sugar==0.9.0 + +{% if cookiecutter.use_elasticbeanstalk_experimental == "y" -%} +# AWS Beanstalk CLI +# ----------------------------------------- +awsebcli==3.10.6 +{%- endif %}