From 274e31a5252181c3e3fd9f7b12d5ec132ed590f6 Mon Sep 17 00:00:00 2001 From: M1ha Date: Thu, 8 Jul 2021 12:03:58 +0500 Subject: [PATCH] 1. Added development section to README 2. Added docker testing environment --- .dockerignore | 15 +++++++ .travis.yml | 101 -------------------------------------------- Dockerfile | 47 +++++++++++++++++++++ docker-compose.yml | 39 +++++++++++++++++ docs/development.md | 30 +++++++++++++ docs/index.md | 1 + runtests.py | 2 +- tests/settings.py | 35 +++++++++------ 8 files changed, 154 insertions(+), 116 deletions(-) create mode 100644 .dockerignore delete mode 100644 .travis.yml create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docs/development.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..80b5541 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +# Docs +docs/ + +# Python cache files +**/__pycache__/ + +# Private and public keys +*.key +*.ppk +*.pub + +# Hidden apps directories +.github/ +.idea/ +.gitignore \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index da00d5c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,101 +0,0 @@ -dist: xenial -sudo: required -language: python -cache: - pip: true - apt: true - -services: - - postgresql - - redis-server -addons: - postgresql: "11" - apt: - sources: - - sourceline: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" - - sourceline: "deb https://packages.erlang-solutions.com/ubuntu xenial contrib" - key_url: "https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc" - - sourceline: "deb https://dl.bintray.com/rabbitmq/debian xenial main" - key_url: "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" - packages: - - dirmngr - - apt-transport-https - - postgresql-contrib-9.6 - - postgresql-10 - - postgresql-contrib-10 - - postgresql-client-10 - - postgresql-11 - - postgresql-contrib-11 - - postgresql-client-11 - - postgresql-12 - - postgresql-contrib-12 - - postgresql-client-12 - - unzip - - rabbitmq-server - -python: - - 3.6 - - 3.7 - - 3.8 - -env: - - PG=9.6 DJANGO=2.1 - - PG=10 DJANGO=2.1 - - PG=11 DJANGO=2.1 - - PG=12 DJANGO=2.1 - - PG=9.6 DJANGO=2.2 - - PG=10 DJANGO=2.2 - - PG=11 DJANGO=2.2 - - PG=12 DJANGO=2.2 - - PG=9.6 DJANGO=3.0 - - PG=10 DJANGO=3.0 - - PG=11 DJANGO=3.0 - - PG=12 DJANGO=3.0 - - PG=9.6 DJANGO=3.1 - - PG=10 DJANGO=3.1 - - PG=11 DJANGO=3.1 - - PG=12 DJANGO=3.1 - -before_install: - # Use default PostgreSQL 11 port - - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf - - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf - - - sudo sed -i 's/port = 5434/port = 5432/' /etc/postgresql/12/main/postgresql.conf - - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf - - # Start PostgreSQL version we need - - sudo systemctl stop postgresql - - sudo systemctl start postgresql@$PG-main - - # ClickHouse sources - - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 - - sudo apt-get update - - -install: - # Install ClickHouse - - sudo apt-get install clickhouse-client clickhouse-server clickhouse-common-static - - sudo service clickhouse-server restart - - - pip install -r requirements-test.txt - - pip install -q Django==$DJANGO.* - - python setup.py -q install - -before_script: - # Output software versions - - erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell - - rabbitmqctl status | grep "RabbitMQ" - - clickhouse-client --query "SELECT version();" - - psql -tc 'SHOW server_version' -U postgres - - - psql -tc 'SHOW server_version' -U postgres - - psql -c 'CREATE ROLE test;' -U postgres - - psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres - - psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres - - psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres - - psql -c 'CREATE DATABASE test OWNER test;' -U postgres - - psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres - -script: - python runtests.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c5a37b5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +ARG PYTHON_IMAGE_TAG=latest + +FROM python:${PYTHON_IMAGE_TAG} AS image_stage + +ARG APP_TAG="1.0.3" + +LABEL \ + org.label-schema.build-date=Now \ + org.label-schema.maintainer="m1ha@carrotquest.io" \ + org.label-schema.schema-version="1.0.0-rc1" \ + org.label-schema.vcs-ref="v${APP_TAG}" \ + org.label-schema.vcs-url="https://github.com/carrotquest/django-clickhouse" \ + org.label-schema.vendor="Carrot quest" \ + org.label-schema.version="${APP_TAG}" + +ENV APP_UID ${APP_UID:-1000} +ENV APP_GID ${APP_GID:-1000} +ENV APP_NAME ${APP_NAME:-"app"} + +# Configure utf-8 locales to make sure Python correctly handles unicode filenames +# Configure pip local path to copy data from pip_stage +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DJANGO_SETTINGS_MODULE=tests.settings PYTHONUSERBASE=/pip PATH=/pip/bin:$PATH + +RUN set -eu && \ + groupadd --gid "${APP_GID}" "app" && \ + useradd --uid ${APP_UID} --gid ${APP_GID} --create-home --shell /bin/bash -d /app app && \ + mkdir -p /pip && \ + chmod 755 /app /pip && \ + chown -R ${APP_UID}:${APP_GID} /app /pip + +WORKDIR /app/src + +# Install dependencies +# set -eu "breaks" pipeline on first error +COPY ./requirements-test.txt /app/requirements-test.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + set -eu && \ + python3 -m pip install --upgrade pip setuptools wheel && \ + python3 -m pip install --upgrade --requirement /app/requirements-test.txt + +COPY . /app/src + +RUN python3 setup.py -q install --user + +USER ${APP_UID} + +CMD ["python3", "runtests.py"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a0f992f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +--- +version: "3.9" +services: + redis_db: + image: redis + command: [sh, -c, "redis-server --save '' --appendonly no"] # disable persistence + mem_limit: 512m + cpus: 1 + + postgres_db: + image: postgres + environment: + - POSTGRES_PASSWORD=postgres + mem_limit: 1g + cpus: 1 + + clickhouse_db: + image: yandex/clickhouse-server + mem_limit: 1g + cpus: 1 + + run_tests: + image: django-clickhouse + build: + context: . + args: + - PYTHON_VER=latest + environment: + - REDIS_HOST=redis_db + - PGHOST=postgres_db + - PGUSER=postgres + - PGPASS=postgres + - "CLICK_HOUSE_HOST=http://clickhouse_db:8123" + depends_on: + - redis_db + - postgres_db + - clickhouse_db + mem_limit: 1g + cpus: 1 diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..f9f8c48 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,30 @@ +# Development +## Basic info +This is an Open source project developed by `Carrot quest` team under MIT license. +Feel free to create issues and make pull requests. +Query and database system wraps [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) library. +If you want to develop QuerySet system, it is better to contribute there. + + +## General info about testing +Library test system is based on [django.test](https://docs.djangoproject.com/en/3.2/topics/testing/overview/). +You can find them in `tests` directory. + +## Tests requirements +* [Redis](https://redis.io/) +* [Yandex ClickHouse](https://clickhouse.yandex/) +* [PostgreSQL](https://www.postgresql.org/) +* Pypi libraries listed in `requirements-test.txt` file + +## Running tests +### Running in docker +1. Install [docker and docker-compose](https://www.docker.com/) +2. Run `docker-compose run run_tests` in project directory + +### Running in virtual environment +1. Install all requirements listed above +2. [Create virtual environment](https://docs.python.org/3/tutorial/venv.html) +3. Install requirements + `pip3 install -U -r requirements-test.txt` +4. Start tests + `python3 runtests.py` diff --git a/docs/index.md b/docs/index.md index 8aedf21..8e5e46f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,3 +20,4 @@ * [RedisStorage](storages.md#redisstorage) * [Monitoring](monitoring.md) * [Performance notes](performance.md) +* [Development](development.md) \ No newline at end of file diff --git a/runtests.py b/runtests.py index 674b064..afb72fa 100644 --- a/runtests.py +++ b/runtests.py @@ -18,6 +18,6 @@ if __name__ == "__main__": os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings' django.setup() TestRunner = get_runner(settings) - test_runner = TestRunner() + test_runner = TestRunner(interactive=False) failures = test_runner.run_tests(["tests"]) sys.exit(bool(failures)) diff --git a/tests/settings.py b/tests/settings.py index 5c4f558..1382d38 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -1,36 +1,39 @@ """ This file contains django settings to run tests with runtests.py """ +from os import environ + SECRET_KEY = 'fake-key' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'test', - 'USER': 'test', - 'PASSWORD': 'test', - 'HOST': '127.0.0.1', - 'PORT': '5432' + 'USER': environ.get('PGUSER', 'test'), + 'PASSWORD': environ.get('PGPASS', 'test'), + 'HOST': environ.get('PGHOST', '127.0.0.1'), + 'PORT': environ.get('PGPORT', 5432) }, 'secondary': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'test2', - 'USER': 'test', - 'PASSWORD': 'test', - 'HOST': '127.0.0.1', - 'PORT': '5432' + 'USER': environ.get('PGUSER', 'test'), + 'PASSWORD': environ.get('PGPASS', 'test'), + 'HOST': environ.get('PGHOST', '127.0.0.1'), + 'PORT': environ.get('PGPORT', 5432) }, # I need separate connections for multiprocessing tests 'test_db': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'test_test', - 'USER': 'test', - 'PASSWORD': 'test', - 'HOST': '127.0.0.1', - 'PORT': '5432' + 'USER': environ.get('PGUSER', 'test'), + 'PASSWORD': environ.get('PGPASS', 'test'), + 'HOST': environ.get('PGHOST', '127.0.0.1'), + 'PORT': environ.get('PGPORT', 5432) }, } +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' LOGGING = { 'version': 1, @@ -58,22 +61,26 @@ INSTALLED_APPS = [ CLICKHOUSE_DATABASES = { 'default': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test', 'username': 'default', 'password': '' }, 'secondary': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test_2', 'username': 'default', 'password': '' }, 'no_migrate': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test_3', 'username': 'default', 'password': '', 'migrate': False }, 'readonly': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test_3', 'username': 'default', 'password': '', @@ -84,8 +91,8 @@ CLICKHOUSE_DATABASES = { CLICKHOUSE_SYNC_BATCH_SIZE = 5000 CLICKHOUSE_REDIS_CONFIG = { - 'host': '127.0.0.1', - 'port': 6379, + 'host': environ.get('REDIS_HOST', '127.0.0.1'), + 'port': environ.get('REDIS_PORT', 6379), 'db': 8, 'socket_timeout': 10 }