1. Added development section to README

2. Added docker testing environment

.dockerignore

@@ -0,0 +1,15 @@
+# Docs
+docs/
+
+# Python cache files
+**/__pycache__/
+
+# Private and public keys
+*.key
+*.ppk
+*.pub
+
+# Hidden apps directories
+.github/
+.idea/
+.gitignore

.travis.yml

@@ -1,101 +0,0 @@
-dist: xenial
-sudo: required
-language: python
-cache:
-  pip: true
-  apt: true
-
-services:
-  - postgresql
-  - redis-server
-addons:
-  postgresql: "11"
-  apt:
-    sources:
-      - sourceline: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
-      - sourceline: "deb https://packages.erlang-solutions.com/ubuntu xenial contrib"
-        key_url: "https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc"
-      - sourceline: "deb https://dl.bintray.com/rabbitmq/debian xenial main"
-        key_url: "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc"
-    packages:
-      - dirmngr
-      - apt-transport-https
-      - postgresql-contrib-9.6
-      - postgresql-10
-      - postgresql-contrib-10
-      - postgresql-client-10
-      - postgresql-11
-      - postgresql-contrib-11
-      - postgresql-client-11
-      - postgresql-12
-      - postgresql-contrib-12
-      - postgresql-client-12
-      - unzip
-      - rabbitmq-server
-
-python:
-  - 3.6
-  - 3.7
-  - 3.8
-
-env:
-  - PG=9.6 DJANGO=2.1
-  - PG=10 DJANGO=2.1
-  - PG=11 DJANGO=2.1
-  - PG=12 DJANGO=2.1
-  - PG=9.6 DJANGO=2.2
-  - PG=10 DJANGO=2.2
-  - PG=11 DJANGO=2.2
-  - PG=12 DJANGO=2.2
-  - PG=9.6 DJANGO=3.0
-  - PG=10 DJANGO=3.0
-  - PG=11 DJANGO=3.0
-  - PG=12 DJANGO=3.0
-  - PG=9.6 DJANGO=3.1
-  - PG=10 DJANGO=3.1
-  - PG=11 DJANGO=3.1
-  - PG=12 DJANGO=3.1
-
-before_install:
-  # Use default PostgreSQL 11 port
-  - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf
-  - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf
-
-  - sudo sed -i 's/port = 5434/port = 5432/' /etc/postgresql/12/main/postgresql.conf
-  - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf
-
-  # Start PostgreSQL version we need
-  - sudo systemctl stop postgresql
-  - sudo systemctl start postgresql@$PG-main
-
-  # ClickHouse sources
-  - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
-  - sudo apt-get update
-
-
-install:
-  # Install ClickHouse
-  - sudo apt-get install clickhouse-client clickhouse-server clickhouse-common-static
-  - sudo service clickhouse-server restart
-
-  - pip install -r requirements-test.txt
-  - pip install -q Django==$DJANGO.*
-  - python setup.py -q install
-
-before_script:
-  # Output software versions
-  - erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().'  -noshell
-  - rabbitmqctl status | grep "RabbitMQ"
-  - clickhouse-client --query "SELECT version();"
-  - psql -tc 'SHOW server_version' -U postgres
-
-  - psql -tc 'SHOW server_version' -U postgres
-  - psql -c 'CREATE ROLE test;' -U postgres
-  - psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres
-  - psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres
-  - psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres
-  - psql -c 'CREATE DATABASE test OWNER test;' -U postgres
-  - psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres
-
-script:
-  python runtests.py

Dockerfile

@@ -0,0 +1,47 @@
+ARG PYTHON_IMAGE_TAG=latest
+
+FROM python:${PYTHON_IMAGE_TAG}  AS image_stage
+
+ARG APP_TAG="1.0.3"
+
+LABEL \
+  org.label-schema.build-date=Now \
+  org.label-schema.maintainer="m1ha@carrotquest.io" \
+  org.label-schema.schema-version="1.0.0-rc1" \
+  org.label-schema.vcs-ref="v${APP_TAG}" \
+  org.label-schema.vcs-url="https://github.com/carrotquest/django-clickhouse" \
+  org.label-schema.vendor="Carrot quest" \
+  org.label-schema.version="${APP_TAG}"
+
+ENV APP_UID ${APP_UID:-1000}
+ENV APP_GID ${APP_GID:-1000}
+ENV APP_NAME ${APP_NAME:-"app"}
+
+# Configure utf-8 locales to make sure Python correctly handles unicode filenames
+# Configure pip local path to copy data from pip_stage
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DJANGO_SETTINGS_MODULE=tests.settings PYTHONUSERBASE=/pip PATH=/pip/bin:$PATH
+
+RUN set -eu && \
+  groupadd --gid "${APP_GID}" "app" && \
+  useradd --uid ${APP_UID} --gid ${APP_GID} --create-home --shell /bin/bash -d /app app && \
+  mkdir -p /pip && \
+  chmod  755 /app /pip && \
+  chown -R ${APP_UID}:${APP_GID} /app /pip
+
+WORKDIR /app/src
+
+# Install dependencies
+# set -eu "breaks" pipeline on first error
+COPY ./requirements-test.txt /app/requirements-test.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+  set -eu && \
+  python3 -m pip install --upgrade pip setuptools wheel  && \
+  python3 -m pip install --upgrade --requirement /app/requirements-test.txt
+
+COPY . /app/src
+
+RUN python3 setup.py -q install --user
+
+USER ${APP_UID}
+
+CMD ["python3", "runtests.py"]

docker-compose.yml

@@ -0,0 +1,39 @@
+---
+version: "3.9"
+services:
+  redis_db:
+    image: redis
+    command: [sh, -c, "redis-server --save '' --appendonly no"]  # disable persistence
+    mem_limit: 512m
+    cpus: 1
+
+  postgres_db:
+    image: postgres
+    environment:
+      - POSTGRES_PASSWORD=postgres
+    mem_limit: 1g
+    cpus: 1
+
+  clickhouse_db:
+    image: yandex/clickhouse-server
+    mem_limit: 1g
+    cpus: 1
+
+  run_tests:
+    image: django-clickhouse
+    build:
+      context: .
+      args:
+        - PYTHON_VER=latest
+    environment:
+      - REDIS_HOST=redis_db
+      - PGHOST=postgres_db
+      - PGUSER=postgres
+      - PGPASS=postgres
+      - "CLICK_HOUSE_HOST=http://clickhouse_db:8123"
+    depends_on:
+      - redis_db
+      - postgres_db
+      - clickhouse_db
+    mem_limit: 1g
+    cpus: 1

docs/development.md

@@ -0,0 +1,30 @@
+# Development
+## Basic info
+This is an Open source project developed by `Carrot quest` team under MIT license. 
+Feel free to create issues and make pull requests.  
+Query and database system wraps [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) library. 
+If you want to develop QuerySet system, it is better to contribute there.  
+
+
+## General info about testing
+Library test system is based on [django.test](https://docs.djangoproject.com/en/3.2/topics/testing/overview/).
+You can find them in `tests` directory. 
+
+## Tests requirements  
+* [Redis](https://redis.io/)  
+* [Yandex ClickHouse](https://clickhouse.yandex/)  
+* [PostgreSQL](https://www.postgresql.org/)  
+* Pypi libraries listed in `requirements-test.txt` file
+
+## Running tests
+### Running in docker
+1. Install [docker and docker-compose](https://www.docker.com/)  
+2. Run `docker-compose run run_tests` in project directory
+
+### Running in virtual environment
+1. Install all requirements listed above 
+2. [Create virtual environment](https://docs.python.org/3/tutorial/venv.html)
+3. Install requirements  
+  `pip3 install -U -r requirements-test.txt`  
+4. Start tests  
+  `python3 runtests.py`

docs/index.md

@@ -20,3 +20,4 @@
         * [RedisStorage](storages.md#redisstorage)
     * [Monitoring](monitoring.md)
     * [Performance notes](performance.md)
+* [Development](development.md)

runtests.py

@@ -18,6 +18,6 @@ if __name__ == "__main__":
     os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings'
     django.setup()
     TestRunner = get_runner(settings)
-    test_runner = TestRunner()
+    test_runner = TestRunner(interactive=False)
     failures = test_runner.run_tests(["tests"])
     sys.exit(bool(failures))

tests/settings.py

@@ -1,36 +1,39 @@
 """
 This file contains django settings to run tests with runtests.py
 """
+from os import environ
+
 SECRET_KEY = 'fake-key'
 
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'NAME': 'test',
-        'USER': 'test',
-        'PASSWORD': 'test',
-        'HOST': '127.0.0.1',
-        'PORT': '5432'
+        'USER': environ.get('PGUSER', 'test'),
+        'PASSWORD': environ.get('PGPASS', 'test'),
+        'HOST': environ.get('PGHOST', '127.0.0.1'),
+        'PORT': environ.get('PGPORT', 5432)
     },
     'secondary': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'NAME': 'test2',
-        'USER': 'test',
-        'PASSWORD': 'test',
-        'HOST': '127.0.0.1',
-        'PORT': '5432'
+        'USER': environ.get('PGUSER', 'test'),
+        'PASSWORD': environ.get('PGPASS', 'test'),
+        'HOST': environ.get('PGHOST', '127.0.0.1'),
+        'PORT': environ.get('PGPORT', 5432)
     },
 
     # I need separate connections for multiprocessing tests
     'test_db': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'NAME': 'test_test',
-        'USER': 'test',
-        'PASSWORD': 'test',
-        'HOST': '127.0.0.1',
-        'PORT': '5432'
+        'USER': environ.get('PGUSER', 'test'),
+        'PASSWORD': environ.get('PGPASS', 'test'),
+        'HOST': environ.get('PGHOST', '127.0.0.1'),
+        'PORT': environ.get('PGPORT', 5432)
     },
 }
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 LOGGING = {
     'version': 1,
@@ -58,22 +61,26 @@ INSTALLED_APPS = [
 
 CLICKHOUSE_DATABASES = {
     'default': {
+        'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'),
         'db_name': 'test',
         'username': 'default',
         'password': ''
     },
     'secondary': {
+        'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'),
         'db_name': 'test_2',
         'username': 'default',
         'password': ''
     },
     'no_migrate': {
+        'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'),
         'db_name': 'test_3',
         'username': 'default',
         'password': '',
         'migrate': False
     },
     'readonly': {
+        'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'),
         'db_name': 'test_3',
         'username': 'default',
         'password': '',
@@ -84,8 +91,8 @@ CLICKHOUSE_DATABASES = {
 CLICKHOUSE_SYNC_BATCH_SIZE = 5000
 
 CLICKHOUSE_REDIS_CONFIG = {
-    'host': '127.0.0.1',
-    'port': 6379,
+    'host': environ.get('REDIS_HOST', '127.0.0.1'),
+    'port': environ.get('REDIS_PORT', 6379),
     'db': 8,
     'socket_timeout': 10
 }