diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..80b5541 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +# Docs +docs/ + +# Python cache files +**/__pycache__/ + +# Private and public keys +*.key +*.ppk +*.pub + +# Hidden apps directories +.github/ +.idea/ +.gitignore \ No newline at end of file diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml new file mode 100644 index 0000000..180817d --- /dev/null +++ b/.github/workflows/python-tests.yml @@ -0,0 +1,90 @@ +name: Python unit tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9"] + postgres-version: ["9.6", "10", "11", "12"] + django-version: ["2.1", "2.2", "3.0", "3.1", "3.2"] + clickhouse-version: ["latest"] + redis-version: ["latest"] + + services: + postgres: + image: postgres:${{ matrix.postgres-version }} + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + clickhouse: + image: yandex/clickhouse-server:${{ matrix.clickhouse-version }} + ports: + - 8123:8123 + + redis: + image: redis:${{ matrix.redis-version }} + ports: + - 6379:6379 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip + uses: actions/cache@v2 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-test.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install pip dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + python -m pip install -r requirements-test.txt + python -m pip install -U django==${{ matrix.django-version }}.* + python setup.py -q install + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude=build/ + + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Set up test databases + run: | + psql -tc 'SHOW server_version' -U postgres -h localhost + psql -c 'CREATE ROLE test;' -U postgres -h localhost + psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres -h localhost + psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres -h localhost + psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres -h localhost + psql -c 'CREATE DATABASE test OWNER test;' -U postgres -h localhost + psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres -h localhost + env: + PGPASSWORD: postgres + + - name: Test with unittest + run: | + python runtests.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index da00d5c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,101 +0,0 @@ -dist: xenial -sudo: required -language: python -cache: - pip: true - apt: true - -services: - - postgresql - - redis-server -addons: - postgresql: "11" - apt: - sources: - - sourceline: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" - - sourceline: "deb https://packages.erlang-solutions.com/ubuntu xenial contrib" - key_url: "https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc" - - sourceline: "deb https://dl.bintray.com/rabbitmq/debian xenial main" - key_url: "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" - packages: - - dirmngr - - apt-transport-https - - postgresql-contrib-9.6 - - postgresql-10 - - postgresql-contrib-10 - - postgresql-client-10 - - postgresql-11 - - postgresql-contrib-11 - - postgresql-client-11 - - postgresql-12 - - postgresql-contrib-12 - - postgresql-client-12 - - unzip - - rabbitmq-server - -python: - - 3.6 - - 3.7 - - 3.8 - -env: - - PG=9.6 DJANGO=2.1 - - PG=10 DJANGO=2.1 - - PG=11 DJANGO=2.1 - - PG=12 DJANGO=2.1 - - PG=9.6 DJANGO=2.2 - - PG=10 DJANGO=2.2 - - PG=11 DJANGO=2.2 - - PG=12 DJANGO=2.2 - - PG=9.6 DJANGO=3.0 - - PG=10 DJANGO=3.0 - - PG=11 DJANGO=3.0 - - PG=12 DJANGO=3.0 - - PG=9.6 DJANGO=3.1 - - PG=10 DJANGO=3.1 - - PG=11 DJANGO=3.1 - - PG=12 DJANGO=3.1 - -before_install: - # Use default PostgreSQL 11 port - - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf - - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf - - - sudo sed -i 's/port = 5434/port = 5432/' /etc/postgresql/12/main/postgresql.conf - - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf - - # Start PostgreSQL version we need - - sudo systemctl stop postgresql - - sudo systemctl start postgresql@$PG-main - - # ClickHouse sources - - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 - - sudo apt-get update - - -install: - # Install ClickHouse - - sudo apt-get install clickhouse-client clickhouse-server clickhouse-common-static - - sudo service clickhouse-server restart - - - pip install -r requirements-test.txt - - pip install -q Django==$DJANGO.* - - python setup.py -q install - -before_script: - # Output software versions - - erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell - - rabbitmqctl status | grep "RabbitMQ" - - clickhouse-client --query "SELECT version();" - - psql -tc 'SHOW server_version' -U postgres - - - psql -tc 'SHOW server_version' -U postgres - - psql -c 'CREATE ROLE test;' -U postgres - - psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres - - psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres - - psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres - - psql -c 'CREATE DATABASE test OWNER test;' -U postgres - - psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres - -script: - python runtests.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c5a37b5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +ARG PYTHON_IMAGE_TAG=latest + +FROM python:${PYTHON_IMAGE_TAG} AS image_stage + +ARG APP_TAG="1.0.3" + +LABEL \ + org.label-schema.build-date=Now \ + org.label-schema.maintainer="m1ha@carrotquest.io" \ + org.label-schema.schema-version="1.0.0-rc1" \ + org.label-schema.vcs-ref="v${APP_TAG}" \ + org.label-schema.vcs-url="https://github.com/carrotquest/django-clickhouse" \ + org.label-schema.vendor="Carrot quest" \ + org.label-schema.version="${APP_TAG}" + +ENV APP_UID ${APP_UID:-1000} +ENV APP_GID ${APP_GID:-1000} +ENV APP_NAME ${APP_NAME:-"app"} + +# Configure utf-8 locales to make sure Python correctly handles unicode filenames +# Configure pip local path to copy data from pip_stage +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DJANGO_SETTINGS_MODULE=tests.settings PYTHONUSERBASE=/pip PATH=/pip/bin:$PATH + +RUN set -eu && \ + groupadd --gid "${APP_GID}" "app" && \ + useradd --uid ${APP_UID} --gid ${APP_GID} --create-home --shell /bin/bash -d /app app && \ + mkdir -p /pip && \ + chmod 755 /app /pip && \ + chown -R ${APP_UID}:${APP_GID} /app /pip + +WORKDIR /app/src + +# Install dependencies +# set -eu "breaks" pipeline on first error +COPY ./requirements-test.txt /app/requirements-test.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + set -eu && \ + python3 -m pip install --upgrade pip setuptools wheel && \ + python3 -m pip install --upgrade --requirement /app/requirements-test.txt + +COPY . /app/src + +RUN python3 setup.py -q install --user + +USER ${APP_UID} + +CMD ["python3", "runtests.py"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a0f992f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +--- +version: "3.9" +services: + redis_db: + image: redis + command: [sh, -c, "redis-server --save '' --appendonly no"] # disable persistence + mem_limit: 512m + cpus: 1 + + postgres_db: + image: postgres + environment: + - POSTGRES_PASSWORD=postgres + mem_limit: 1g + cpus: 1 + + clickhouse_db: + image: yandex/clickhouse-server + mem_limit: 1g + cpus: 1 + + run_tests: + image: django-clickhouse + build: + context: . + args: + - PYTHON_VER=latest + environment: + - REDIS_HOST=redis_db + - PGHOST=postgres_db + - PGUSER=postgres + - PGPASS=postgres + - "CLICK_HOUSE_HOST=http://clickhouse_db:8123" + depends_on: + - redis_db + - postgres_db + - clickhouse_db + mem_limit: 1g + cpus: 1 diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..f9f8c48 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,30 @@ +# Development +## Basic info +This is an Open source project developed by `Carrot quest` team under MIT license. +Feel free to create issues and make pull requests. +Query and database system wraps [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) library. +If you want to develop QuerySet system, it is better to contribute there. + + +## General info about testing +Library test system is based on [django.test](https://docs.djangoproject.com/en/3.2/topics/testing/overview/). +You can find them in `tests` directory. + +## Tests requirements +* [Redis](https://redis.io/) +* [Yandex ClickHouse](https://clickhouse.yandex/) +* [PostgreSQL](https://www.postgresql.org/) +* Pypi libraries listed in `requirements-test.txt` file + +## Running tests +### Running in docker +1. Install [docker and docker-compose](https://www.docker.com/) +2. Run `docker-compose run run_tests` in project directory + +### Running in virtual environment +1. Install all requirements listed above +2. [Create virtual environment](https://docs.python.org/3/tutorial/venv.html) +3. Install requirements + `pip3 install -U -r requirements-test.txt` +4. Start tests + `python3 runtests.py` diff --git a/docs/index.md b/docs/index.md index 8aedf21..8e5e46f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,3 +20,4 @@ * [RedisStorage](storages.md#redisstorage) * [Monitoring](monitoring.md) * [Performance notes](performance.md) +* [Development](development.md) \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index cf60e8d..a51322b 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -9,3 +9,6 @@ psycopg2-binary django-pg-returning django-pg-bulk-update redis + +# Linter +flake8 diff --git a/runtests.py b/runtests.py index 674b064..afb72fa 100644 --- a/runtests.py +++ b/runtests.py @@ -18,6 +18,6 @@ if __name__ == "__main__": os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings' django.setup() TestRunner = get_runner(settings) - test_runner = TestRunner() + test_runner = TestRunner(interactive=False) failures = test_runner.run_tests(["tests"]) sys.exit(bool(failures)) diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py index b4817f5..bd009e7 100644 --- a/src/django_clickhouse/clickhouse_models.py +++ b/src/django_clickhouse/clickhouse_models.py @@ -93,7 +93,7 @@ class ClickHouseModel(InfiModel, metaclass=ClickHouseModelMeta): return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values) @classmethod - def objects_in(cls, database: Database)-> QuerySet: + def objects_in(cls, database: Database) -> QuerySet: return QuerySet(cls, database) @classmethod @@ -120,7 +120,7 @@ class ClickHouseModel(InfiModel, metaclass=ClickHouseModelMeta): return connections[db_alias] @classmethod - def get_django_model_serializer(cls, writable: bool= False, defaults: Optional[dict] = None + def get_django_model_serializer(cls, writable: bool = False, defaults: Optional[dict] = None ) -> Django2ClickHouseModelSerializer: serializer_cls = lazy_class_import(cls.django_model_serializer) return serializer_cls(cls, writable=writable, defaults=defaults) diff --git a/src/django_clickhouse/compatibility.py b/src/django_clickhouse/compatibility.py index 2354d7a..55cadbe 100644 --- a/src/django_clickhouse/compatibility.py +++ b/src/django_clickhouse/compatibility.py @@ -28,7 +28,7 @@ def django_pg_returning_available(using: str) -> bool: :return: Boolean """ try: - import django_pg_returning + import django_pg_returning # noqa: F401 return connections[using].vendor == 'postgresql' except ImportError: return False diff --git a/src/django_clickhouse/database.py b/src/django_clickhouse/database.py index 2668015..9b79f11 100644 --- a/src/django_clickhouse/database.py +++ b/src/django_clickhouse/database.py @@ -1,5 +1,5 @@ import logging -from typing import Generator, Optional, Type, Iterable +from typing import Optional, Type, Iterable from infi.clickhouse_orm.database import Database as InfiDatabase, DatabaseException from infi.clickhouse_orm.utils import parse_tsv @@ -34,8 +34,8 @@ class Database(InfiDatabase): def _get_applied_migrations(self, migrations_package_name): raise NotImplementedError("This method is not supported by django_clickhouse.") - def select_tuples(self, query: str, model_class: Type['ClickHouseModel'], settings: Optional[dict] = None - ) -> Iterable[tuple]: + def select_tuples(self, query: str, model_class: Type['ClickHouseModel'], # noqa: F821 + settings: Optional[dict] = None) -> Iterable[tuple]: """ This method selects model_class namedtuples, instead of class instances. Less memory consumption, greater speed @@ -66,7 +66,7 @@ class Database(InfiDatabase): yield item - def insert_tuples(self, model_class: Type['ClickHouseModel'], model_tuples: Iterable[tuple], + def insert_tuples(self, model_class: Type['ClickHouseModel'], model_tuples: Iterable[tuple], # noqa: F821 batch_size: Optional[int] = None, formatted: bool = False) -> None: """ Inserts model_class namedtuples diff --git a/src/django_clickhouse/engines.py b/src/django_clickhouse/engines.py index 55e1647..8d467dc 100644 --- a/src/django_clickhouse/engines.py +++ b/src/django_clickhouse/engines.py @@ -2,19 +2,20 @@ This file contains wrappers for infi.clckhouse_orm engines to use in django-clickhouse """ import datetime -from typing import List, Type, Union, Iterable, Generator, Optional +from typing import List, Type, Union, Iterable, Optional from django.db.models import Model as DjangoModel from infi.clickhouse_orm import engines as infi_engines from statsd.defaults.django import statsd +from .clickhouse_models import ClickHouseModel from .configuration import config from .database import connections from .utils import format_datetime class InsertOnlyEngineMixin: - def get_insert_batch(self, model_cls: Type['ClickHouseModel'], objects: List[DjangoModel]) -> Iterable[tuple]: + def get_insert_batch(self, model_cls: Type[ClickHouseModel], objects: List[DjangoModel]) -> Iterable[tuple]: """ Gets a list of model_cls instances to insert into database :param model_cls: ClickHouseModel subclass to import @@ -47,9 +48,9 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre def _get_final_versions_by_version(self, db_alias, model_cls, min_date, max_date, object_pks, date_col, columns): query = """ SELECT {columns} FROM $table WHERE (`{pk_column}`, `{version_col}`) IN ( - SELECT `{pk_column}`, MAX(`{version_col}`) - FROM $table - PREWHERE `{date_col}` >= '{min_date}' AND `{date_col}` <= '{max_date}' + SELECT `{pk_column}`, MAX(`{version_col}`) + FROM $table + PREWHERE `{date_col}` >= '{min_date}' AND `{date_col}` <= '{max_date}' AND `{pk_column}` IN ({object_pks}) GROUP BY `{pk_column}` ) @@ -68,7 +69,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre max_date=max_date, object_pks=','.join(object_pks)) return connections[db_alias].select_tuples(query, model_cls) - def get_final_versions(self, model_cls: Type['ClickHouseModel'], objects: Iterable[DjangoModel], + def get_final_versions(self, model_cls: Type[ClickHouseModel], objects: Iterable[DjangoModel], date_col: Optional[str] = None) -> Iterable[tuple]: """ Get objects, that are currently stored in ClickHouse. @@ -122,7 +123,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre else: return self._get_final_versions_by_final(*params) - def get_insert_batch(self, model_cls: Type['ClickHouseModel'], objects: List[DjangoModel]) -> Iterable[tuple]: + def get_insert_batch(self, model_cls: Type[ClickHouseModel], objects: List[DjangoModel]) -> Iterable[tuple]: """ Gets a list of model_cls instances to insert into database :param model_cls: ClickHouseModel subclass to import diff --git a/src/django_clickhouse/migrations.py b/src/django_clickhouse/migrations.py index 9d47688..a5bcd5f 100644 --- a/src/django_clickhouse/migrations.py +++ b/src/django_clickhouse/migrations.py @@ -9,7 +9,7 @@ from django.db.models.signals import post_migrate from django.dispatch import receiver # In order to support all operations import here -from infi.clickhouse_orm.migrations import * +from infi.clickhouse_orm.migrations import * # noqa F401, F403 from infi.clickhouse_orm.database import ServerError, DatabaseException from infi.clickhouse_orm.fields import StringField, DateField diff --git a/src/django_clickhouse/models.py b/src/django_clickhouse/models.py index e7a5ec1..cd7c8ec 100644 --- a/src/django_clickhouse/models.py +++ b/src/django_clickhouse/models.py @@ -154,7 +154,7 @@ class ClickHouseSyncModel(DjangoModel): return storage_cls() @classmethod - def register_clickhouse_sync_model(cls, model_cls: Type['ClickHouseModel']) -> None: + def register_clickhouse_sync_model(cls, model_cls: Type['ClickHouseModel']) -> None: # noqa: F821 """ Registers ClickHouse model to listen to this model updates :param model_cls: Model class to register @@ -166,7 +166,7 @@ class ClickHouseSyncModel(DjangoModel): cls._clickhouse_sync_models.add(model_cls) @classmethod - def get_clickhouse_sync_models(cls) -> Set['ClickHouseModel']: + def get_clickhouse_sync_models(cls) -> Set['ClickHouseModel']: # noqa: F821 """ Returns all clickhouse models, listening to this class :return: A set of model classes to sync diff --git a/src/django_clickhouse/serializers.py b/src/django_clickhouse/serializers.py index 4b3907e..490039a 100644 --- a/src/django_clickhouse/serializers.py +++ b/src/django_clickhouse/serializers.py @@ -7,7 +7,7 @@ from .utils import model_to_dict class Django2ClickHouseModelSerializer: - def __init__(self, model_cls: Type['ClickHouseModel'], fields: Optional[Iterable[str]] = None, + def __init__(self, model_cls: Type['ClickHouseModel'], fields: Optional[Iterable[str]] = None, # noqa: F821 exclude_fields: Optional[Iterable[str]] = None, writable: bool = False, defaults: Optional[dict] = None) -> None: """ diff --git a/src/django_clickhouse/storages.py b/src/django_clickhouse/storages.py index cf2ba4f..63115ae 100644 --- a/src/django_clickhouse/storages.py +++ b/src/django_clickhouse/storages.py @@ -82,7 +82,7 @@ class Storage: :param kwargs: Storage dependant arguments :return: Number of records in queue """ - raise NotImplemented() + raise NotImplementedError() def get_operations(self, import_key: str, count: int, **kwargs) -> List[Tuple[str, str]]: """ @@ -93,7 +93,7 @@ class Storage: :param kwargs: Storage dependant arguments :return: A list of tuples (operation, pk) in incoming order. """ - raise NotImplemented() + raise NotImplementedError() def register_operations(self, import_key: str, operation: str, *pks: Any) -> int: """ @@ -134,21 +134,21 @@ class Storage: This method is used in tests to drop all storage data :return: None """ - raise NotImplemented() + raise NotImplementedError() def get_last_sync_time(self, import_key: str) -> Optional[datetime.datetime]: """ Gets the last time, sync has been executed :return: datetime.datetime if last sync has been. Otherwise - None. """ - raise NotImplemented() + raise NotImplementedError() def set_last_sync_time(self, import_key: str, dt: datetime.datetime) -> None: """ Sets successful sync time :return: None """ - raise NotImplemented() + raise NotImplementedError() class RedisStorage(Storage, metaclass=SingletonMeta): diff --git a/src/django_clickhouse/utils.py b/src/django_clickhouse/utils.py index a580d0d..74588a9 100644 --- a/src/django_clickhouse/utils.py +++ b/src/django_clickhouse/utils.py @@ -287,4 +287,4 @@ class SingletonMeta(type): def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(SingletonMeta, cls).__call__(*args, **kwargs) - return cls._instances[cls] \ No newline at end of file + return cls._instances[cls] diff --git a/tests/kill_test_sub_process.py b/tests/kill_test_sub_process.py index 642d638..36f9385 100644 --- a/tests/kill_test_sub_process.py +++ b/tests/kill_test_sub_process.py @@ -15,9 +15,9 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'tests.settings') django.setup() # This imports must be after django activation -from django.db.models import F -from tests.clickhouse_models import ClickHouseCollapseTestModel -from tests.models import TestModel +from django.db.models import F # noqa: E402 +from tests.clickhouse_models import ClickHouseCollapseTestModel # noqa: E402 +from tests.models import TestModel # noqa: E402 logger = logging.getLogger('django-clickhouse') diff --git a/tests/settings.py b/tests/settings.py index 5c4f558..1382d38 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -1,36 +1,39 @@ """ This file contains django settings to run tests with runtests.py """ +from os import environ + SECRET_KEY = 'fake-key' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'test', - 'USER': 'test', - 'PASSWORD': 'test', - 'HOST': '127.0.0.1', - 'PORT': '5432' + 'USER': environ.get('PGUSER', 'test'), + 'PASSWORD': environ.get('PGPASS', 'test'), + 'HOST': environ.get('PGHOST', '127.0.0.1'), + 'PORT': environ.get('PGPORT', 5432) }, 'secondary': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'test2', - 'USER': 'test', - 'PASSWORD': 'test', - 'HOST': '127.0.0.1', - 'PORT': '5432' + 'USER': environ.get('PGUSER', 'test'), + 'PASSWORD': environ.get('PGPASS', 'test'), + 'HOST': environ.get('PGHOST', '127.0.0.1'), + 'PORT': environ.get('PGPORT', 5432) }, # I need separate connections for multiprocessing tests 'test_db': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'test_test', - 'USER': 'test', - 'PASSWORD': 'test', - 'HOST': '127.0.0.1', - 'PORT': '5432' + 'USER': environ.get('PGUSER', 'test'), + 'PASSWORD': environ.get('PGPASS', 'test'), + 'HOST': environ.get('PGHOST', '127.0.0.1'), + 'PORT': environ.get('PGPORT', 5432) }, } +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' LOGGING = { 'version': 1, @@ -58,22 +61,26 @@ INSTALLED_APPS = [ CLICKHOUSE_DATABASES = { 'default': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test', 'username': 'default', 'password': '' }, 'secondary': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test_2', 'username': 'default', 'password': '' }, 'no_migrate': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test_3', 'username': 'default', 'password': '', 'migrate': False }, 'readonly': { + 'db_url': environ.get('CLICK_HOUSE_HOST', 'http://localhost:8123/'), 'db_name': 'test_3', 'username': 'default', 'password': '', @@ -84,8 +91,8 @@ CLICKHOUSE_DATABASES = { CLICKHOUSE_SYNC_BATCH_SIZE = 5000 CLICKHOUSE_REDIS_CONFIG = { - 'host': '127.0.0.1', - 'port': 6379, + 'host': environ.get('REDIS_HOST', '127.0.0.1'), + 'port': environ.get('REDIS_PORT', 6379), 'db': 8, 'socket_timeout': 10 } diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index c369277..3a5c434 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -48,5 +48,3 @@ class NamedTupleTest(TestCase): t2 = TestTuple(1, 2, 3) self.assertEqual(t1, t2) self.assertEqual((1, 2, 3), t1) - - diff --git a/tests/test_models.py b/tests/test_models.py index b89957f..cf378f2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,7 +2,7 @@ import datetime from unittest import skipIf import django -from django.test import TransactionTestCase, TestCase +from django.test import TransactionTestCase from django.utils.timezone import now from tests.clickhouse_models import ClickHouseTestModel, ClickHouseSecondTestModel, ClickHouseCollapseTestModel, \ diff --git a/tests/test_storages.py b/tests/test_storages.py index 7e74ba0..549c4af 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -12,7 +12,7 @@ class StorageTest(TestCase): def setUp(self): self.storage.flush() - + def tearDown(self): self.storage.flush() @@ -75,11 +75,10 @@ class StorageTest(TestCase): def test_locks(self): # Test that multiple can acquire locks in parallel # And single model can't - l = self.storage.get_lock(ClickHouseTestModel.get_import_key()) - l.acquire() + lock = self.storage.get_lock(ClickHouseTestModel.get_import_key()) + lock.acquire() with self.assertRaises(RedisLockTimeoutError): - l.acquire() - - l2 = self.storage.get_lock(ClickHouseCollapseTestModel.get_import_key()) - l2.acquire() + lock.acquire() + lock_2 = self.storage.get_lock(ClickHouseCollapseTestModel.get_import_key()) + lock_2.acquire() diff --git a/tests/test_utils.py b/tests/test_utils.py index c6f5eca..85ec154 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -118,4 +118,3 @@ class TestSingletonMeta(TestCase): b = Single() self.assertEqual(a, b) self.assertEqual(2, b.test) -