From 96364b54183af6865a1eef511bc1a881a694e264 Mon Sep 17 00:00:00 2001 From: sw <935405794@qq.com> Date: Fri, 3 Jun 2022 14:23:49 +0800 Subject: [PATCH] test: workflows --- .github/workflows/unittest.yml | 42 ++++++++++++++++++++++++++++++ pyproject.toml | 7 ++++- src/clickhouse_orm/aio/database.py | 6 +++-- src/clickhouse_orm/database.py | 6 +++-- src/clickhouse_orm/engines.py | 4 +-- src/clickhouse_orm/models.py | 12 ++++----- src/clickhouse_orm/query.py | 15 +++++++++++ tests/test_buffer.py | 1 - tests/test_database.py | 12 ++++----- tests/test_funcs.py | 19 +++++++++----- tests/test_mutations.py | 7 ++--- tests/test_querysets.py | 12 ++++----- 12 files changed, 107 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/unittest.yml diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 0000000..d0cf682 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,42 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + - develop + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: ch_orm + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Pull images + run: | + docker pull clickhouse/clickhouse-server + docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server + + - name: Build and Install + run: | + pip install build + python -m build + pip install dist/* + + - name: UnitTest + run: | + python -m unittest diff --git a/pyproject.toml b/pyproject.toml index c7ebe86..16dc35c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,8 +8,13 @@ description = "A Python library for working with the ClickHouse database" readme = "README.md" keywords = ["ClickHouse", "ORM", 'DB', 'DATABASE', 'OLAP'] license = {text = "BSD License"} +homepage = "https://github.com/sswest/ch-orm" classifiers = [ "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Database" @@ -20,7 +25,7 @@ dependencies = [ "iso8601 >= 0.1.12", "setuptools" ] -version = "0.1.0" +version = "0.1.1" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/clickhouse_orm/aio/database.py b/src/clickhouse_orm/aio/database.py index 652532e..2498314 100644 --- a/src/clickhouse_orm/aio/database.py +++ b/src/clickhouse_orm/aio/database.py @@ -45,7 +45,7 @@ class AioDatabase(Database): async def _send( self, - data: str | bytes, + data: str | bytes | AsyncGenerator, settings: dict = None, stream: bool = False ): @@ -255,8 +255,10 @@ class AioDatabase(Database): field_types = parse_tsv(line) model_class = model_class or ModelBase.create_ad_hoc_model( zip(field_names, field_types)) - elif line: + elif line.strip(): yield model_class.from_tsv(line, field_names, self.server_timezone, self) + except StopIteration: + return finally: await r.aclose() diff --git a/src/clickhouse_orm/database.py b/src/clickhouse_orm/database.py index 4bfca8d..9921016 100644 --- a/src/clickhouse_orm/database.py +++ b/src/clickhouse_orm/database.py @@ -320,8 +320,10 @@ class Database: model_class = ModelBase.create_ad_hoc_model(zip(field_names, field_types)) for line in lines: # skip blank line left by WITH TOTALS modifier - if line: + if line.strip(): yield model_class.from_tsv(line, field_names, self.server_timezone, self) + except StopIteration: + return finally: r.close() @@ -432,7 +434,7 @@ class Database: def _send( self, - data: str | bytes, + data: str | bytes | Generator, settings: dict = None, stream: bool = False ): diff --git a/src/clickhouse_orm/engines.py b/src/clickhouse_orm/engines.py index 3c39751..7996ca5 100644 --- a/src/clickhouse_orm/engines.py +++ b/src/clickhouse_orm/engines.py @@ -280,11 +280,11 @@ class Distributed(Engine): @property def table_name(self) -> str: - from clickhouse_orm.models import Model + from clickhouse_orm.models import ModelBase table = self.table - if isinstance(table, Model): + if isinstance(table, ModelBase): return table.table_name() return table diff --git a/src/clickhouse_orm/models.py b/src/clickhouse_orm/models.py index d73c646..e0a2636 100644 --- a/src/clickhouse_orm/models.py +++ b/src/clickhouse_orm/models.py @@ -234,14 +234,14 @@ class ModelBase(type): ) # Arrays if db_type.startswith('Array'): - inner_field = cls.create_ad_hoc_field(db_type[6 : -1]) + inner_field = cls.create_ad_hoc_field(db_type[6:-1]) return orm_fields.ArrayField(inner_field) - # Tuples (poor man's version - convert to array) + # Tuples if db_type.startswith('Tuple'): - types = [s.strip() for s in db_type[6 : -1].split(',')] - assert len(set(types)) == 1, 'No support for mixed types in tuples - ' + db_type - inner_field = cls.create_ad_hoc_field(types[0]) - return orm_fields.ArrayField(inner_field) + types = [s.strip() for s in db_type[6:-1].split(',')] + return orm_fields.TupleField(name_fields=[ + (str(i), cls.create_ad_hoc_field(type_name)) for i, type_name in enumerate(types)] + ) # FixedString if db_type.startswith('FixedString'): length = int(db_type[12:-1]) diff --git a/src/clickhouse_orm/query.py b/src/clickhouse_orm/query.py index 54f0b1f..5912500 100644 --- a/src/clickhouse_orm/query.py +++ b/src/clickhouse_orm/query.py @@ -360,6 +360,9 @@ class QuerySet(Generic[MODEL]): queryset._final = self._final return queryset + def __deepcopy__(self, memodict={}): + return self._clone() + def __iter__(self) -> Iterator[MODEL]: """ Iterates over the model instances matching this queryset @@ -712,6 +715,18 @@ class AggregateQuerySet(QuerySet[MODEL]): self._limits = base_queryset._limits self._distinct = base_queryset._distinct + def _clone(self) -> "AggregateQuerySet[MODEL]": + queryset = copy(self) + queryset._fields = copy(self._fields) + queryset._grouping_fields = copy(self._grouping_fields) + queryset._calculated_fields = copy(self._calculated_fields) + queryset._order_by = copy(self._order_by) + queryset._where_q = copy(self._where_q) + queryset._prewhere_q = copy(self._prewhere_q) + queryset._limits = copy(self._limits) + queryset._distinct = copy(self._distinct) + return queryset + def group_by(self, *args) -> "AggregateQuerySet[MODEL]": """ This method lets you specify the grouping fields explicitly. The `args` must diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 71dafea..dbbe98d 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -2,7 +2,6 @@ import unittest from clickhouse_orm.models import BufferModel -from clickhouse_orm.engines import * from .base_test_with_data import * diff --git a/tests/test_database.py b/tests/test_database.py index 8958ff4..a95c0c6 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -3,10 +3,6 @@ import unittest import datetime from clickhouse_orm.database import ServerError, DatabaseException -from clickhouse_orm.models import Model -from clickhouse_orm.engines import Memory -from clickhouse_orm.fields import * -from clickhouse_orm.funcs import F from clickhouse_orm.query import Q from .base_test_with_data import * @@ -190,7 +186,7 @@ class DatabaseTestCase(TestCaseWithData): raise Exception('Unexpected error code - %s %s' % (exc.code, exc.message)) def test_nonexisting_db(self): - db = Database('db_not_here', autocreate=False) + db = Database('db_not_here', auto_create=False) with self.assertRaises(ServerError) as cm: db.create_table(Person) exc = cm.exception @@ -207,7 +203,7 @@ class DatabaseTestCase(TestCaseWithData): self.assertFalse(db.db_exists) def test_preexisting_db(self): - db = Database(self.database.db_name, autocreate=False) + db = Database(self.database.db_name, auto_create=False) db.count(Person) def test_missing_engine(self): @@ -289,5 +285,9 @@ class DatabaseTestCase(TestCaseWithData): except ServerError as e: if 'Not enough privileges' in e.message: pass + elif 'no certificate file has been specified' in e.message: + pass + elif 'table must contain condition' in e.message: + pass else: raise diff --git a/tests/test_funcs.py b/tests/test_funcs.py index fb250e6..077e010 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -40,7 +40,7 @@ class FuncsTestCase(TestCaseWithData): except ServerError as e: if 'Unknown function' in e.message: logging.warning(e.message) - return # ignore functions that don't exist in the used ClickHouse version + return # ignore functions that don't exist in the used ClickHouse version raise def _test_aggr(self, func, expected_value=NO_VALUE): @@ -615,14 +615,21 @@ class FuncsTestCase(TestCaseWithData): self._test_func(F.IPv4NumToString(F.toUInt32(1)), '0.0.0.1') self._test_func(F.IPv4NumToStringClassC(F.toUInt32(1)), '0.0.0.xxx') self._test_func(F.IPv4StringToNum('0.0.0.17'), 17) - self._test_func(F.IPv6NumToString(F.IPv4ToIPv6(F.IPv4StringToNum('192.168.0.1'))), '::ffff:192.168.0.1') + self._test_func( + F.IPv6NumToString(F.IPv4ToIPv6(F.IPv4StringToNum('192.168.0.1'))), + '::ffff:192.168.0.1' + ) self._test_func(F.IPv6NumToString(F.IPv6StringToNum('2a02:6b8::11')), '2a02:6b8::11') self._test_func(F.toIPv4('10.20.30.40'), IPv4Address('10.20.30.40')) - self._test_func(F.toIPv6('2001:438:ffff::407d:1bc1'), IPv6Address('2001:438:ffff::407d:1bc1')) + self._test_func( + F.toIPv6('2001:438:ffff::407d:1bc1'), IPv6Address('2001:438:ffff::407d:1bc1') + ) self._test_func(F.IPv4CIDRToRange(F.toIPv4('192.168.5.2'), 16), - [IPv4Address('192.168.0.0'), IPv4Address('192.168.255.255')]) - self._test_func(F.IPv6CIDRToRange(F.toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32), - [IPv6Address('2001:db8::'), IPv6Address('2001:db8:ffff:ffff:ffff:ffff:ffff:ffff')]) + (IPv4Address('192.168.0.0'), IPv4Address('192.168.255.255'))) + self._test_func( + F.IPv6CIDRToRange(F.toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32), + (IPv6Address('2001:db8::'), IPv6Address('2001:db8:ffff:ffff:ffff:ffff:ffff:ffff')) + ) def test_aggregate_funcs(self): self._test_aggr(F.any(Person.first_name)) diff --git a/tests/test_mutations.py b/tests/test_mutations.py index ec291b9..8ff9fd2 100644 --- a/tests/test_mutations.py +++ b/tests/test_mutations.py @@ -1,8 +1,9 @@ -import unittest -from clickhouse_orm import F -from .base_test_with_data import * from time import sleep +from clickhouse_orm.funcs import F + +from .base_test_with_data import * + class MutationsTestCase(TestCaseWithData): diff --git a/tests/test_querysets.py b/tests/test_querysets.py index ec81c6a..0c96760 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- -import unittest -from clickhouse_orm.database import Database -from clickhouse_orm.query import Q -from clickhouse_orm.funcs import F -from .base_test_with_data import * from datetime import date, datetime from enum import Enum -from decimal import Decimal + +from clickhouse_orm.query import Q +from clickhouse_orm.funcs import F + +from .base_test_with_data import * from logging import getLogger logger = getLogger('tests') - class QuerySetTestCase(TestCaseWithData): def setUp(self):