From 97f776792fb9e3a3e156bbaadb5c6eb0c853e5e6 Mon Sep 17 00:00:00 2001 From: "Evgeni (Gene) Makarov" Date: Mon, 14 Dec 2020 14:36:21 +0300 Subject: [PATCH 01/11] initializing changes related to string enums for pull request From a8ab206849194c0d1ebfd91abb3a905921bfe5f7 Mon Sep 17 00:00:00 2001 From: "Evgeni (Gene) Makarov" Date: Mon, 14 Dec 2020 14:39:36 +0300 Subject: [PATCH 02/11] changes reverted after rebase --- src/infi/clickhouse_orm/fields.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index b63997e..4f631cc 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -468,9 +468,16 @@ class BaseEnumField(Field): return value try: if isinstance(value, str): - return self.enum_cls[value] + try: + return self.enum_cls[value] + except Exception: + return self.enum_cls(value) if isinstance(value, bytes): - return self.enum_cls[value.decode('UTF-8')] + decoded = value.decode('UTF-8') + try: + return self.enum_cls[decoded] + except Exception: + return self.enum_cls(decoded) if isinstance(value, int): return self.enum_cls(value) except (KeyError, ValueError): @@ -665,4 +672,3 @@ class LowCardinalityField(Field): # Expose only relevant classes in import * __all__ = get_subclass_names(locals(), Field) - From 45d807eb02efff1d04c958d9025ce6982562e045 Mon Sep 17 00:00:00 2001 From: "k.peskov" Date: Mon, 18 Jan 2021 22:43:00 +0300 Subject: [PATCH 03/11] 1. add stddevPop func 2. add stddevSamp func --- src/infi/clickhouse_orm/funcs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/infi/clickhouse_orm/funcs.py b/src/infi/clickhouse_orm/funcs.py index d84c761..951dce5 100644 --- a/src/infi/clickhouse_orm/funcs.py +++ b/src/infi/clickhouse_orm/funcs.py @@ -1633,6 +1633,16 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): def varSamp(x): return F('varSamp', x) + @staticmethod + @aggregate + def stddevPop(expr): + return F('stddevPop', expr) + + @staticmethod + @aggregate + def stddevSamp(expr): + return F('stddevSamp', expr) + @staticmethod @aggregate @parametric From e14de715f7ac2709f7fb6f34ad5c88a2687fa49b Mon Sep 17 00:00:00 2001 From: meanmail Date: Wed, 3 Mar 2021 21:04:54 +0700 Subject: [PATCH 04/11] Support for adding a column to the beginning of a table --- src/infi/clickhouse_orm/migrations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/infi/clickhouse_orm/migrations.py b/src/infi/clickhouse_orm/migrations.py index c8c656a..6fea1c0 100644 --- a/src/infi/clickhouse_orm/migrations.py +++ b/src/infi/clickhouse_orm/migrations.py @@ -83,10 +83,12 @@ class AlterTable(ModelOperation): is_regular_field = not (field.materialized or field.alias) if name not in table_fields: logger.info(' Add column %s', name) - assert prev_name, 'Cannot add a column to the beginning of the table' cmd = 'ADD COLUMN %s %s' % (name, field.get_sql(db=database)) if is_regular_field: - cmd += ' AFTER %s' % prev_name + if prev_name is not None: + cmd += ' AFTER %s' % prev_name + else: + cmd += ' FIRST' self._alter_table(database, cmd) if is_regular_field: From 6a5889280d7072ff7e57ad8bbc8f33baeac0568c Mon Sep 17 00:00:00 2001 From: meanmail Date: Thu, 4 Mar 2021 11:28:27 +0700 Subject: [PATCH 05/11] Simplified --- src/infi/clickhouse_orm/migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/infi/clickhouse_orm/migrations.py b/src/infi/clickhouse_orm/migrations.py index 6fea1c0..af7dc51 100644 --- a/src/infi/clickhouse_orm/migrations.py +++ b/src/infi/clickhouse_orm/migrations.py @@ -85,7 +85,7 @@ class AlterTable(ModelOperation): logger.info(' Add column %s', name) cmd = 'ADD COLUMN %s %s' % (name, field.get_sql(db=database)) if is_regular_field: - if prev_name is not None: + if prev_name: cmd += ' AFTER %s' % prev_name else: cmd += ' FIRST' From fbf2207690f4f893500abba62d367d2258522645 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Mar 2021 04:43:18 +0000 Subject: [PATCH 06/11] Bump jinja2 from 2.11.2 to 2.11.3 in /examples/db_explorer Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.2 to 2.11.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.11.2...2.11.3) Signed-off-by: dependabot[bot] --- examples/db_explorer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/db_explorer/requirements.txt b/examples/db_explorer/requirements.txt index 8dee9f8..4710051 100644 --- a/examples/db_explorer/requirements.txt +++ b/examples/db_explorer/requirements.txt @@ -6,7 +6,7 @@ idna==2.9 infi.clickhouse-orm==2.0.1 iso8601==0.1.12 itsdangerous==1.1.0 -Jinja2==2.11.2 +Jinja2==2.11.3 MarkupSafe==1.1.1 pygal==2.4.0 pytz==2020.1 From 0f4547be879684e3e529bf99806fda574a71aa21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jun 2021 02:29:29 +0000 Subject: [PATCH 07/11] Bump urllib3 from 1.25.9 to 1.26.5 in /examples/db_explorer Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.9 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.25.9...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- examples/db_explorer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/db_explorer/requirements.txt b/examples/db_explorer/requirements.txt index 8dee9f8..6a02075 100644 --- a/examples/db_explorer/requirements.txt +++ b/examples/db_explorer/requirements.txt @@ -11,5 +11,5 @@ MarkupSafe==1.1.1 pygal==2.4.0 pytz==2020.1 requests==2.23.0 -urllib3==1.25.9 +urllib3==1.26.5 Werkzeug==1.0.1 From 5abe39ed2443b2811df3df69273b5878277c26ad Mon Sep 17 00:00:00 2001 From: mangototango Date: Sat, 16 Oct 2021 07:47:17 +0000 Subject: [PATCH 08/11] Fixes to make the tests pass on ClickHouse v21.9 --- src/infi/clickhouse_orm/database.py | 5 +++++ src/infi/clickhouse_orm/fields.py | 2 +- src/infi/clickhouse_orm/funcs.py | 34 ++++++++++++++--------------- tests/test_compressed_fields.py | 4 ++-- tests/test_database.py | 12 ++++++++-- tests/test_dictionaries.py | 2 +- tests/test_engines.py | 8 +++---- 7 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index d42e224..37c6bf5 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -51,6 +51,11 @@ class ServerError(DatabaseException): Code:\ (?P\d+), \ e\.displayText\(\)\ =\ (?P[^ \n]+):\ (?P.+) ''', re.VERBOSE | re.DOTALL), + # ClickHouse v21+ + re.compile(r''' + Code:\ (?P\d+). + \ (?P[^ \n]+):\ (?P.+) + ''', re.VERBOSE | re.DOTALL), ) @classmethod diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index 4f631cc..6e73e3f 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -109,7 +109,7 @@ class Field(FunctionOperatorsMixin): elif self.default: default = self.to_db_string(self.default) sql += ' DEFAULT %s' % default - if self.codec and db and db.has_codec_support: + if self.codec and db and db.has_codec_support and not self.alias: sql += ' CODEC(%s)' % self.codec return sql diff --git a/src/infi/clickhouse_orm/funcs.py b/src/infi/clickhouse_orm/funcs.py index d84c761..e2ded48 100644 --- a/src/infi/clickhouse_orm/funcs.py +++ b/src/infi/clickhouse_orm/funcs.py @@ -391,11 +391,11 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toYear', d) @staticmethod - def toISOYear(d, timezone=''): + def toISOYear(d, timezone=NO_VALUE): return F('toISOYear', d, timezone) @staticmethod - def toQuarter(d, timezone=''): + def toQuarter(d, timezone=NO_VALUE): return F('toQuarter', d, timezone) if timezone else F('toQuarter', d) @staticmethod @@ -403,11 +403,11 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toMonth', d) @staticmethod - def toWeek(d, mode=0, timezone=''): + def toWeek(d, mode=0, timezone=NO_VALUE): return F('toWeek', d, mode, timezone) @staticmethod - def toISOWeek(d, timezone=''): + def toISOWeek(d, timezone=NO_VALUE): return F('toISOWeek', d, timezone) if timezone else F('toISOWeek', d) @staticmethod @@ -483,7 +483,7 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toStartOfDay', d) @staticmethod - def toTime(d, timezone=''): + def toTime(d, timezone=NO_VALUE): return F('toTime', d, timezone) @staticmethod @@ -491,47 +491,47 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toTimeZone', dt, timezone) @staticmethod - def toUnixTimestamp(dt, timezone=''): + def toUnixTimestamp(dt, timezone=NO_VALUE): return F('toUnixTimestamp', dt, timezone) @staticmethod - def toYYYYMM(dt, timezone=''): + def toYYYYMM(dt, timezone=NO_VALUE): return F('toYYYYMM', dt, timezone) if timezone else F('toYYYYMM', dt) @staticmethod - def toYYYYMMDD(dt, timezone=''): + def toYYYYMMDD(dt, timezone=NO_VALUE): return F('toYYYYMMDD', dt, timezone) if timezone else F('toYYYYMMDD', dt) @staticmethod - def toYYYYMMDDhhmmss(dt, timezone=''): + def toYYYYMMDDhhmmss(dt, timezone=NO_VALUE): return F('toYYYYMMDDhhmmss', dt, timezone) if timezone else F('toYYYYMMDDhhmmss', dt) @staticmethod - def toRelativeYearNum(d, timezone=''): + def toRelativeYearNum(d, timezone=NO_VALUE): return F('toRelativeYearNum', d, timezone) @staticmethod - def toRelativeMonthNum(d, timezone=''): + def toRelativeMonthNum(d, timezone=NO_VALUE): return F('toRelativeMonthNum', d, timezone) @staticmethod - def toRelativeWeekNum(d, timezone=''): + def toRelativeWeekNum(d, timezone=NO_VALUE): return F('toRelativeWeekNum', d, timezone) @staticmethod - def toRelativeDayNum(d, timezone=''): + def toRelativeDayNum(d, timezone=NO_VALUE): return F('toRelativeDayNum', d, timezone) @staticmethod - def toRelativeHourNum(d, timezone=''): + def toRelativeHourNum(d, timezone=NO_VALUE): return F('toRelativeHourNum', d, timezone) @staticmethod - def toRelativeMinuteNum(d, timezone=''): + def toRelativeMinuteNum(d, timezone=NO_VALUE): return F('toRelativeMinuteNum', d, timezone) @staticmethod - def toRelativeSecondNum(d, timezone=''): + def toRelativeSecondNum(d, timezone=NO_VALUE): return F('toRelativeSecondNum', d, timezone) @staticmethod @@ -555,7 +555,7 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('timeSlots', start_time, F.toUInt32(duration)) @staticmethod - def formatDateTime(d, format, timezone=''): + def formatDateTime(d, format, timezone=NO_VALUE): return F('formatDateTime', d, format, timezone) @staticmethod diff --git a/tests/test_compressed_fields.py b/tests/test_compressed_fields.py index 8d17571..440e0b3 100644 --- a/tests/test_compressed_fields.py +++ b/tests/test_compressed_fields.py @@ -106,7 +106,7 @@ class CompressedFieldsTestCase(unittest.TestCase): ('nullable_field', 'CODEC(ZSTD(1))'), ('array_field', 'CODEC(Delta(2), LZ4HC(0))'), ('float_field', 'CODEC(NONE)'), - ('alias_field', 'CODEC(ZSTD(4))')]) + ('alias_field', '')]) class CompressedModel(Model): @@ -120,4 +120,4 @@ class CompressedModel(Model): float_field = Float32Field(codec='NONE') alias_field = Float32Field(alias='float_field', codec='ZSTD(4)') - engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) \ No newline at end of file + engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) diff --git a/tests/test_database.py b/tests/test_database.py index 38681d4..44971e1 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -181,12 +181,13 @@ class DatabaseTestCase(TestCaseWithData): Database(self.database.db_name, username='default', password='wrong') exc = cm.exception + print(exc.code, exc.message) if exc.code == 193: # ClickHouse version < 20.3 self.assertTrue(exc.message.startswith('Wrong password for user default')) elif exc.code == 516: # ClickHouse version >= 20.3 self.assertTrue(exc.message.startswith('default: Authentication failed')) else: - raise Exception('Unexpected error code - %s' % exc.code) + raise Exception('Unexpected error code - %s %s' % (exc.code, exc.message)) def test_nonexisting_db(self): db = Database('db_not_here', autocreate=False) @@ -251,6 +252,8 @@ class DatabaseTestCase(TestCaseWithData): from infi.clickhouse_orm.models import ModelBase query = "SELECT DISTINCT type FROM system.columns" for row in self.database.select(query): + if row.type.startswith('Map'): + continue # Not supported yet ModelBase.create_ad_hoc_field(row.type) def test_get_model_for_table(self): @@ -271,7 +274,12 @@ class DatabaseTestCase(TestCaseWithData): query = "SELECT name FROM system.tables WHERE database='system'" for row in self.database.select(query): print(row.name) - model = self.database.get_model_for_table(row.name, system_table=True) + if row.name in ('distributed_ddl_queue',): + continue # Not supported + try: + model = self.database.get_model_for_table(row.name, system_table=True) + except NotImplementedError: + continue # Table contains an unsupported field type self.assertTrue(model.is_system_model()) self.assertTrue(model.is_read_only()) self.assertEqual(model.table_name(), row.name) diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 7da4160..c031d2e 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -105,7 +105,7 @@ class HierarchicalDictionaryTest(DictionaryTestMixin, unittest.TestCase): def test_dictgethierarchy(self): self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(3)), [3, 2, 1]) - self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(99)), [99]) + self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(99)), []) def test_dictisin(self): self._test_func(F.dictIsIn(self.dict_name, F.toUInt64(3), F.toUInt64(1)), 1) diff --git a/tests/test_engines.py b/tests/test_engines.py index 2fcc8c2..b2e0e5d 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -17,10 +17,10 @@ class _EnginesHelperTestCase(unittest.TestCase): class EnginesTestCase(_EnginesHelperTestCase): - def _create_and_insert(self, model_class): + def _create_and_insert(self, model_class, **kwargs): self.database.create_table(model_class) self.database.insert([ - model_class(date='2017-01-01', event_id=23423, event_group=13, event_count=7, event_version=1) + model_class(date='2017-01-01', event_id=23423, event_group=13, event_count=7, event_version=1, **kwargs) ]) def test_merge_tree(self): @@ -155,7 +155,7 @@ class EnginesTestCase(_EnginesHelperTestCase): ) self._create_and_insert(TestModel) - self._create_and_insert(TestCollapseModel) + self._create_and_insert(TestCollapseModel, sign=1) # Result order may be different, lets sort manually parts = sorted(list(SystemPart.get(self.database)), key=lambda x: x.table) @@ -188,7 +188,7 @@ class EnginesTestCase(_EnginesHelperTestCase): ) self._create_and_insert(TestModel) - self._create_and_insert(TestCollapseModel) + self._create_and_insert(TestCollapseModel, sign=1) self.assertEqual(2, len(list(SystemPart.get(self.database)))) From 0787efc1cc705554f25ad4272e7d45ff6e87ccbc Mon Sep 17 00:00:00 2001 From: mangototango Date: Thu, 21 Oct 2021 05:32:53 +0000 Subject: [PATCH 09/11] ignore non-numeric parts of version string --- src/infi/clickhouse_orm/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 37c6bf5..703e982 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -413,7 +413,7 @@ class Database(object): except ServerError as e: logger.exception('Cannot determine server version (%s), assuming 1.1.0', e) ver = '1.1.0' - return tuple(int(n) for n in ver.split('.')) if as_tuple else ver + return tuple(int(n) for n in ver.split('.') if n.isdigit()) if as_tuple else ver def _is_existing_database(self): r = self._send("SELECT count() FROM system.databases WHERE name = '%s'" % self.db_name) From 3b0aaebe50fd3b9f77f7f085b2a39363a7ec9ee3 Mon Sep 17 00:00:00 2001 From: mangototango Date: Thu, 21 Oct 2021 05:33:23 +0000 Subject: [PATCH 10/11] fix precedence of ~ operator in Q objects --- src/infi/clickhouse_orm/query.py | 4 ++-- tests/test_querysets.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py index 92efec4..897d45a 100644 --- a/src/infi/clickhouse_orm/query.py +++ b/src/infi/clickhouse_orm/query.py @@ -209,10 +209,10 @@ class Q(object): @classmethod def _construct_from(cls, l_child, r_child, mode): - if mode == l_child._mode: + if mode == l_child._mode and not l_child._negate: q = deepcopy(l_child) q._children.append(deepcopy(r_child)) - elif mode == r_child._mode: + elif mode == r_child._mode and not r_child._negate: q = deepcopy(r_child) q._children.append(deepcopy(l_child)) else: diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 7f161e0..f458370 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -302,6 +302,18 @@ class QuerySetTestCase(TestCaseWithData): self.assertEqual(qs.conditions_as_sql(), "(first_name = 'a') AND (greater(`height`, 1.7)) AND (last_name = 'b')") + def test_precedence_of_negation(self): + p = ~Q(first_name='a') + q = Q(last_name='b') + r = p & q + self.assertEqual(r.to_sql(Person), "(last_name = 'b') AND (NOT (first_name = 'a'))") + r = q & p + self.assertEqual(r.to_sql(Person), "(last_name = 'b') AND (NOT (first_name = 'a'))") + r = q | p + self.assertEqual(r.to_sql(Person), "(last_name = 'b') OR (NOT (first_name = 'a'))") + r = ~q & p + self.assertEqual(r.to_sql(Person), "(NOT (last_name = 'b')) AND (NOT (first_name = 'a'))") + def test_invalid_filter(self): qs = Person.objects_in(self.database) with self.assertRaises(TypeError): From 4cbaf5e0fb3c64a9906655799c219f25e6b61f1a Mon Sep 17 00:00:00 2001 From: Kobi Tal Date: Thu, 21 Oct 2021 14:12:46 +0300 Subject: [PATCH 11/11] Releasing v2.1.1 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3286b1..5beb084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ Change Log ========== +v2.1.1 +------ +- Improve support of ClickHouse v21.9 (mangototango) +- Ignore non-numeric parts in ClickHouse version (mangototango) +- Fix precedence of ~ operator in Q objects (mangototango) +- Support for adding a column to the beginning of a table (meanmail) +- Add stddevPop and stddevSamp functions (k.peskov) + v2.1.0 ------ - Support for model constraints