From 5abe39ed2443b2811df3df69273b5878277c26ad Mon Sep 17 00:00:00 2001 From: mangototango Date: Sat, 16 Oct 2021 07:47:17 +0000 Subject: [PATCH] Fixes to make the tests pass on ClickHouse v21.9 --- src/infi/clickhouse_orm/database.py | 5 +++++ src/infi/clickhouse_orm/fields.py | 2 +- src/infi/clickhouse_orm/funcs.py | 34 ++++++++++++++--------------- tests/test_compressed_fields.py | 4 ++-- tests/test_database.py | 12 ++++++++-- tests/test_dictionaries.py | 2 +- tests/test_engines.py | 8 +++---- 7 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index d42e224..37c6bf5 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -51,6 +51,11 @@ class ServerError(DatabaseException): Code:\ (?P\d+), \ e\.displayText\(\)\ =\ (?P[^ \n]+):\ (?P.+) ''', re.VERBOSE | re.DOTALL), + # ClickHouse v21+ + re.compile(r''' + Code:\ (?P\d+). + \ (?P[^ \n]+):\ (?P.+) + ''', re.VERBOSE | re.DOTALL), ) @classmethod diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index 4f631cc..6e73e3f 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -109,7 +109,7 @@ class Field(FunctionOperatorsMixin): elif self.default: default = self.to_db_string(self.default) sql += ' DEFAULT %s' % default - if self.codec and db and db.has_codec_support: + if self.codec and db and db.has_codec_support and not self.alias: sql += ' CODEC(%s)' % self.codec return sql diff --git a/src/infi/clickhouse_orm/funcs.py b/src/infi/clickhouse_orm/funcs.py index d84c761..e2ded48 100644 --- a/src/infi/clickhouse_orm/funcs.py +++ b/src/infi/clickhouse_orm/funcs.py @@ -391,11 +391,11 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toYear', d) @staticmethod - def toISOYear(d, timezone=''): + def toISOYear(d, timezone=NO_VALUE): return F('toISOYear', d, timezone) @staticmethod - def toQuarter(d, timezone=''): + def toQuarter(d, timezone=NO_VALUE): return F('toQuarter', d, timezone) if timezone else F('toQuarter', d) @staticmethod @@ -403,11 +403,11 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toMonth', d) @staticmethod - def toWeek(d, mode=0, timezone=''): + def toWeek(d, mode=0, timezone=NO_VALUE): return F('toWeek', d, mode, timezone) @staticmethod - def toISOWeek(d, timezone=''): + def toISOWeek(d, timezone=NO_VALUE): return F('toISOWeek', d, timezone) if timezone else F('toISOWeek', d) @staticmethod @@ -483,7 +483,7 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toStartOfDay', d) @staticmethod - def toTime(d, timezone=''): + def toTime(d, timezone=NO_VALUE): return F('toTime', d, timezone) @staticmethod @@ -491,47 +491,47 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('toTimeZone', dt, timezone) @staticmethod - def toUnixTimestamp(dt, timezone=''): + def toUnixTimestamp(dt, timezone=NO_VALUE): return F('toUnixTimestamp', dt, timezone) @staticmethod - def toYYYYMM(dt, timezone=''): + def toYYYYMM(dt, timezone=NO_VALUE): return F('toYYYYMM', dt, timezone) if timezone else F('toYYYYMM', dt) @staticmethod - def toYYYYMMDD(dt, timezone=''): + def toYYYYMMDD(dt, timezone=NO_VALUE): return F('toYYYYMMDD', dt, timezone) if timezone else F('toYYYYMMDD', dt) @staticmethod - def toYYYYMMDDhhmmss(dt, timezone=''): + def toYYYYMMDDhhmmss(dt, timezone=NO_VALUE): return F('toYYYYMMDDhhmmss', dt, timezone) if timezone else F('toYYYYMMDDhhmmss', dt) @staticmethod - def toRelativeYearNum(d, timezone=''): + def toRelativeYearNum(d, timezone=NO_VALUE): return F('toRelativeYearNum', d, timezone) @staticmethod - def toRelativeMonthNum(d, timezone=''): + def toRelativeMonthNum(d, timezone=NO_VALUE): return F('toRelativeMonthNum', d, timezone) @staticmethod - def toRelativeWeekNum(d, timezone=''): + def toRelativeWeekNum(d, timezone=NO_VALUE): return F('toRelativeWeekNum', d, timezone) @staticmethod - def toRelativeDayNum(d, timezone=''): + def toRelativeDayNum(d, timezone=NO_VALUE): return F('toRelativeDayNum', d, timezone) @staticmethod - def toRelativeHourNum(d, timezone=''): + def toRelativeHourNum(d, timezone=NO_VALUE): return F('toRelativeHourNum', d, timezone) @staticmethod - def toRelativeMinuteNum(d, timezone=''): + def toRelativeMinuteNum(d, timezone=NO_VALUE): return F('toRelativeMinuteNum', d, timezone) @staticmethod - def toRelativeSecondNum(d, timezone=''): + def toRelativeSecondNum(d, timezone=NO_VALUE): return F('toRelativeSecondNum', d, timezone) @staticmethod @@ -555,7 +555,7 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): return F('timeSlots', start_time, F.toUInt32(duration)) @staticmethod - def formatDateTime(d, format, timezone=''): + def formatDateTime(d, format, timezone=NO_VALUE): return F('formatDateTime', d, format, timezone) @staticmethod diff --git a/tests/test_compressed_fields.py b/tests/test_compressed_fields.py index 8d17571..440e0b3 100644 --- a/tests/test_compressed_fields.py +++ b/tests/test_compressed_fields.py @@ -106,7 +106,7 @@ class CompressedFieldsTestCase(unittest.TestCase): ('nullable_field', 'CODEC(ZSTD(1))'), ('array_field', 'CODEC(Delta(2), LZ4HC(0))'), ('float_field', 'CODEC(NONE)'), - ('alias_field', 'CODEC(ZSTD(4))')]) + ('alias_field', '')]) class CompressedModel(Model): @@ -120,4 +120,4 @@ class CompressedModel(Model): float_field = Float32Field(codec='NONE') alias_field = Float32Field(alias='float_field', codec='ZSTD(4)') - engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) \ No newline at end of file + engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) diff --git a/tests/test_database.py b/tests/test_database.py index 38681d4..44971e1 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -181,12 +181,13 @@ class DatabaseTestCase(TestCaseWithData): Database(self.database.db_name, username='default', password='wrong') exc = cm.exception + print(exc.code, exc.message) if exc.code == 193: # ClickHouse version < 20.3 self.assertTrue(exc.message.startswith('Wrong password for user default')) elif exc.code == 516: # ClickHouse version >= 20.3 self.assertTrue(exc.message.startswith('default: Authentication failed')) else: - raise Exception('Unexpected error code - %s' % exc.code) + raise Exception('Unexpected error code - %s %s' % (exc.code, exc.message)) def test_nonexisting_db(self): db = Database('db_not_here', autocreate=False) @@ -251,6 +252,8 @@ class DatabaseTestCase(TestCaseWithData): from infi.clickhouse_orm.models import ModelBase query = "SELECT DISTINCT type FROM system.columns" for row in self.database.select(query): + if row.type.startswith('Map'): + continue # Not supported yet ModelBase.create_ad_hoc_field(row.type) def test_get_model_for_table(self): @@ -271,7 +274,12 @@ class DatabaseTestCase(TestCaseWithData): query = "SELECT name FROM system.tables WHERE database='system'" for row in self.database.select(query): print(row.name) - model = self.database.get_model_for_table(row.name, system_table=True) + if row.name in ('distributed_ddl_queue',): + continue # Not supported + try: + model = self.database.get_model_for_table(row.name, system_table=True) + except NotImplementedError: + continue # Table contains an unsupported field type self.assertTrue(model.is_system_model()) self.assertTrue(model.is_read_only()) self.assertEqual(model.table_name(), row.name) diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 7da4160..c031d2e 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -105,7 +105,7 @@ class HierarchicalDictionaryTest(DictionaryTestMixin, unittest.TestCase): def test_dictgethierarchy(self): self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(3)), [3, 2, 1]) - self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(99)), [99]) + self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(99)), []) def test_dictisin(self): self._test_func(F.dictIsIn(self.dict_name, F.toUInt64(3), F.toUInt64(1)), 1) diff --git a/tests/test_engines.py b/tests/test_engines.py index 2fcc8c2..b2e0e5d 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -17,10 +17,10 @@ class _EnginesHelperTestCase(unittest.TestCase): class EnginesTestCase(_EnginesHelperTestCase): - def _create_and_insert(self, model_class): + def _create_and_insert(self, model_class, **kwargs): self.database.create_table(model_class) self.database.insert([ - model_class(date='2017-01-01', event_id=23423, event_group=13, event_count=7, event_version=1) + model_class(date='2017-01-01', event_id=23423, event_group=13, event_count=7, event_version=1, **kwargs) ]) def test_merge_tree(self): @@ -155,7 +155,7 @@ class EnginesTestCase(_EnginesHelperTestCase): ) self._create_and_insert(TestModel) - self._create_and_insert(TestCollapseModel) + self._create_and_insert(TestCollapseModel, sign=1) # Result order may be different, lets sort manually parts = sorted(list(SystemPart.get(self.database)), key=lambda x: x.table) @@ -188,7 +188,7 @@ class EnginesTestCase(_EnginesHelperTestCase): ) self._create_and_insert(TestModel) - self._create_and_insert(TestCollapseModel) + self._create_and_insert(TestCollapseModel, sign=1) self.assertEqual(2, len(list(SystemPart.get(self.database))))