From 9dd1a8f4090e6e090c702f16c26a598d18311c25 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 13 Jun 2019 08:12:56 +0300 Subject: [PATCH] - Add `log_statements` parameter to database initializer - Fix test_merge which fails on ClickHouse v19.8.3 --- CHANGELOG.md | 2 ++ src/infi/clickhouse_orm/database.py | 6 +++++- src/infi/clickhouse_orm/models.py | 15 +++++++++++---- tests/base_test_with_data.py | 4 ++-- tests/test_alias_fields.py | 2 +- tests/test_array_fields.py | 2 +- tests/test_custom_fields.py | 2 +- tests/test_datetime_fields.py | 2 +- tests/test_decimal_fields.py | 2 +- tests/test_engines.py | 6 +++--- tests/test_enum_fields.py | 2 +- tests/test_fixed_string_fields.py | 2 +- tests/test_join.py | 2 +- tests/test_materialized_fields.py | 2 +- tests/test_migrations.py | 2 +- tests/test_nullable_fields.py | 2 +- tests/test_system_models.py | 4 ++-- tests/test_uuid_fields.py | 2 +- 18 files changed, 37 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f82621..96e3724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Unreleased - Use HTTP Basic Authentication instead of passing the credentials in the URL - Support default/alias/materialized for nullable fields - Add UUIDField (kpotehin) +- Add `log_statements` parameter to database initializer +- Fix test_merge which fails on ClickHouse v19.8.3 v1.0.4 ------ diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index b7379da..d9eac52 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -83,7 +83,7 @@ class Database(object): def __init__(self, db_name, db_url='http://localhost:8123/', username=None, password=None, readonly=False, autocreate=True, - timeout=60, verify_ssl_cert=True): + timeout=60, verify_ssl_cert=True, log_statements=False): ''' Initializes a database instance. Unless it's readonly, the database will be created on the ClickHouse server if it does not already exist. @@ -96,6 +96,7 @@ class Database(object): - `autocreate`: automatically create the database if it does not exist (unless in readonly mode). - `timeout`: the connection timeout in seconds. - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS. + - `log_statements`: when True, all database statements are logged. ''' self.db_name = db_name self.db_url = db_url @@ -105,6 +106,7 @@ class Database(object): self.request_session.verify = verify_ssl_cert if username: self.request_session.auth = (username, password or '') + self.log_statements = log_statements self.settings = {} self.db_exists = False # this is required before running _is_existing_database self.db_exists = self._is_existing_database() @@ -334,6 +336,8 @@ class Database(object): def _send(self, data, settings=None, stream=False): if isinstance(data, string_types): data = data.encode('utf-8') + if self.log_statements: + logger.info(data) params = self._build_params(settings) r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout) if r.status_code != 200: diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index d008513..beae53f 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -311,9 +311,16 @@ class MergeModel(Model): @classmethod def create_table_sql(cls, db): - assert isinstance(cls.engine, Merge), "engine must be engines.Merge instance" - return super(MergeModel, cls).create_table_sql(db) - + assert isinstance(cls.engine, Merge), "engine must be an instance of engines.Merge" + parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())] + cols = [] + for name, field in iteritems(cls.fields()): + if name != '_table': + cols.append(' %s %s' % (name, field.get_sql())) + parts.append(',\n'.join(cols)) + parts.append(')') + parts.append('ENGINE = ' + cls.engine.create_table_sql(db)) + return '\n'.join(parts) # TODO: base class for models that require specific engine @@ -324,7 +331,7 @@ class DistributedModel(Model): """ def set_database(self, db): - assert isinstance(self.engine, Distributed), "engine must be engines.Distributed instance" + assert isinstance(self.engine, Distributed), "engine must be an instance of engines.Distributed" res = super(DistributedModel, self).set_database(db) return res diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index c3fc376..f080f85 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING) class TestCaseWithData(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(Person) def tearDown(self): @@ -46,7 +46,7 @@ class Person(Model): data = [ {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63", "passport": 35052255}, - + {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74", "passport": 36052255}, diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index aa692c2..de33993 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import * class MaterializedFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(ModelWithAliasFields) def tearDown(self): diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py index 9aa352e..f63ac2f 100644 --- a/tests/test_array_fields.py +++ b/tests/test_array_fields.py @@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import * class ArrayFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(ModelWithArrays) def tearDown(self): diff --git a/tests/test_custom_fields.py b/tests/test_custom_fields.py index 76dbd15..c0b739c 100644 --- a/tests/test_custom_fields.py +++ b/tests/test_custom_fields.py @@ -9,7 +9,7 @@ from infi.clickhouse_orm.engines import Memory class CustomFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) def tearDown(self): self.database.drop_database() diff --git a/tests/test_datetime_fields.py b/tests/test_datetime_fields.py index 1c11849..abb8c47 100644 --- a/tests/test_datetime_fields.py +++ b/tests/test_datetime_fields.py @@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import * class DateFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(ModelWithDate) def tearDown(self): diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py index db87d62..3a862c1 100644 --- a/tests/test_decimal_fields.py +++ b/tests/test_decimal_fields.py @@ -12,7 +12,7 @@ from infi.clickhouse_orm.engines import * class DecimalFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.add_setting('allow_experimental_decimal_type', 1) try: self.database.create_table(DecimalModel) diff --git a/tests/test_engines.py b/tests/test_engines.py index 846e825..a05d359 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING) class _EnginesHelperTestCase(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) def tearDown(self): self.database.drop_database() @@ -115,8 +115,8 @@ class EnginesTestCase(_EnginesHelperTestCase): TestModel2(date='2017-01-02', event_id=2, event_group=2, event_count=2, event_version=2) ]) # event_uversion is materialized field. So * won't select it and it will be zero - res = self.database.select('SELECT *, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel) - res = [row for row in res] + res = self.database.select('SELECT *, _table, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel) + res = list(res) self.assertEqual(2, len(res)) self.assertDictEqual({ '_table': 'testmodel1', diff --git a/tests/test_enum_fields.py b/tests/test_enum_fields.py index 239a9b6..c6b7b1f 100644 --- a/tests/test_enum_fields.py +++ b/tests/test_enum_fields.py @@ -15,7 +15,7 @@ except NameError: class EnumFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(ModelWithEnum) self.database.create_table(ModelWithEnumArray) diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py index 4ed663e..e9e0124 100644 --- a/tests/test_fixed_string_fields.py +++ b/tests/test_fixed_string_fields.py @@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import * class FixedStringFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(FixedStringModel) def tearDown(self): diff --git a/tests/test_join.py b/tests/test_join.py index 49c0ac6..e3d5d12 100644 --- a/tests/test_join.py +++ b/tests/test_join.py @@ -9,7 +9,7 @@ from infi.clickhouse_orm import database, engines, fields, models class JoinTest(unittest.TestCase): def setUp(self): - self.database = database.Database('test-db') + self.database = database.Database('test-db', log_statements=True) self.database.create_table(Foo) self.database.create_table(Bar) self.database.insert([Foo(id=i) for i in range(3)]) diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py index 54d2111..3bfadcd 100644 --- a/tests/test_materialized_fields.py +++ b/tests/test_materialized_fields.py @@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import * class MaterializedFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(ModelWithMaterializedFields) def tearDown(self): diff --git a/tests/test_migrations.py b/tests/test_migrations.py index c421ac7..d84450f 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -24,7 +24,7 @@ logging.getLogger("requests").setLevel(logging.WARNING) class MigrationsTestCase(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.drop_table(MigrationHistory) def tearDown(self): diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py index 74bc66e..b395a9b 100644 --- a/tests/test_nullable_fields.py +++ b/tests/test_nullable_fields.py @@ -14,7 +14,7 @@ from datetime import date, datetime class NullableFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(ModelWithNullable) def tearDown(self): diff --git a/tests/test_system_models.py b/tests/test_system_models.py index b9576ac..01229c8 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -14,7 +14,7 @@ from infi.clickhouse_orm.system_models import SystemPart class SystemTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) def tearDown(self): self.database.drop_database() @@ -38,7 +38,7 @@ class SystemPartTest(unittest.TestCase): BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/'] def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) self.database.create_table(TestTable) self.database.create_table(CustomPartitionedTable) self.database.insert([TestTable(date_field=date.today())]) diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py index bea3d3c..247b757 100644 --- a/tests/test_uuid_fields.py +++ b/tests/test_uuid_fields.py @@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import Memory class UUIDFieldsTest(unittest.TestCase): def setUp(self): - self.database = Database('test-db') + self.database = Database('test-db', log_statements=True) def tearDown(self): self.database.drop_database()