- Add log_statements parameter to database initializer

- Fix test_merge which fails on ClickHouse v19.8.3

CHANGELOG.md

@@ -11,6 +11,8 @@ Unreleased
 - Use HTTP Basic Authentication instead of passing the credentials in the URL
 - Support default/alias/materialized for nullable fields
 - Add UUIDField (kpotehin)
+- Add `log_statements` parameter to database initializer
+- Fix test_merge which fails on ClickHouse v19.8.3
 
 v1.0.4
 ------

src/infi/clickhouse_orm/database.py

@@ -83,7 +83,7 @@ class Database(object):
 
     def __init__(self, db_name, db_url='http://localhost:8123/',
                  username=None, password=None, readonly=False, autocreate=True,
-                 timeout=60, verify_ssl_cert=True):
+                 timeout=60, verify_ssl_cert=True, log_statements=False):
         '''
         Initializes a database instance. Unless it's readonly, the database will be
         created on the ClickHouse server if it does not already exist.
@@ -96,6 +96,7 @@ class Database(object):
         - `autocreate`: automatically create the database if it does not exist (unless in readonly mode).
         - `timeout`: the connection timeout in seconds.
         - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS.
+        - `log_statements`: when True, all database statements are logged.
         '''
         self.db_name = db_name
         self.db_url = db_url
@@ -105,6 +106,7 @@ class Database(object):
         self.request_session.verify = verify_ssl_cert
         if username:
             self.request_session.auth = (username, password or '')
+        self.log_statements = log_statements
         self.settings = {}
         self.db_exists = False # this is required before running _is_existing_database
         self.db_exists = self._is_existing_database()
@@ -334,6 +336,8 @@ class Database(object):
     def _send(self, data, settings=None, stream=False):
         if isinstance(data, string_types):
             data = data.encode('utf-8')
+            if self.log_statements:
+                logger.info(data)
         params = self._build_params(settings)
         r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout)
         if r.status_code != 200:

src/infi/clickhouse_orm/models.py

@@ -311,9 +311,16 @@ class MergeModel(Model):
 
     @classmethod
     def create_table_sql(cls, db):
-        assert isinstance(cls.engine, Merge), "engine must be engines.Merge instance"
-        return super(MergeModel, cls).create_table_sql(db)
-
+        assert isinstance(cls.engine, Merge), "engine must be an instance of engines.Merge"
+        parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())]
+        cols = []
+        for name, field in iteritems(cls.fields()):
+            if name != '_table':
+                cols.append('    %s %s' % (name, field.get_sql()))
+        parts.append(',\n'.join(cols))
+        parts.append(')')
+        parts.append('ENGINE = ' + cls.engine.create_table_sql(db))
+        return '\n'.join(parts)
 
 # TODO: base class for models that require specific engine
 
@@ -324,7 +331,7 @@ class DistributedModel(Model):
     """
 
     def set_database(self, db):
-        assert isinstance(self.engine, Distributed), "engine must be engines.Distributed instance"
+        assert isinstance(self.engine, Distributed), "engine must be an instance of engines.Distributed"
         res = super(DistributedModel, self).set_database(db)
         return res
 

tests/base_test_with_data.py

@@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class TestCaseWithData(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(Person)
 
     def tearDown(self):
@@ -46,7 +46,7 @@ class Person(Model):
 data = [
     {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63",
      "passport": 35052255},
-    
+
     {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74",
      "passport": 36052255},
 

tests/test_alias_fields.py

@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class MaterializedFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithAliasFields)
 
     def tearDown(self):

tests/test_array_fields.py

@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class ArrayFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithArrays)
 
     def tearDown(self):

tests/test_custom_fields.py

@@ -9,7 +9,7 @@ from infi.clickhouse_orm.engines import Memory
 class CustomFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()

tests/test_datetime_fields.py

@@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import *
 class DateFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithDate)
 
     def tearDown(self):

tests/test_decimal_fields.py

@@ -12,7 +12,7 @@ from infi.clickhouse_orm.engines import *
 class DecimalFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.add_setting('allow_experimental_decimal_type', 1)
         try:
             self.database.create_table(DecimalModel)

tests/test_engines.py

@@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class _EnginesHelperTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()
@@ -115,8 +115,8 @@ class EnginesTestCase(_EnginesHelperTestCase):
             TestModel2(date='2017-01-02', event_id=2, event_group=2, event_count=2, event_version=2)
         ])
         # event_uversion is materialized field. So * won't select it and it will be zero
-        res = self.database.select('SELECT *, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel)
-        res = [row for row in res]
+        res = self.database.select('SELECT *, _table, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel)
+        res = list(res)
         self.assertEqual(2, len(res))
         self.assertDictEqual({
             '_table': 'testmodel1',

tests/test_enum_fields.py

@@ -15,7 +15,7 @@ except NameError:
 class EnumFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithEnum)
         self.database.create_table(ModelWithEnumArray)
 

tests/test_fixed_string_fields.py

@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class FixedStringFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(FixedStringModel)
 
     def tearDown(self):

tests/test_join.py

@@ -9,7 +9,7 @@ from infi.clickhouse_orm import database, engines, fields, models
 class JoinTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = database.Database('test-db')
+        self.database = database.Database('test-db', log_statements=True)
         self.database.create_table(Foo)
         self.database.create_table(Bar)
         self.database.insert([Foo(id=i) for i in range(3)])

tests/test_materialized_fields.py

@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class MaterializedFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithMaterializedFields)
 
     def tearDown(self):

tests/test_migrations.py

@@ -24,7 +24,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class MigrationsTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.drop_table(MigrationHistory)
 
     def tearDown(self):

tests/test_nullable_fields.py

@@ -14,7 +14,7 @@ from datetime import date, datetime
 class NullableFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithNullable)
 
     def tearDown(self):

tests/test_system_models.py

@@ -14,7 +14,7 @@ from infi.clickhouse_orm.system_models import SystemPart
 
 class SystemTest(unittest.TestCase):
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()
@@ -38,7 +38,7 @@ class SystemPartTest(unittest.TestCase):
     BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/']
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(TestTable)
         self.database.create_table(CustomPartitionedTable)
         self.database.insert([TestTable(date_field=date.today())])

tests/test_uuid_fields.py

@@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import Memory
 class UUIDFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()