diff --git a/docs/expressions.md b/docs/expressions.md index bad4cf0..dda04e7 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -58,6 +58,16 @@ class Person(Model): ... ``` +### Parametric functions + +Some of ClickHouse's aggregate functions can accept not only argument columns, but a set of parameters - constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. For example: +```python +# Most common last names +F.topK(5)(Person.last_name) +# Find 90th, 95th and 99th percentile of heights +F.quantiles(0.9, 0.95, 0.99)(Person.height) +``` + ### Creating new "functions" Since expressions are just Python objects until they get converted to SQL, it is possible to invent new "functions" by combining existing ones into useful building blocks. For example, we can create a reusable expression that takes a string and trims whitespace, converts it to uppercase, and changes blanks to underscores: diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 9487109..6679bc9 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -165,6 +165,24 @@ class Database(object): r = self._send(sql % (self.db_name, model_class.table_name())) return r.text.strip() == '1' + def get_model_for_table(self, table_name, system_table=False): + ''' + Generates a model class from an existing table in the database. + This can be used for querying tables which don't have a corresponding model class, + for example system tables. + + - `table_name`: the table to create a model for + - `system_table`: whether the table is a system table, or belongs to the current database + ''' + db_name = 'system' if system_table else self.db_name + sql = "DESCRIBE `%s`.`%s` FORMAT TSV" % (db_name, table_name) + lines = self._send(sql).iter_lines() + fields = [parse_tsv(line)[:2] for line in lines] + model = ModelBase.create_ad_hoc_model(fields, table_name) + if system_table: + model._system = model._readonly = True + return model + def add_setting(self, name, value): ''' Adds a database setting that will be sent with every request. @@ -363,7 +381,7 @@ class Database(object): mapping = dict(db="`%s`" % self.db_name) if model_class: if model_class.is_system_model(): - mapping['table'] = model_class.table_name() + mapping['table'] = "`system`.`%s`" % model_class.table_name() else: mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name()) query = Template(query).safe_substitute(mapping) diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index 4f12691..dbb5aee 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -423,7 +423,7 @@ class BaseEnumField(Field): import re from enum import Enum members = {} - for match in re.finditer("'(\w+)' = (\d+)", db_type): + for match in re.finditer("'([\w ]+)' = (\d+)", db_type): members[match.group(1)] = int(match.group(2)) enum_cls = Enum('AdHocEnum', members) field_class = Enum8Field if db_type.startswith('Enum8') else Enum16Field diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index aca3e95..c2d830b 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -68,7 +68,7 @@ class ModelBase(type): # fields is a list of tuples (name, db_type) # Check if model exists in cache fields = list(fields) - cache_key = str(fields) + cache_key = model_name + ' ' + str(fields) if cache_key in cls.ad_hoc_model_cache: return cls.ad_hoc_model_cache[cache_key] # Create an ad hoc model class diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py index 69a7f06..66212e8 100644 --- a/src/infi/clickhouse_orm/query.py +++ b/src/infi/clickhouse_orm/query.py @@ -375,10 +375,9 @@ class QuerySet(object): """ distinct = 'DISTINCT ' if self._distinct else '' final = ' FINAL' if self._final else '' - table_name = self._model_cls.table_name() - if not self._model_cls.is_system_model(): - table_name = '`%s`' % table_name - + table_name = '`%s`' % self._model_cls.table_name() + if self._model_cls.is_system_model(): + table_name = '`system`.' + table_name params = (distinct, self.select_fields_as_sql(), table_name, final) sql = u'SELECT %s%s\nFROM %s%s' % params diff --git a/src/infi/clickhouse_orm/system_models.py b/src/infi/clickhouse_orm/system_models.py index dfbe46b..bcb3217 100644 --- a/src/infi/clickhouse_orm/system_models.py +++ b/src/infi/clickhouse_orm/system_models.py @@ -51,7 +51,7 @@ class SystemPart(Model): @classmethod def table_name(cls): - return 'system.parts' + return 'parts' """ Next methods return SQL for some operations, which can be done with partitions @@ -141,8 +141,8 @@ class SystemPart(Model): if conditions: conditions += " AND" field_names = ','.join(cls.fields()) - return database.select("SELECT %s FROM %s WHERE %s database='%s'" % - (field_names, cls.table_name(), conditions, database.db_name), model_class=cls) + return database.select("SELECT %s FROM `system`.%s WHERE %s database='%s'" % + (field_names, cls.table_name(), conditions, database.db_name), model_class=cls) @classmethod def get_active(cls, database, conditions=""): diff --git a/tests/test_database.py b/tests/test_database.py index ab9aa15..a2bb8d7 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -166,8 +166,12 @@ class DatabaseTestCase(TestCaseWithData): Database(self.database.db_name, username='default', password='wrong') exc = cm.exception - self.assertEqual(exc.code, 193) - self.assertTrue(exc.message.startswith('Wrong password for user default')) + print(exc.code, exc.message) + self.assertIn(exc.code, (193, 516)) + if exc.code == 193: + self.assertTrue('Wrong password for user default' in exc.message) + else: + self.assertTrue('default: Authentication failed: password is incorrect' in exc.message) def test_nonexisting_db(self): db = Database('db_not_here', autocreate=False) @@ -233,3 +237,28 @@ class DatabaseTestCase(TestCaseWithData): query = "SELECT DISTINCT type FROM system.columns" for row in self.database.select(query): ModelBase.create_ad_hoc_field(row.type) + + def test_get_model_for_table(self): + # Tests that get_model_for_table works for a non-system model + model = self.database.get_model_for_table('person') + self.assertFalse(model.is_system_model()) + self.assertFalse(model.is_read_only()) + self.assertEqual(model.table_name(), 'person') + # Read a few records + list(model.objects_in(self.database)[:10]) + # Inserts should work too + self.database.insert([ + model(first_name='aaa', last_name='bbb', height=1.77) + ]) + + def test_get_model_for_table__system(self): + # Tests that get_model_for_table works for all system tables + query = "SELECT name FROM system.tables WHERE database='system'" + for row in self.database.select(query): + print(row.name) + model = self.database.get_model_for_table(row.name, system_table=True) + self.assertTrue(model.is_system_model()) + self.assertTrue(model.is_read_only()) + self.assertEqual(model.table_name(), row.name) + # Read a few records + list(model.objects_in(self.database)[:10])