Functions WIP

docs/expressions.md

@@ -58,6 +58,16 @@ class Person(Model):
     ...
 ```
 
+### Parametric functions
+
+Some of ClickHouse's aggregate functions can accept not only argument columns, but a set of parameters - constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. For example:
+```python
+# Most common last names
+F.topK(5)(Person.last_name)
+# Find 90th, 95th and 99th percentile of heights
+F.quantiles(0.9, 0.95, 0.99)(Person.height)
+```
+
 ### Creating new "functions"
 
 Since expressions are just Python objects until they get converted to SQL, it is possible to invent new "functions" by combining existing ones into useful building blocks. For example, we can create a reusable expression that takes a string and trims whitespace, converts it to uppercase, and changes blanks to underscores:

src/infi/clickhouse_orm/database.py

@@ -165,6 +165,24 @@ class Database(object):
         r = self._send(sql % (self.db_name, model_class.table_name()))
         return r.text.strip() == '1'
 
+    def get_model_for_table(self, table_name, system_table=False):
+        '''
+        Generates a model class from an existing table in the database.
+        This can be used for querying tables which don't have a corresponding model class,
+        for example system tables.
+
+        - `table_name`: the table to create a model for
+        - `system_table`: whether the table is a system table, or belongs to the current database
+        '''
+        db_name = 'system' if system_table else self.db_name
+        sql = "DESCRIBE `%s`.`%s` FORMAT TSV" % (db_name, table_name)
+        lines = self._send(sql).iter_lines()
+        fields = [parse_tsv(line)[:2] for line in lines]
+        model = ModelBase.create_ad_hoc_model(fields, table_name)
+        if system_table:
+            model._system = model._readonly = True
+        return model
+
     def add_setting(self, name, value):
         '''
         Adds a database setting that will be sent with every request.
@@ -363,7 +381,7 @@ class Database(object):
             mapping = dict(db="`%s`" % self.db_name)
             if model_class:
                 if model_class.is_system_model():
-                    mapping['table'] = model_class.table_name()
+                    mapping['table'] = "`system`.`%s`" % model_class.table_name()
                 else:
                     mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
             query = Template(query).safe_substitute(mapping)

src/infi/clickhouse_orm/fields.py

@@ -423,7 +423,7 @@ class BaseEnumField(Field):
         import re
         from enum import Enum
         members = {}
-        for match in re.finditer("'(\w+)' = (\d+)", db_type):
+        for match in re.finditer("'([\w ]+)' = (\d+)", db_type):
             members[match.group(1)] = int(match.group(2))
         enum_cls = Enum('AdHocEnum', members)
         field_class = Enum8Field if db_type.startswith('Enum8') else Enum16Field

src/infi/clickhouse_orm/models.py

@@ -68,7 +68,7 @@ class ModelBase(type):
         # fields is a list of tuples (name, db_type)
         # Check if model exists in cache
         fields = list(fields)
-        cache_key = str(fields)
+        cache_key = model_name + ' ' + str(fields)
         if cache_key in cls.ad_hoc_model_cache:
             return cls.ad_hoc_model_cache[cache_key]
         # Create an ad hoc model class

src/infi/clickhouse_orm/query.py

@@ -375,10 +375,9 @@ class QuerySet(object):
         """
         distinct = 'DISTINCT ' if self._distinct else ''
         final = ' FINAL' if self._final else ''
-        table_name = self._model_cls.table_name()
+        table_name = '`%s`' % self._model_cls.table_name()
-        if not self._model_cls.is_system_model():
+        if self._model_cls.is_system_model():
-            table_name = '`%s`' % table_name
+            table_name = '`system`.' + table_name
-
         params = (distinct, self.select_fields_as_sql(), table_name, final)
         sql = u'SELECT %s%s\nFROM %s%s' % params
 

src/infi/clickhouse_orm/system_models.py

@@ -51,7 +51,7 @@ class SystemPart(Model):
 
     @classmethod
     def table_name(cls):
-        return 'system.parts'
+        return 'parts'
 
     """
     Next methods return SQL for some operations, which can be done with partitions
@@ -141,8 +141,8 @@ class SystemPart(Model):
         if conditions:
             conditions += " AND"
         field_names = ','.join(cls.fields())
-        return database.select("SELECT %s FROM %s WHERE %s database='%s'" %
+        return database.select("SELECT %s FROM `system`.%s WHERE %s database='%s'" %
-                               (field_names,  cls.table_name(), conditions, database.db_name), model_class=cls)
+                               (field_names, cls.table_name(), conditions, database.db_name), model_class=cls)
 
     @classmethod
     def get_active(cls, database, conditions=""):

tests/test_database.py

@@ -166,8 +166,12 @@ class DatabaseTestCase(TestCaseWithData):
             Database(self.database.db_name, username='default', password='wrong')
 
         exc = cm.exception
-        self.assertEqual(exc.code, 193)
+        print(exc.code, exc.message)
-        self.assertTrue(exc.message.startswith('Wrong password for user default'))
+        self.assertIn(exc.code, (193, 516))
+        if exc.code == 193:
+            self.assertTrue('Wrong password for user default' in exc.message)
+        else:
+            self.assertTrue('default: Authentication failed: password is incorrect' in exc.message)
 
     def test_nonexisting_db(self):
         db = Database('db_not_here', autocreate=False)
@@ -233,3 +237,28 @@ class DatabaseTestCase(TestCaseWithData):
         query = "SELECT DISTINCT type FROM system.columns"
         for row in self.database.select(query):
             ModelBase.create_ad_hoc_field(row.type)
+
+    def test_get_model_for_table(self):
+        # Tests that get_model_for_table works for a non-system model
+        model = self.database.get_model_for_table('person')
+        self.assertFalse(model.is_system_model())
+        self.assertFalse(model.is_read_only())
+        self.assertEqual(model.table_name(), 'person')
+        # Read a few records
+        list(model.objects_in(self.database)[:10])
+        # Inserts should work too
+        self.database.insert([
+            model(first_name='aaa', last_name='bbb', height=1.77)
+        ])
+
+    def test_get_model_for_table__system(self):
+        # Tests that get_model_for_table works for all system tables
+        query = "SELECT name FROM system.tables WHERE database='system'"
+        for row in self.database.select(query):
+            print(row.name)
+            model = self.database.get_model_for_table(row.name, system_table=True)
+            self.assertTrue(model.is_system_model())
+            self.assertTrue(model.is_read_only())
+            self.assertEqual(model.table_name(), row.name)
+            # Read a few records
+            list(model.objects_in(self.database)[:10])