Added MaterializedField and AliasField

src/infi/clickhouse_orm/database.py

@@ -50,11 +50,11 @@ class Database(object):
         model_class = first_instance.__class__
         def gen():
             yield self._substitute('INSERT INTO $table FORMAT TabSeparated\n', model_class).encode('utf-8')
-            yield (first_instance.to_tsv() + '\n').encode('utf-8')
+            yield (first_instance.to_tsv(insertable_only=True) + '\n').encode('utf-8')
             # Collect lines in batches of batch_size
             batch = []
             for instance in i:
-                batch.append(instance.to_tsv())
+                batch.append(instance.to_tsv(insertable_only=True))
                 if len(batch) >= batch_size:
                     # Return the current batch of lines
                     yield ('\n'.join(batch) + '\n').encode('utf-8')

src/infi/clickhouse_orm/fields.py

@@ -12,6 +12,9 @@ class Field(object):
     class_default = 0
     db_type = None
 
+    # This flag indicates, if we should take this field value when inserting data
+    insertable = True
+
     def __init__(self, default=None):
         self.creation_counter = Field.creation_counter
         Field.creation_counter += 1
@@ -295,3 +298,76 @@ class ArrayField(Field):
     def get_sql(self, with_default=True):
         from .utils import escape
         return 'Array(%s)' % self.inner_field.get_sql(with_default=False)
+
+
+class RelativeField(Field):
+    insertable = False
+
+    def __init__(self, inner_field):
+        """
+        Creates MATERIALIZED or ALIAS field
+        :param inner_field: Field subclass this field is acting like
+        """
+        assert isinstance(inner_field, Field), "field must be Field subclass"
+        self.class_default = inner_field.class_default
+        self.default = inner_field.default
+        super(RelativeField, self).__init__()
+        self.inner_field = inner_field
+
+    def to_python(self, value):
+        return self.inner_field.to_python(value)
+
+    def validate(self, value):
+        return self.inner_field.validate(value)
+
+    def to_db_string(self, value, quote=True):
+        return self.inner_field.to_db_string(value, quote=quote)
+
+
+class MaterializedField(RelativeField):
+    """
+    Creates ClickHouse MATERIALIZED field. It doesn't contain real data in database, it is counted on the spot
+    https://clickhouse.yandex/reference_en.html#Default values
+    """
+
+    def __init__(self, inner_field, code):
+        """
+        Creates MATERIALIZED field
+        :param inner_field: Field subclass this field is acting like
+        :param code: ClickHouse code to execute when materialized field is called. See ClickHouse docs.
+        """
+        super(MaterializedField, self).__init__(inner_field)
+
+        self._code = code
+
+    def get_sql(self, with_default=True):
+        """
+        Generates SQL for create table command
+        :param with_default: This flag is inherited from Field model. Does nothing (MATERIALIZED have no default)
+        :return: Creation SQL string
+        """
+        return '%s MATERIALIZED %s' % (self.inner_field.db_type, self._code)
+
+
+class AliasField(RelativeField):
+    """
+    Creates ClickHouse ALIAS field. It doesn't contain real data in database, only copies other one
+    https://clickhouse.yandex/reference_en.html#Default values
+    """
+
+    def __init__(self, inner_field, base_field_name):
+        """
+        Creates ALIAS field
+        :param inner_field: Field instance this field is acting like
+        :param base_field_name: Name of field, to which alias is built
+        """
+        super(AliasField, self).__init__(inner_field)
+        self.base_field_name = base_field_name
+
+    def get_sql(self, with_default=True):
+        """
+        Generates SQL for create table command
+        :param with_default: This flag is inherited from Field model. Does nothing (ALIAS have no default)
+        :return: Creation SQL string
+        """
+        return '%s ALIAS %s' % (self.inner_field.db_type, self.base_field_name)

src/infi/clickhouse_orm/models.py

@@ -150,9 +150,14 @@ class Model(with_metaclass(ModelBase)):
             kwargs[name] = next(values)
         return cls(**kwargs)
 
-    def to_tsv(self):
+    def to_tsv(self, insertable_only=False):
         '''
         Returns the instance's column values as a tab-separated line. A newline is not included.
+        :param bool insertable_only: If True, returns only fields, that can be inserted into database
         '''
         data = self.__dict__
-        return '\t'.join(field.to_db_string(data[name], quote=False) for name, field in self._fields)
+
+        fields = [f for f in self._fields if f[1].insertable] if insertable_only else self._fields
+        return '\t'.join(field.to_db_string(data[name], quote=False) for name, field in fields)
+
+

tests/sample_migrations/0008.py

@@ -0,0 +1,6 @@
+from infi.clickhouse_orm import migrations
+from ..test_migrations import *
+
+operations = [
+    migrations.CreateTable(MaterializedModel)
+]

tests/sample_migrations/0009.py

@@ -0,0 +1,6 @@
+from infi.clickhouse_orm import migrations
+from ..test_migrations import *
+
+operations = [
+    migrations.CreateTable(AliasModel)
+]

tests/test_alias_fields.py

@@ -0,0 +1,57 @@
+import unittest
+from datetime import date
+
+from infi.clickhouse_orm.database import Database
+from infi.clickhouse_orm.models import Model
+from infi.clickhouse_orm.fields import *
+from infi.clickhouse_orm.engines import *
+
+
+class MaterializedFieldsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.database = Database('test-db')
+        self.database.create_table(ModelWithAliasFields)
+
+    def tearDown(self):
+        self.database.drop_database()
+
+    def test_insert_and_select(self):
+        instance = ModelWithAliasFields(
+            date_field='2016-08-30',
+            int_field=-10,
+            str_field='TEST'
+        )
+        self.database.insert([instance])
+        # We can't select * from table, as it doesn't select materialized and alias fields
+        query = 'SELECT date_field, int_field, str_field, alias_int, alias_date, alias_str' \
+                ' FROM $db.%s ORDER BY alias_date' % ModelWithAliasFields.table_name()
+        for model_cls in (ModelWithAliasFields, None):
+            results = list(self.database.select(query, model_cls))
+            self.assertEquals(len(results), 1)
+            self.assertEquals(results[0].date_field, instance.date_field)
+            self.assertEquals(results[0].int_field, instance.int_field)
+            self.assertEquals(results[0].str_field, instance.str_field)
+            self.assertEquals(results[0].alias_int, instance.int_field)
+            self.assertEquals(results[0].alias_str, instance.str_field)
+            self.assertEquals(results[0].alias_date, instance.date_field)
+
+    def test_assignment_error(self):
+        # I can't prevent assigning at all, in case db.select statements with model provided sets model fields.
+        instance = ModelWithAliasFields()
+        for value in ('x', [date.today()], ['aaa'], [None]):
+            with self.assertRaises(ValueError):
+                instance.alias_date = value
+
+
+class ModelWithAliasFields(Model):
+    int_field = Int32Field()
+    date_field = DateField()
+    str_field = StringField()
+
+    alias_str = AliasField(StringField(), 'str_field')
+    alias_int = MaterializedField(Int32Field(), 'int_field')
+    alias_date = MaterializedField(DateField(), 'date_field')
+
+    engine = MergeTree('date_field', ('date_field',))
+

tests/test_materialized_fields.py

@@ -0,0 +1,57 @@
+import unittest
+from datetime import date
+
+from infi.clickhouse_orm.database import Database
+from infi.clickhouse_orm.models import Model
+from infi.clickhouse_orm.fields import *
+from infi.clickhouse_orm.engines import *
+
+
+class MaterializedFieldsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.database = Database('test-db')
+        self.database.create_table(ModelWithMaterializedFields)
+
+    def tearDown(self):
+        self.database.drop_database()
+
+    def test_insert_and_select(self):
+        instance = ModelWithMaterializedFields(
+            date_time_field='2016-08-30 11:00:00',
+            int_field=-10,
+            str_field='TEST'
+        )
+        self.database.insert([instance])
+        # We can't select * from table, as it doesn't select materialized and alias fields
+        query = 'SELECT date_time_field, int_field, str_field, mat_int, mat_date, mat_str' \
+                ' FROM $db.%s ORDER BY mat_date' % ModelWithMaterializedFields.table_name()
+        for model_cls in (ModelWithMaterializedFields, None):
+            results = list(self.database.select(query, model_cls))
+            self.assertEquals(len(results), 1)
+            self.assertEquals(results[0].date_time_field, instance.date_time_field)
+            self.assertEquals(results[0].int_field, instance.int_field)
+            self.assertEquals(results[0].str_field, instance.str_field)
+            self.assertEquals(results[0].mat_int, abs(instance.int_field))
+            self.assertEquals(results[0].mat_str, instance.str_field.lower())
+            self.assertEquals(results[0].mat_date, instance.date_time_field.date())
+
+    def test_assignment_error(self):
+        # I can't prevent assigning at all, in case db.select statements with model provided sets model fields.
+        instance = ModelWithMaterializedFields()
+        for value in ('x', [date.today()], ['aaa'], [None]):
+            with self.assertRaises(ValueError):
+                instance.mat_date = value
+
+
+class ModelWithMaterializedFields(Model):
+    int_field = Int32Field()
+    date_time_field = DateTimeField()
+    str_field = StringField()
+
+    mat_str = MaterializedField(StringField(), 'lower(str_field)')
+    mat_int = MaterializedField(Int32Field(), 'abs(int_field)')
+    mat_date = MaterializedField(DateField(), 'toDate(date_time_field)')
+
+    engine = MergeTree('mat_date', ('mat_date',))
+

tests/test_migrations.py

@@ -60,6 +60,15 @@ class MigrationsTestCase(unittest.TestCase):
         self.assertTrue(self.tableExists(EnumModel1))
         self.assertEquals(self.getTableFields(EnumModel2), 
                           [('date', 'Date'), ('f1', "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")])
+        self.database.migrate('tests.sample_migrations', 8)
+        self.assertTrue(self.tableExists(MaterializedModel))
+        self.assertEquals(self.getTableFields(MaterializedModel),
+                          [('date_time', "DateTime"), ('date', 'Date')])
+        self.database.migrate('tests.sample_migrations', 9)
+        self.assertTrue(self.tableExists(AliasModel))
+        self.assertEquals(self.getTableFields(AliasModel),
+                          [('date', 'Date'), ('date_alias', "Date")])
+
 
 # Several different models with the same table name, to simulate a table that changes over time
 
@@ -127,3 +136,25 @@ class EnumModel2(Model):
     @classmethod
     def table_name(cls):
         return 'enum_mig'
+
+
+class MaterializedModel(Model):
+    date_time = DateTimeField()
+    date = MaterializedField(DateField(), 'toDate(date_time)')
+
+    engine = MergeTree('date', ('date',))
+
+    @classmethod
+    def table_name(cls):
+        return 'materalized_date'
+
+
+class AliasModel(Model):
+    date = DateField()
+    date_alias = AliasField(DateField(), 'date')
+
+    engine = MergeTree('date', ('date',))
+
+    @classmethod
+    def table_name(cls):
+        return 'alias_date'