RAMEN-206 Support LowCardinality in infi.clickhouse_orm

src/infi/clickhouse_orm/database.py

@@ -122,6 +122,8 @@ class Database(object):
         self.server_timezone = self._get_server_timezone() if self.server_version > (1, 1, 53981) else pytz.utc
         # Versions 19.1.16 and above support codec compression
         self.has_codec_support = self.server_version >= (19, 1, 16)
+        # Version 19.0 and above support LowCardinality
+        self.has_low_cardinality_support = self.server_version >= (19, 0)
 
     def create_database(self):
         '''

src/infi/clickhouse_orm/fields.py

@@ -6,9 +6,10 @@ import pytz
 from calendar import timegm
 from decimal import Decimal, localcontext
 from uuid import UUID
-
+from logging import getLogger
 from .utils import escape, parse_array, comma_join
 
+logger = getLogger('clickhouse_orm')
 
 class Field(object):
     '''
@@ -520,3 +521,41 @@ class NullableField(Field):
             if self.codec and db and db.has_codec_support:
                 sql+= ' CODEC(%s)' % self.codec
         return sql
+
+
+class LowCardinalityField(Field):
+
+    def __init__(self, inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None):
+        assert isinstance(inner_field, Field), "The first argument of LowCardinalityField must be a Field instance. Not: {}".format(inner_field)
+        assert not isinstance(inner_field, LowCardinalityField), "LowCardinality inner fields are not supported by the ORM"
+        assert not isinstance(inner_field, ArrayField), "Array field inside LowCardinality are not supported by the ORM. Use Array(LowCardinality) instead"
+        self.inner_field = inner_field
+        self.class_default = self.inner_field.class_default
+        super(LowCardinalityField, self).__init__(default, alias, materialized, readonly, codec)
+
+    def to_python(self, value, timezone_in_use):
+        return self.inner_field.to_python(value, timezone_in_use)
+
+    def validate(self, value):
+        self.inner_field.validate(value)
+
+    def to_db_string(self, value, quote=True):
+        return self.inner_field.to_db_string(value, quote=quote)
+
+    def get_sql(self, with_default_expression=True, db=None):
+        if db and db.has_low_cardinality_support:
+            sql = 'LowCardinality(%s)' % self.inner_field.get_sql(with_default_expression=False)
+        else:
+            sql = self.inner_field.get_sql(with_default_expression=False)
+            logger.warning('LowCardinalityField not supported on clickhouse-server version < 19.0 using {} as fallback'.format(self.inner_field.__class__.__name__))
+        if with_default_expression:
+            if self.alias:
+                sql += ' ALIAS %s' % self.alias
+            elif self.materialized:
+                sql += ' MATERIALIZED %s' % self.materialized
+            elif self.default:
+                default = self.to_db_string(self.default)
+                sql += ' DEFAULT %s' % default
+            if self.codec and db and db.has_codec_support:
+                sql+= ' CODEC(%s)' % self.codec
+        return sql

tests/sample_migrations/0015.py

@@ -3,4 +3,5 @@ from ..test_migrations import *
 
 operations = [
     migrations.AlterTable(Model4_compressed),
+    migrations.AlterTable(Model2LowCardinality)
 ]

tests/test_migrations.py

@@ -96,6 +96,15 @@ class MigrationsTestCase(unittest.TestCase):
                           [('date', 'Date'), ('int_field', 'Int8'), ('date_alias', 'Date'), ('int_field_plus_one', 'Int8')])
         self.database.migrate('tests.sample_migrations', 15)
         self.assertTrue(self.tableExists(Model4_compressed))
+        if self.database.has_low_cardinality_support:
+            self.assertEqual(self.getTableFields(Model2LowCardinality),
+                             [('date', 'Date'), ('f1', 'LowCardinality(Int32)'), ('f3', 'LowCardinality(Float32)'),
+                              ('f2', 'LowCardinality(String)'), ('f4', 'LowCardinality(Nullable(String))'), ('f5', 'Array(LowCardinality(UInt64))')])
+        else:
+            logging.warning('No support for low cardinality')
+            self.assertEqual(self.getTableFields(Model2),
+                             [('date', 'Date'), ('f1', 'Int32'), ('f3', 'Float32'), ('f2', 'String'), ('f4', 'Nullable(String)'),
+                              ('f5', 'Array(UInt64)')])
 
 
 # Several different models with the same table name, to simulate a table that changes over time
@@ -270,3 +279,18 @@ class Model4_compressed(Model):
     @classmethod
     def table_name(cls):
         return 'model4'
+
+
+class Model2LowCardinality(Model):
+    date = DateField()
+    f1 = LowCardinalityField(Int32Field())
+    f3 = LowCardinalityField(Float32Field())
+    f2 = LowCardinalityField(StringField())
+    f4 = LowCardinalityField(NullableField(StringField()))
+    f5 = ArrayField(LowCardinalityField(UInt64Field()))
+
+    engine = MergeTree('date', ('date',))
+
+    @classmethod
+    def table_name(cls):
+        return 'mig'