diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 939a12d..ae47bc0 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -122,6 +122,8 @@ class Database(object): self.server_timezone = self._get_server_timezone() if self.server_version > (1, 1, 53981) else pytz.utc # Versions 19.1.16 and above support codec compression self.has_codec_support = self.server_version >= (19, 1, 16) + # Version 19.0 and above support LowCardinality + self.has_low_cardinality_support = self.server_version >= (19, 0) def create_database(self): ''' diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index ae47efd..8b6d9f7 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -6,9 +6,10 @@ import pytz from calendar import timegm from decimal import Decimal, localcontext from uuid import UUID - +from logging import getLogger from .utils import escape, parse_array, comma_join +logger = getLogger('clickhouse_orm') class Field(object): ''' @@ -520,3 +521,41 @@ class NullableField(Field): if self.codec and db and db.has_codec_support: sql+= ' CODEC(%s)' % self.codec return sql + + +class LowCardinalityField(Field): + + def __init__(self, inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None): + assert isinstance(inner_field, Field), "The first argument of LowCardinalityField must be a Field instance. Not: {}".format(inner_field) + assert not isinstance(inner_field, LowCardinalityField), "LowCardinality inner fields are not supported by the ORM" + assert not isinstance(inner_field, ArrayField), "Array field inside LowCardinality are not supported by the ORM. Use Array(LowCardinality) instead" + self.inner_field = inner_field + self.class_default = self.inner_field.class_default + super(LowCardinalityField, self).__init__(default, alias, materialized, readonly, codec) + + def to_python(self, value, timezone_in_use): + return self.inner_field.to_python(value, timezone_in_use) + + def validate(self, value): + self.inner_field.validate(value) + + def to_db_string(self, value, quote=True): + return self.inner_field.to_db_string(value, quote=quote) + + def get_sql(self, with_default_expression=True, db=None): + if db and db.has_low_cardinality_support: + sql = 'LowCardinality(%s)' % self.inner_field.get_sql(with_default_expression=False) + else: + sql = self.inner_field.get_sql(with_default_expression=False) + logger.warning('LowCardinalityField not supported on clickhouse-server version < 19.0 using {} as fallback'.format(self.inner_field.__class__.__name__)) + if with_default_expression: + if self.alias: + sql += ' ALIAS %s' % self.alias + elif self.materialized: + sql += ' MATERIALIZED %s' % self.materialized + elif self.default: + default = self.to_db_string(self.default) + sql += ' DEFAULT %s' % default + if self.codec and db and db.has_codec_support: + sql+= ' CODEC(%s)' % self.codec + return sql diff --git a/tests/sample_migrations/0015.py b/tests/sample_migrations/0015.py index 1ab9b5b..be1d378 100644 --- a/tests/sample_migrations/0015.py +++ b/tests/sample_migrations/0015.py @@ -3,4 +3,5 @@ from ..test_migrations import * operations = [ migrations.AlterTable(Model4_compressed), + migrations.AlterTable(Model2LowCardinality) ] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 6c1af1a..f92b1e9 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -96,6 +96,15 @@ class MigrationsTestCase(unittest.TestCase): [('date', 'Date'), ('int_field', 'Int8'), ('date_alias', 'Date'), ('int_field_plus_one', 'Int8')]) self.database.migrate('tests.sample_migrations', 15) self.assertTrue(self.tableExists(Model4_compressed)) + if self.database.has_low_cardinality_support: + self.assertEqual(self.getTableFields(Model2LowCardinality), + [('date', 'Date'), ('f1', 'LowCardinality(Int32)'), ('f3', 'LowCardinality(Float32)'), + ('f2', 'LowCardinality(String)'), ('f4', 'LowCardinality(Nullable(String))'), ('f5', 'Array(LowCardinality(UInt64))')]) + else: + logging.warning('No support for low cardinality') + self.assertEqual(self.getTableFields(Model2), + [('date', 'Date'), ('f1', 'Int32'), ('f3', 'Float32'), ('f2', 'String'), ('f4', 'Nullable(String)'), + ('f5', 'Array(UInt64)')]) # Several different models with the same table name, to simulate a table that changes over time @@ -269,4 +278,19 @@ class Model4_compressed(Model): @classmethod def table_name(cls): - return 'model4' \ No newline at end of file + return 'model4' + + +class Model2LowCardinality(Model): + date = DateField() + f1 = LowCardinalityField(Int32Field()) + f3 = LowCardinalityField(Float32Field()) + f2 = LowCardinalityField(StringField()) + f4 = LowCardinalityField(NullableField(StringField())) + f5 = ArrayField(LowCardinalityField(UInt64Field())) + + engine = MergeTree('date', ('date',)) + + @classmethod + def table_name(cls): + return 'mig'