Infinidat/infi.clickhouse_orm
1
1
Fork 0
mirror of https://github.com/Infinidat/infi.clickhouse_orm.git synced 2024-11-22 09:06:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rewritten Alias and Materialized fields to field parameters like default.

Browse Source
This commit is contained in:
M1ha 2017-01-27 10:42:37 +05:00
parent 41e73a5cbb
commit 2509b5b2e3
6 changed files with 59 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.rst
View File

@ -31,6 +31,8 @@ Models are defined in a way reminiscent of Django's ORM::
engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday'))
It is possible to provide a default value for a field, instead of its "natural" default (empty string for string fields, zero for numeric fields etc.).
It is always possible to pass alias or materialized parameters. See below for usage examples.
Only one of default, alias and materialized parameters can be provided
See below for the supported field types and table engines.
@ -208,8 +210,6 @@ Float64Field Float64 float
Enum8Field Enum8 Enum See below
Enum16Field Enum16 Enum See below
ArrayField Array list See below
AliasField See below See below See below
MaterializedField See below See below See below
=================== ========== ================= ===================================================
Working with enum fields
@ -270,9 +270,9 @@ Usage::
class Event(models.Model):
created = fields.DateTimeField()
created_date = fields.MaterializedField(fields.DateTimeField(), 'toDate(created)')
name = StringField()
username = AliasField(StringField(), 'name')
created_date = fields.DateTimeField(materialized='toDate(created)')
name = fields.StringField()
username = fields.StringField(alias='name')
engine = engines.MergeTree('created_date', ('created_date', 'created'))

101
src/infi/clickhouse_orm/fields.py
View File

@ -12,13 +12,17 @@ class Field(object):
class_default = 0
db_type = None
# This flag indicates, if we should take this field value when inserting data
insertable = True
def __init__(self, default=None, alias=None, materialized=None):
assert (None, None) in {(default, alias), (alias, materialized), (default, materialized)}, \
"Only one of default, alias and materialized parameters can be given"
assert alias is None or isinstance(alias, str), "Alias field must be string field name, if given"
assert materialized is None or isinstance(materialized, str), "Materialized field must be string, if given"
def __init__(self, default=None):
self.creation_counter = Field.creation_counter
Field.creation_counter += 1
self.default = self.class_default if default is None else default
self.alias = alias
self.materialized = materialized
def to_python(self, value):
'''
@ -51,13 +55,22 @@ class Field(object):
def get_sql(self, with_default=True):
'''
Returns an SQL expression describing the field (e.g. for CREATE TABLE).
:param with_default: If True, adds default value to sql.
It doesn't affect fields with alias and materialized values.
'''
if with_default:
if self.alias:
return '%s ALIAS %s' % (self.db_type, self.alias)
elif self.materialized:
return '%s MATERIALIZED %s' % (self.db_type, self.materialized)
elif with_default:
default = self.to_db_string(self.default)
return '%s DEFAULT %s' % (self.db_type, default)
else:
return self.db_type
def is_insertable(self):
return self.alias is None and self.materialized is None
class StringField(Field):
@ -210,11 +223,11 @@ class Float64Field(BaseFloatField):
class BaseEnumField(Field):
def __init__(self, enum_cls, default=None):
def __init__(self, enum_cls, default=None, alias=None, materialized=None):
self.enum_cls = enum_cls
if default is None:
default = list(enum_cls)[0]
super(BaseEnumField, self).__init__(default)
super(BaseEnumField, self).__init__(default, alias, materialized)
def to_python(self, value):
if isinstance(value, self.enum_cls):
@ -274,9 +287,9 @@ class ArrayField(Field):
class_default = []
def __init__(self, inner_field, default=None):
def __init__(self, inner_field, default=None, alias=None, materialized=None):
self.inner_field = inner_field
super(ArrayField, self).__init__(default)
super(ArrayField, self).__init__(default, alias, materialized)
def to_python(self, value):
if isinstance(value, text_type):
@ -299,75 +312,3 @@ class ArrayField(Field):
from .utils import escape
return 'Array(%s)' % self.inner_field.get_sql(with_default=False)
class RelativeField(Field):
insertable = False
def __init__(self, inner_field):
"""
Creates MATERIALIZED or ALIAS field
:param inner_field: Field subclass this field is acting like
"""
assert isinstance(inner_field, Field), "field must be Field subclass"
self.class_default = inner_field.class_default
self.default = inner_field.default
super(RelativeField, self).__init__()
self.inner_field = inner_field
def to_python(self, value):
return self.inner_field.to_python(value)
def validate(self, value):
return self.inner_field.validate(value)
def to_db_string(self, value, quote=True):
return self.inner_field.to_db_string(value, quote=quote)
class MaterializedField(RelativeField):
"""
Creates ClickHouse MATERIALIZED field. It doesn't contain real data in database, it is counted on the spot
https://clickhouse.yandex/reference_en.html#Default values
"""
def __init__(self, inner_field, code):
"""
Creates MATERIALIZED field
:param inner_field: Field subclass this field is acting like
:param code: ClickHouse code to execute when materialized field is called. See ClickHouse docs.
"""
super(MaterializedField, self).__init__(inner_field)
self._code = code
def get_sql(self, with_default=True):
"""
Generates SQL for create table command
:param with_default: This flag is inherited from Field model. Does nothing (MATERIALIZED have no default)
:return: Creation SQL string
"""
return '%s MATERIALIZED %s' % (self.inner_field.db_type, self._code)
class AliasField(RelativeField):
"""
Creates ClickHouse ALIAS field. It doesn't contain real data in database, only copies other one
https://clickhouse.yandex/reference_en.html#Default values
"""
def __init__(self, inner_field, base_field_name):
"""
Creates ALIAS field
:param inner_field: Field instance this field is acting like
:param base_field_name: Name of field, to which alias is built
"""
super(AliasField, self).__init__(inner_field)
self.base_field_name = base_field_name
def get_sql(self, with_default=True):
"""
Generates SQL for create table command
:param with_default: This flag is inherited from Field model. Does nothing (ALIAS have no default)
:return: Creation SQL string
"""
return '%s ALIAS %s' % (self.inner_field.db_type, self.base_field_name)

2
src/infi/clickhouse_orm/models.py
View File

@ -157,7 +157,7 @@ class Model(with_metaclass(ModelBase)):
'''
data = self.__dict__
fields = [f for f in self._fields if f[1].insertable] if insertable_only else self._fields
fields = [f for f in self._fields if f[1].is_insertable()] if insertable_only else self._fields
return '\t'.join(field.to_db_string(data[name], quote=False) for name, field in fields)

18
tests/test_alias_fields.py
View File

@ -43,15 +43,27 @@ class MaterializedFieldsTest(unittest.TestCase):
with self.assertRaises(ValueError):
instance.alias_date = value
def test_wrong_field(self):
with self.assertRaises(AssertionError):
StringField(alias=123)
def test_duplicate_default(self):
with self.assertRaises(AssertionError):
StringField(alias='str_field', default='with default')
with self.assertRaises(AssertionError):
StringField(alias='str_field', materialized='str_field')
class ModelWithAliasFields(Model):
int_field = Int32Field()
date_field = DateField()
str_field = StringField()
alias_str = AliasField(StringField(), 'str_field')
alias_int = MaterializedField(Int32Field(), 'int_field')
alias_date = MaterializedField(DateField(), 'date_field')
alias_str = StringField(alias='str_field')
alias_int = Int32Field(alias='int_field')
alias_date = DateField(alias='date_field')
engine = MergeTree('date_field', ('date_field',))

18
tests/test_materialized_fields.py
View File

@ -43,15 +43,27 @@ class MaterializedFieldsTest(unittest.TestCase):
with self.assertRaises(ValueError):
instance.mat_date = value
def test_wrong_field(self):
with self.assertRaises(AssertionError):
StringField(materialized=123)
def test_duplicate_default(self):
with self.assertRaises(AssertionError):
StringField(materialized='str_field', default='with default')
with self.assertRaises(AssertionError):
StringField(materialized='str_field', alias='str_field')
class ModelWithMaterializedFields(Model):
int_field = Int32Field()
date_time_field = DateTimeField()
str_field = StringField()
mat_str = MaterializedField(StringField(), 'lower(str_field)')
mat_int = MaterializedField(Int32Field(), 'abs(int_field)')
mat_date = MaterializedField(DateField(), 'toDate(date_time_field)')
mat_str = StringField(materialized='lower(str_field)')
mat_int = Int32Field(materialized='abs(int_field)')
mat_date = DateField(materialized='toDate(date_time_field)')
engine = MergeTree('mat_date', ('mat_date',))

4
tests/test_migrations.py
View File

@ -140,7 +140,7 @@ class EnumModel2(Model):
class MaterializedModel(Model):
date_time = DateTimeField()
date = MaterializedField(DateField(), 'toDate(date_time)')
date = DateField(materialized='toDate(date_time)')
engine = MergeTree('date', ('date',))
@ -151,7 +151,7 @@ class MaterializedModel(Model):
class AliasModel(Model):
date = DateField()
date_alias = AliasField(DateField(), 'date')
date_alias = DateField(alias='date')
engine = MergeTree('date', ('date',))