mirror of
https://github.com/Infinidat/infi.clickhouse_orm.git
synced 2024-11-10 19:36:33 +03:00
Added MaterializedField and AliasField
This commit is contained in:
parent
01cd88a938
commit
ca341ea997
|
@ -50,11 +50,11 @@ class Database(object):
|
|||
model_class = first_instance.__class__
|
||||
def gen():
|
||||
yield self._substitute('INSERT INTO $table FORMAT TabSeparated\n', model_class).encode('utf-8')
|
||||
yield (first_instance.to_tsv() + '\n').encode('utf-8')
|
||||
yield (first_instance.to_tsv(insertable_only=True) + '\n').encode('utf-8')
|
||||
# Collect lines in batches of batch_size
|
||||
batch = []
|
||||
for instance in i:
|
||||
batch.append(instance.to_tsv())
|
||||
batch.append(instance.to_tsv(insertable_only=True))
|
||||
if len(batch) >= batch_size:
|
||||
# Return the current batch of lines
|
||||
yield ('\n'.join(batch) + '\n').encode('utf-8')
|
||||
|
|
|
@ -12,6 +12,9 @@ class Field(object):
|
|||
class_default = 0
|
||||
db_type = None
|
||||
|
||||
# This flag indicates, if we should take this field value when inserting data
|
||||
insertable = True
|
||||
|
||||
def __init__(self, default=None):
|
||||
self.creation_counter = Field.creation_counter
|
||||
Field.creation_counter += 1
|
||||
|
@ -295,3 +298,76 @@ class ArrayField(Field):
|
|||
def get_sql(self, with_default=True):
|
||||
from .utils import escape
|
||||
return 'Array(%s)' % self.inner_field.get_sql(with_default=False)
|
||||
|
||||
|
||||
class RelativeField(Field):
|
||||
insertable = False
|
||||
|
||||
def __init__(self, inner_field):
|
||||
"""
|
||||
Creates MATERIALIZED or ALIAS field
|
||||
:param inner_field: Field subclass this field is acting like
|
||||
"""
|
||||
assert isinstance(inner_field, Field), "field must be Field subclass"
|
||||
self.class_default = inner_field.class_default
|
||||
self.default = inner_field.default
|
||||
super(RelativeField, self).__init__()
|
||||
self.inner_field = inner_field
|
||||
|
||||
def to_python(self, value):
|
||||
return self.inner_field.to_python(value)
|
||||
|
||||
def validate(self, value):
|
||||
return self.inner_field.validate(value)
|
||||
|
||||
def to_db_string(self, value, quote=True):
|
||||
return self.inner_field.to_db_string(value, quote=quote)
|
||||
|
||||
|
||||
class MaterializedField(RelativeField):
|
||||
"""
|
||||
Creates ClickHouse MATERIALIZED field. It doesn't contain real data in database, it is counted on the spot
|
||||
https://clickhouse.yandex/reference_en.html#Default values
|
||||
"""
|
||||
|
||||
def __init__(self, inner_field, code):
|
||||
"""
|
||||
Creates MATERIALIZED field
|
||||
:param inner_field: Field subclass this field is acting like
|
||||
:param code: ClickHouse code to execute when materialized field is called. See ClickHouse docs.
|
||||
"""
|
||||
super(MaterializedField, self).__init__(inner_field)
|
||||
|
||||
self._code = code
|
||||
|
||||
def get_sql(self, with_default=True):
|
||||
"""
|
||||
Generates SQL for create table command
|
||||
:param with_default: This flag is inherited from Field model. Does nothing (MATERIALIZED have no default)
|
||||
:return: Creation SQL string
|
||||
"""
|
||||
return '%s MATERIALIZED %s' % (self.inner_field.db_type, self._code)
|
||||
|
||||
|
||||
class AliasField(RelativeField):
|
||||
"""
|
||||
Creates ClickHouse ALIAS field. It doesn't contain real data in database, only copies other one
|
||||
https://clickhouse.yandex/reference_en.html#Default values
|
||||
"""
|
||||
|
||||
def __init__(self, inner_field, base_field_name):
|
||||
"""
|
||||
Creates ALIAS field
|
||||
:param inner_field: Field instance this field is acting like
|
||||
:param base_field_name: Name of field, to which alias is built
|
||||
"""
|
||||
super(AliasField, self).__init__(inner_field)
|
||||
self.base_field_name = base_field_name
|
||||
|
||||
def get_sql(self, with_default=True):
|
||||
"""
|
||||
Generates SQL for create table command
|
||||
:param with_default: This flag is inherited from Field model. Does nothing (ALIAS have no default)
|
||||
:return: Creation SQL string
|
||||
"""
|
||||
return '%s ALIAS %s' % (self.inner_field.db_type, self.base_field_name)
|
||||
|
|
|
@ -150,9 +150,14 @@ class Model(with_metaclass(ModelBase)):
|
|||
kwargs[name] = next(values)
|
||||
return cls(**kwargs)
|
||||
|
||||
def to_tsv(self):
|
||||
def to_tsv(self, insertable_only=False):
|
||||
'''
|
||||
Returns the instance's column values as a tab-separated line. A newline is not included.
|
||||
:param bool insertable_only: If True, returns only fields, that can be inserted into database
|
||||
'''
|
||||
data = self.__dict__
|
||||
return '\t'.join(field.to_db_string(data[name], quote=False) for name, field in self._fields)
|
||||
|
||||
fields = [f for f in self._fields if f[1].insertable] if insertable_only else self._fields
|
||||
return '\t'.join(field.to_db_string(data[name], quote=False) for name, field in fields)
|
||||
|
||||
|
||||
|
|
6
tests/sample_migrations/0008.py
Normal file
6
tests/sample_migrations/0008.py
Normal file
|
@ -0,0 +1,6 @@
|
|||
from infi.clickhouse_orm import migrations
|
||||
from ..test_migrations import *
|
||||
|
||||
operations = [
|
||||
migrations.CreateTable(MaterializedModel)
|
||||
]
|
6
tests/sample_migrations/0009.py
Normal file
6
tests/sample_migrations/0009.py
Normal file
|
@ -0,0 +1,6 @@
|
|||
from infi.clickhouse_orm import migrations
|
||||
from ..test_migrations import *
|
||||
|
||||
operations = [
|
||||
migrations.CreateTable(AliasModel)
|
||||
]
|
57
tests/test_alias_fields.py
Normal file
57
tests/test_alias_fields.py
Normal file
|
@ -0,0 +1,57 @@
|
|||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from infi.clickhouse_orm.database import Database
|
||||
from infi.clickhouse_orm.models import Model
|
||||
from infi.clickhouse_orm.fields import *
|
||||
from infi.clickhouse_orm.engines import *
|
||||
|
||||
|
||||
class MaterializedFieldsTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.database = Database('test-db')
|
||||
self.database.create_table(ModelWithAliasFields)
|
||||
|
||||
def tearDown(self):
|
||||
self.database.drop_database()
|
||||
|
||||
def test_insert_and_select(self):
|
||||
instance = ModelWithAliasFields(
|
||||
date_field='2016-08-30',
|
||||
int_field=-10,
|
||||
str_field='TEST'
|
||||
)
|
||||
self.database.insert([instance])
|
||||
# We can't select * from table, as it doesn't select materialized and alias fields
|
||||
query = 'SELECT date_field, int_field, str_field, alias_int, alias_date, alias_str' \
|
||||
' FROM $db.%s ORDER BY alias_date' % ModelWithAliasFields.table_name()
|
||||
for model_cls in (ModelWithAliasFields, None):
|
||||
results = list(self.database.select(query, model_cls))
|
||||
self.assertEquals(len(results), 1)
|
||||
self.assertEquals(results[0].date_field, instance.date_field)
|
||||
self.assertEquals(results[0].int_field, instance.int_field)
|
||||
self.assertEquals(results[0].str_field, instance.str_field)
|
||||
self.assertEquals(results[0].alias_int, instance.int_field)
|
||||
self.assertEquals(results[0].alias_str, instance.str_field)
|
||||
self.assertEquals(results[0].alias_date, instance.date_field)
|
||||
|
||||
def test_assignment_error(self):
|
||||
# I can't prevent assigning at all, in case db.select statements with model provided sets model fields.
|
||||
instance = ModelWithAliasFields()
|
||||
for value in ('x', [date.today()], ['aaa'], [None]):
|
||||
with self.assertRaises(ValueError):
|
||||
instance.alias_date = value
|
||||
|
||||
|
||||
class ModelWithAliasFields(Model):
|
||||
int_field = Int32Field()
|
||||
date_field = DateField()
|
||||
str_field = StringField()
|
||||
|
||||
alias_str = AliasField(StringField(), 'str_field')
|
||||
alias_int = MaterializedField(Int32Field(), 'int_field')
|
||||
alias_date = MaterializedField(DateField(), 'date_field')
|
||||
|
||||
engine = MergeTree('date_field', ('date_field',))
|
||||
|
57
tests/test_materialized_fields.py
Normal file
57
tests/test_materialized_fields.py
Normal file
|
@ -0,0 +1,57 @@
|
|||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from infi.clickhouse_orm.database import Database
|
||||
from infi.clickhouse_orm.models import Model
|
||||
from infi.clickhouse_orm.fields import *
|
||||
from infi.clickhouse_orm.engines import *
|
||||
|
||||
|
||||
class MaterializedFieldsTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.database = Database('test-db')
|
||||
self.database.create_table(ModelWithMaterializedFields)
|
||||
|
||||
def tearDown(self):
|
||||
self.database.drop_database()
|
||||
|
||||
def test_insert_and_select(self):
|
||||
instance = ModelWithMaterializedFields(
|
||||
date_time_field='2016-08-30 11:00:00',
|
||||
int_field=-10,
|
||||
str_field='TEST'
|
||||
)
|
||||
self.database.insert([instance])
|
||||
# We can't select * from table, as it doesn't select materialized and alias fields
|
||||
query = 'SELECT date_time_field, int_field, str_field, mat_int, mat_date, mat_str' \
|
||||
' FROM $db.%s ORDER BY mat_date' % ModelWithMaterializedFields.table_name()
|
||||
for model_cls in (ModelWithMaterializedFields, None):
|
||||
results = list(self.database.select(query, model_cls))
|
||||
self.assertEquals(len(results), 1)
|
||||
self.assertEquals(results[0].date_time_field, instance.date_time_field)
|
||||
self.assertEquals(results[0].int_field, instance.int_field)
|
||||
self.assertEquals(results[0].str_field, instance.str_field)
|
||||
self.assertEquals(results[0].mat_int, abs(instance.int_field))
|
||||
self.assertEquals(results[0].mat_str, instance.str_field.lower())
|
||||
self.assertEquals(results[0].mat_date, instance.date_time_field.date())
|
||||
|
||||
def test_assignment_error(self):
|
||||
# I can't prevent assigning at all, in case db.select statements with model provided sets model fields.
|
||||
instance = ModelWithMaterializedFields()
|
||||
for value in ('x', [date.today()], ['aaa'], [None]):
|
||||
with self.assertRaises(ValueError):
|
||||
instance.mat_date = value
|
||||
|
||||
|
||||
class ModelWithMaterializedFields(Model):
|
||||
int_field = Int32Field()
|
||||
date_time_field = DateTimeField()
|
||||
str_field = StringField()
|
||||
|
||||
mat_str = MaterializedField(StringField(), 'lower(str_field)')
|
||||
mat_int = MaterializedField(Int32Field(), 'abs(int_field)')
|
||||
mat_date = MaterializedField(DateField(), 'toDate(date_time_field)')
|
||||
|
||||
engine = MergeTree('mat_date', ('mat_date',))
|
||||
|
|
@ -60,6 +60,15 @@ class MigrationsTestCase(unittest.TestCase):
|
|||
self.assertTrue(self.tableExists(EnumModel1))
|
||||
self.assertEquals(self.getTableFields(EnumModel2),
|
||||
[('date', 'Date'), ('f1', "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")])
|
||||
self.database.migrate('tests.sample_migrations', 8)
|
||||
self.assertTrue(self.tableExists(MaterializedModel))
|
||||
self.assertEquals(self.getTableFields(MaterializedModel),
|
||||
[('date_time', "DateTime"), ('date', 'Date')])
|
||||
self.database.migrate('tests.sample_migrations', 9)
|
||||
self.assertTrue(self.tableExists(AliasModel))
|
||||
self.assertEquals(self.getTableFields(AliasModel),
|
||||
[('date', 'Date'), ('date_alias', "Date")])
|
||||
|
||||
|
||||
# Several different models with the same table name, to simulate a table that changes over time
|
||||
|
||||
|
@ -127,3 +136,25 @@ class EnumModel2(Model):
|
|||
@classmethod
|
||||
def table_name(cls):
|
||||
return 'enum_mig'
|
||||
|
||||
|
||||
class MaterializedModel(Model):
|
||||
date_time = DateTimeField()
|
||||
date = MaterializedField(DateField(), 'toDate(date_time)')
|
||||
|
||||
engine = MergeTree('date', ('date',))
|
||||
|
||||
@classmethod
|
||||
def table_name(cls):
|
||||
return 'materalized_date'
|
||||
|
||||
|
||||
class AliasModel(Model):
|
||||
date = DateField()
|
||||
date_alias = AliasField(DateField(), 'date')
|
||||
|
||||
engine = MergeTree('date', ('date',))
|
||||
|
||||
@classmethod
|
||||
def table_name(cls):
|
||||
return 'alias_date'
|
Loading…
Reference in New Issue
Block a user