carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2024-11-22 00:56:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Opportunity to pass database data as strings without formatting

Browse Source
This commit is contained in:
M1ha 2019-07-15 16:56:16 +05:00
parent 007ff70d5e
commit 85a785eb59
2 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/django_clickhouse/clickhouse_models.py
View File

@ -58,6 +58,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
sync_storage = None
sync_delay = None
sync_lock_timeout = None
sync_formatted_tuples = False
# This attribute is initialized in metaclass, as it must get model class as a parameter
objects = None # type: QuerySet
@ -204,11 +205,11 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
def insert_batch(cls, batch):
"""
Inserts batch into database
:param batch:
:param batch: Batch of tuples to insert
:return:
"""
if batch:
cls.get_database(for_write=True).insert_tuples(cls, batch)
cls.get_database(for_write=True).insert_tuples(cls, batch, formatted=cls.sync_formatted_tuples)
@classmethod
def sync_batch_from_storage(cls):

23
src/django_clickhouse/database.py
View File

@ -67,13 +67,14 @@ class Database(InfiDatabase):
yield item
def insert_tuples(self, model_class, model_tuples, batch_size=None):
# type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int]) -> None
def insert_tuples(self, model_class, model_tuples, batch_size=None, formatted=False):
# type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int], bool) -> None
"""
Inserts model_class namedtuples
:param model_class: Clickhouse model, namedtuples are made from
:param model_tuples: An iterable of tuples to insert
:param batch_size: Size of batch
:param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string
:return: None
"""
tuples_iterator = iter(model_tuples)
@ -90,17 +91,21 @@ class Database(InfiDatabase):
fields_dict = model_class.fields(writable=True)
statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__)
query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
% (self.db_name, model_class.table_name(), fields_list)
query_enc = query.encode('utf-8')
def tuple_to_csv(tup):
return '\t'.join(
fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
for field_name in first_tuple._fields
) + '\n'
if formatted:
str_gen = (getattr(tup, field_name) for field_name in first_tuple._fields)
else:
str_gen = (fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
for field_name in first_tuple._fields)
return '%s\n' % '\t'.join(str_gen)
def gen():
buf = BytesIO()
query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
% (self.db_name, model_class.table_name(), fields_list)
query_enc = query.encode('utf-8')
buf.write(query_enc)
buf.write(tuple_to_csv(first_tuple).encode('utf-8'))