carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2025-02-16 19:00:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Opportunity to pass database data as strings without formatting

Browse Source
This commit is contained in:
M1ha 2019-07-15 16:56:16 +05:00
parent 007ff70d5e
commit 85a785eb59
2 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/django_clickhouse/clickhouse_models.py
View File

@ -58,6 +58,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
sync_storage = None sync_storage = None
sync_delay = None sync_delay = None
sync_lock_timeout = None sync_lock_timeout = None
sync_formatted_tuples = False
# This attribute is initialized in metaclass, as it must get model class as a parameter # This attribute is initialized in metaclass, as it must get model class as a parameter
objects = None # type: QuerySet objects = None # type: QuerySet
@ -204,11 +205,11 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
def insert_batch(cls, batch): def insert_batch(cls, batch):
""" """
Inserts batch into database Inserts batch into database
:param batch: :param batch: Batch of tuples to insert
:return: :return:
""" """
if batch: if batch:
cls.get_database(for_write=True).insert_tuples(cls, batch) cls.get_database(for_write=True).insert_tuples(cls, batch, formatted=cls.sync_formatted_tuples)
@classmethod @classmethod
def sync_batch_from_storage(cls): def sync_batch_from_storage(cls):

23
src/django_clickhouse/database.py
View File

@ -67,13 +67,14 @@ class Database(InfiDatabase):
yield item yield item
def insert_tuples(self, model_class, model_tuples, batch_size=None): def insert_tuples(self, model_class, model_tuples, batch_size=None, formatted=False):
# type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int]) -> None # type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int], bool) -> None
""" """
Inserts model_class namedtuples Inserts model_class namedtuples
:param model_class: Clickhouse model, namedtuples are made from :param model_class: Clickhouse model, namedtuples are made from
:param model_tuples: An iterable of tuples to insert :param model_tuples: An iterable of tuples to insert
:param batch_size: Size of batch :param batch_size: Size of batch
:param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string
:return: None :return: None
""" """
tuples_iterator = iter(model_tuples) tuples_iterator = iter(model_tuples)
@ -90,17 +91,21 @@ class Database(InfiDatabase):
fields_dict = model_class.fields(writable=True) fields_dict = model_class.fields(writable=True)
statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__) statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__)
query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
% (self.db_name, model_class.table_name(), fields_list)
query_enc = query.encode('utf-8')
def tuple_to_csv(tup): def tuple_to_csv(tup):
return '\t'.join( if formatted:
fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False) str_gen = (getattr(tup, field_name) for field_name in first_tuple._fields)
for field_name in first_tuple._fields else:
) + '\n' str_gen = (fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
for field_name in first_tuple._fields)
return '%s\n' % '\t'.join(str_gen)
def gen(): def gen():
buf = BytesIO() buf = BytesIO()
query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
% (self.db_name, model_class.table_name(), fields_list)
query_enc = query.encode('utf-8')
buf.write(query_enc) buf.write(query_enc)
buf.write(tuple_to_csv(first_tuple).encode('utf-8')) buf.write(tuple_to_csv(first_tuple).encode('utf-8'))