carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2025-06-25 23:43:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

Disabled generators to monitor time better

Browse Source
This commit is contained in:
M1ha 2019-02-05 10:46:29 +05:00
parent cc1502d4a4
commit df422d11f1
2 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/django_clickhouse/clickhouse_models.py
View File

@ -234,7 +234,9 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
if import_objects: if import_objects:
with statsd.timer(statsd_key.format('steps.get_insert_batch')): with statsd.timer(statsd_key.format('steps.get_insert_batch')):
batch = cls.get_insert_batch(import_objects) # NOTE I don't use generator pattern here, as it move all time into insert.
# That makes hard to understand where real problem is in monitoring
batch = tuple(cls.get_insert_batch(import_objects))
with statsd.timer(statsd_key.format('steps.insert')): with statsd.timer(statsd_key.format('steps.insert')):
cls.insert_batch(batch) cls.insert_batch(batch)
@ -312,7 +314,9 @@ class ClickHouseMultiModel(ClickHouseModel):
def _sub_model_func(model_cls): def _sub_model_func(model_cls):
model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__) model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(model_statsd_key.format('steps.get_insert_batch')): with statsd.timer(model_statsd_key.format('steps.get_insert_batch')):
batch = model_cls.get_insert_batch(import_objects) # NOTE I don't use generator pattern here, as it move all time into insert.
# That makes hard to understand where real problem is in monitoring
batch = tuple(model_cls.get_insert_batch(import_objects))
return model_cls, batch return model_cls, batch
res = exec_multi_arg_func(_sub_model_func, cls.sub_models, threads_count=len(cls.sub_models)) res = exec_multi_arg_func(_sub_model_func, cls.sub_models, threads_count=len(cls.sub_models))

4
src/django_clickhouse/engines.py
View File

@ -139,7 +139,9 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
statsd_key = "%s.sync.%s.steps.get_final_versions" % (config.STATSD_PREFIX, model_cls.__name__) statsd_key = "%s.sync.%s.steps.get_final_versions" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(statsd_key): with statsd.timer(statsd_key):
old_objs = self.get_final_versions(model_cls, new_objs) # NOTE I don't use generator pattern here, as it move all time into insert.
# That makes hard to understand where real problem is in monitoring
old_objs = tuple(self.get_final_versions(model_cls, new_objs))
# -1 sign has been set get_final_versions() # -1 sign has been set get_final_versions()
old_objs_versions = {} old_objs_versions = {}