carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2024-11-22 00:56:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixes in statsd keys

Browse Source
This commit is contained in:
M1ha 2018-12-19 12:11:55 +05:00
parent 578fc6c939
commit 38db27626c
3 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/django_clickhouse/clickhouse_models.py
View File

@ -190,26 +190,26 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
storage = cls.get_storage()
import_key = cls.get_import_key()
with statsd.timer(statsd_key.format('pre_sync')):
with statsd.timer(statsd_key.format('steps.pre_sync')):
storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout())
with statsd.timer(statsd_key.format('get_operations')):
with statsd.timer(statsd_key.format('steps.get_operations')):
operations = storage.get_operations(import_key, cls.get_sync_batch_size())
if operations:
with statsd.timer(statsd_key.format('get_sync_objects')):
with statsd.timer(statsd_key.format('steps.get_sync_objects')):
import_objects = cls.get_sync_objects(operations)
else:
import_objects = []
if import_objects:
with statsd.timer(statsd_key.format('get_insert_batch')):
with statsd.timer(statsd_key.format('steps.get_insert_batch')):
batch = cls.get_insert_batch(import_objects)
with statsd.timer(statsd_key.format('insert')):
with statsd.timer(statsd_key.format('steps.insert')):
cls.insert_batch(batch)
with statsd.timer(statsd_key.format('post_sync')):
with statsd.timer(statsd_key.format('steps.post_sync')):
storage.post_sync(import_key)
storage.set_last_sync_time(import_key, now())
@ -252,29 +252,29 @@ class ClickHouseMultiModel(ClickHouseModel):
storage = cls.get_storage()
import_key = cls.get_import_key()
with statsd.timer(statsd_key.format('pre_sync')):
with statsd.timer(statsd_key.format('steps.pre_sync')):
storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout())
with statsd.timer(statsd_key.format('get_operations')):
with statsd.timer(statsd_key.format('steps.get_operations')):
operations = storage.get_operations(import_key, cls.get_sync_batch_size())
if operations:
with statsd.timer(statsd_key.format('get_sync_objects')):
with statsd.timer(statsd_key.format('steps.get_sync_objects')):
import_objects = cls.get_sync_objects(operations)
else:
import_objects = []
if import_objects:
batches = {}
with statsd.timer(statsd_key.format('get_insert_batch')):
with statsd.timer(statsd_key.format('steps.get_insert_batch')):
for model_cls in cls.sub_models:
batches[model_cls] = model_cls.get_insert_batch(import_objects)
with statsd.timer(statsd_key.format('insert')):
with statsd.timer(statsd_key.format('steps.insert')):
for model_cls, batch in batches.items():
model_cls.insert_batch(batch)
with statsd.timer(statsd_key.format('post_sync')):
with statsd.timer(statsd_key.format('steps.post_sync')):
storage.post_sync(import_key)
storage.set_last_sync_time(import_key, now())

4
src/django_clickhouse/storages.py
View File

@ -116,7 +116,9 @@ class Storage:
logger.debug('django-clickhouse: registered %s on %d items (%s) to storage'
% (operation, len(pks), import_key))
return self.register_operations(import_key, operation, *pks)
statsd_key = "%s.sync.%s.register_operations" % (config.STATSD_PREFIX, import_key)
with statsd.timer(statsd_key):
return self.register_operations(import_key, operation, *pks)
def flush(self):
"""

4
src/django_clickhouse/tasks.py
View File

@ -17,9 +17,7 @@ def sync_clickhouse_converter(cls): # type: (ClickHouseModel) -> None
:param cls: ClickHouseModel subclass
:return: None
"""
statsd_key = "%s.sync.%s.time" % (config.STATSD_PREFIX, cls.__name__)
with statsd.timer(statsd_key):
cls.sync_batch_from_storage()
cls.sync_batch_from_storage()
@shared_task(queue=config.CELERY_QUEUE)