carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2025-07-10 16:12:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixes in statsd keys

Browse Source
This commit is contained in:
M1ha 2018-12-19 12:11:55 +05:00
parent 578fc6c939
commit 38db27626c
3 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/django_clickhouse/clickhouse_models.py
View File

@ -190,26 +190,26 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
storage = cls.get_storage() storage = cls.get_storage()
import_key = cls.get_import_key() import_key = cls.get_import_key()
with statsd.timer(statsd_key.format('pre_sync')): with statsd.timer(statsd_key.format('steps.pre_sync')):
storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout()) storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout())
with statsd.timer(statsd_key.format('get_operations')): with statsd.timer(statsd_key.format('steps.get_operations')):
operations = storage.get_operations(import_key, cls.get_sync_batch_size()) operations = storage.get_operations(import_key, cls.get_sync_batch_size())
if operations: if operations:
with statsd.timer(statsd_key.format('get_sync_objects')): with statsd.timer(statsd_key.format('steps.get_sync_objects')):
import_objects = cls.get_sync_objects(operations) import_objects = cls.get_sync_objects(operations)
else: else:
import_objects = [] import_objects = []
if import_objects: if import_objects:
with statsd.timer(statsd_key.format('get_insert_batch')): with statsd.timer(statsd_key.format('steps.get_insert_batch')):
batch = cls.get_insert_batch(import_objects) batch = cls.get_insert_batch(import_objects)
with statsd.timer(statsd_key.format('insert')): with statsd.timer(statsd_key.format('steps.insert')):
cls.insert_batch(batch) cls.insert_batch(batch)
with statsd.timer(statsd_key.format('post_sync')): with statsd.timer(statsd_key.format('steps.post_sync')):
storage.post_sync(import_key) storage.post_sync(import_key)
storage.set_last_sync_time(import_key, now()) storage.set_last_sync_time(import_key, now())
@ -252,29 +252,29 @@ class ClickHouseMultiModel(ClickHouseModel):
storage = cls.get_storage() storage = cls.get_storage()
import_key = cls.get_import_key() import_key = cls.get_import_key()
with statsd.timer(statsd_key.format('pre_sync')): with statsd.timer(statsd_key.format('steps.pre_sync')):
storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout()) storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout())
with statsd.timer(statsd_key.format('get_operations')): with statsd.timer(statsd_key.format('steps.get_operations')):
operations = storage.get_operations(import_key, cls.get_sync_batch_size()) operations = storage.get_operations(import_key, cls.get_sync_batch_size())
if operations: if operations:
with statsd.timer(statsd_key.format('get_sync_objects')): with statsd.timer(statsd_key.format('steps.get_sync_objects')):
import_objects = cls.get_sync_objects(operations) import_objects = cls.get_sync_objects(operations)
else: else:
import_objects = [] import_objects = []
if import_objects: if import_objects:
batches = {} batches = {}
with statsd.timer(statsd_key.format('get_insert_batch')): with statsd.timer(statsd_key.format('steps.get_insert_batch')):
for model_cls in cls.sub_models: for model_cls in cls.sub_models:
batches[model_cls] = model_cls.get_insert_batch(import_objects) batches[model_cls] = model_cls.get_insert_batch(import_objects)
with statsd.timer(statsd_key.format('insert')): with statsd.timer(statsd_key.format('steps.insert')):
for model_cls, batch in batches.items(): for model_cls, batch in batches.items():
model_cls.insert_batch(batch) model_cls.insert_batch(batch)
with statsd.timer(statsd_key.format('post_sync')): with statsd.timer(statsd_key.format('steps.post_sync')):
storage.post_sync(import_key) storage.post_sync(import_key)
storage.set_last_sync_time(import_key, now()) storage.set_last_sync_time(import_key, now())

4
src/django_clickhouse/storages.py
View File

@ -116,7 +116,9 @@ class Storage:
logger.debug('django-clickhouse: registered %s on %d items (%s) to storage' logger.debug('django-clickhouse: registered %s on %d items (%s) to storage'
% (operation, len(pks), import_key)) % (operation, len(pks), import_key))
return self.register_operations(import_key, operation, *pks) statsd_key = "%s.sync.%s.register_operations" % (config.STATSD_PREFIX, import_key)
with statsd.timer(statsd_key):
return self.register_operations(import_key, operation, *pks)
def flush(self): def flush(self):
""" """

4
src/django_clickhouse/tasks.py
View File

@ -17,9 +17,7 @@ def sync_clickhouse_converter(cls): # type: (ClickHouseModel) -> None
:param cls: ClickHouseModel subclass :param cls: ClickHouseModel subclass
:return: None :return: None
""" """
statsd_key = "%s.sync.%s.time" % (config.STATSD_PREFIX, cls.__name__) cls.sync_batch_from_storage()
with statsd.timer(statsd_key):
cls.sync_batch_from_storage()
@shared_task(queue=config.CELERY_QUEUE) @shared_task(queue=config.CELERY_QUEUE)