carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2024-11-22 00:56:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed problem for single lock for multiple models

Browse Source
This commit is contained in:
M1ha 2019-01-24 11:31:16 +05:00
parent 13a226501f
commit cc76166ae1
2 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/django_clickhouse/storages.py
View File

@ -164,7 +164,7 @@ class RedisStorage(with_metaclass(SingletonMeta, Storage)):
from redis import StrictRedis from redis import StrictRedis
self._redis = StrictRedis(**config.REDIS_CONFIG) self._redis = StrictRedis(**config.REDIS_CONFIG)
self._lock = None self._locks = {}
def register_operations(self, import_key, operation, *pks): def register_operations(self, import_key, operation, *pks):
key = self.REDIS_KEY_OPS_TEMPLATE.format(import_key=import_key) key = self.REDIS_KEY_OPS_TEMPLATE.format(import_key=import_key)
@ -193,14 +193,14 @@ class RedisStorage(with_metaclass(SingletonMeta, Storage)):
return [] return []
def get_lock(self, import_key, **kwargs): def get_lock(self, import_key, **kwargs):
if self._lock is None: if self._locks.get(import_key) is None:
from .redis import RedisLock from .redis import RedisLock
lock_key = self.REDIS_KEY_LOCK.format(import_key=import_key) lock_key = self.REDIS_KEY_LOCK.format(import_key=import_key)
lock_timeout = kwargs.get('lock_timeout', config.SYNC_DELAY * 10) lock_timeout = kwargs.get('lock_timeout', config.SYNC_DELAY * 10)
self._lock = RedisLock(self._redis, lock_key, timeout=lock_timeout, blocking_timeout=0.1, self._locks[import_key] = RedisLock(self._redis, lock_key, timeout=lock_timeout, blocking_timeout=0.1,
thread_local=False) thread_local=False)
return self._lock return self._locks[import_key]
def pre_sync(self, import_key, **kwargs): def pre_sync(self, import_key, **kwargs):
# Block process to be single threaded. Default sync delay is 10 * default sync delay. # Block process to be single threaded. Default sync delay is 10 * default sync delay.

14
tests/test_storages.py
View File

@ -2,7 +2,9 @@ import datetime
from django.test import TestCase from django.test import TestCase
from django_clickhouse.exceptions import RedisLockTimeoutError
from django_clickhouse.storages import RedisStorage from django_clickhouse.storages import RedisStorage
from tests.clickhouse_models import ClickHouseTestModel, ClickHouseCollapseTestModel
class StorageTest(TestCase): class StorageTest(TestCase):
@ -69,3 +71,15 @@ class StorageTest(TestCase):
self.assertEqual(2, self.storage.operations_count('test')) self.assertEqual(2, self.storage.operations_count('test'))
self.storage.register_operations_wrapped('test', 'insert', 100502) self.storage.register_operations_wrapped('test', 'insert', 100502)
self.assertEqual(3, self.storage.operations_count('test')) self.assertEqual(3, self.storage.operations_count('test'))
def test_locks(self):
# Test that multiple can acquire locks in parallel
# And single model can't
l = self.storage.get_lock(ClickHouseTestModel.get_import_key())
l.acquire()
with self.assertRaises(RedisLockTimeoutError):
l.acquire()
l2 = self.storage.get_lock(ClickHouseCollapseTestModel.get_import_key())
l2.acquire()