Added lock timeouts monitoring

This commit is contained in:
M1ha 2019-01-23 16:28:27 +05:00
parent 42f823d3a5
commit fae4247e79

View File

@ -210,10 +210,12 @@ class RedisStorage(Storage):
lock.acquire()
self._redis.set(lock_pid_key, os.getpid())
except RedisLockTimeoutError:
statsd.incr('clickhouse.sync.%s.lock.timeout' % import_key)
# Lock is busy. But If the process has been killed, I don't want to wait any more.
# Let's check if pid exists
pid = int(self._redis.get(lock_pid_key) or 0)
if pid and not check_pid(pid):
statsd.incr('clickhouse.sync.%s.lock.hard_release' % import_key)
logger.warning('django-clickhouse: hard releasing lock "%s" locked by pid %d (process is dead)'
% (import_key, pid))
self._redis.delete(lock_pid_key)