Merge pull request #23 from carrotquest/fix-json-serializer/issue-22

Fixed compatibility with celery JSON serializer
Added django 3.1 to Travis tests
This commit is contained in:
M1ha Shvn 2021-03-31 12:29:32 +05:00 committed by GitHub
commit 803626d883
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 62 additions and 9 deletions

View File

@ -51,6 +51,10 @@ env:
- PG=10 DJANGO=3.0
- PG=11 DJANGO=3.0
- PG=12 DJANGO=3.0
- PG=9.6 DJANGO=3.1
- PG=10 DJANGO=3.1
- PG=11 DJANGO=3.1
- PG=12 DJANGO=3.1
before_install:
# Use default PostgreSQL 11 port

View File

@ -13,7 +13,7 @@ with open('requirements.txt') as f:
setup(
name='django-clickhouse',
version='1.0.2',
version='1.0.3',
packages=['django_clickhouse'],
package_dir={'': 'src'},
url='https://github.com/carrotquest/django-clickhouse',

View File

@ -7,16 +7,19 @@ from infi.clickhouse_orm.utils import import_submodules
from django_clickhouse.clickhouse_models import ClickHouseModel
from .configuration import config
from .utils import get_subclasses
from .utils import get_subclasses, lazy_class_import
@shared_task(queue=config.CELERY_QUEUE)
def sync_clickhouse_model(model_cls) -> None:
"""
Syncs one batch of given ClickHouseModel
:param model_cls: ClickHouseModel subclass
:param model_cls: ClickHouseModel subclass or python path to it
:return: None
"""
model_cls = lazy_class_import(model_cls)
# If sync will not finish it is not fatal to set up sync period here: sync will be executed next time
model_cls.get_storage().set_last_sync_time(model_cls.get_import_key(), datetime.datetime.now())
model_cls.sync_batch_from_storage()
@ -37,9 +40,8 @@ def clickhouse_auto_sync():
except ImportError:
pass
# Start
for cls in get_subclasses(ClickHouseModel, recursive=True):
if cls.need_sync():
# Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации
# Она выполнится следующей таской через интервал.
sync_clickhouse_model.delay(cls)
# I pass class as a string in order to make it JSON serializable
cls_path = "%s.%s" % (cls.__module__, cls.__name__)
sync_clickhouse_model.delay(cls_path)

View File

@ -2,7 +2,7 @@ import datetime
from unittest import skipIf
import django
from django.test import TransactionTestCase
from django.test import TransactionTestCase, TestCase
from django.utils.timezone import now
from tests.clickhouse_models import ClickHouseTestModel, ClickHouseSecondTestModel, ClickHouseCollapseTestModel, \
@ -25,6 +25,8 @@ class TestOperations(TransactionTestCase):
fixtures = ['test_model']
django_model = TestModel
clickhouse_model = ClickHouseTestModel
databases = ['default', 'secondary']
db_alias = 'default'
multi_db = True

View File

@ -2,15 +2,18 @@ import datetime
import logging
from subprocess import Popen
from time import sleep
from unittest import expectedFailure, skip
from unittest import expectedFailure, skip, mock
import os
from django.test import TransactionTestCase
from django.test.testcases import TestCase
from django.utils.timezone import now
from random import randint
from django_clickhouse.database import connections
from django_clickhouse.migrations import migrate_app
from django_clickhouse.storages import RedisStorage
from django_clickhouse.tasks import sync_clickhouse_model, clickhouse_auto_sync
from django_clickhouse.utils import int_ranges
from tests.clickhouse_models import ClickHouseTestModel, ClickHouseCollapseTestModel, ClickHouseMultiTestModel
from tests.models import TestModel
@ -207,6 +210,48 @@ class KillTest(TransactionTestCase):
self._check_data()
@mock.patch.object(ClickHouseTestModel, 'sync_batch_from_storage')
class SyncClickHouseModelTest(TestCase):
def test_model_as_class(self, sync_mock):
sync_clickhouse_model(ClickHouseTestModel)
sync_mock.assert_called()
def test_model_as_string(self, sync_mock):
sync_clickhouse_model('tests.clickhouse_models.ClickHouseTestModel')
sync_mock.assert_called()
@mock.patch.object(RedisStorage, 'set_last_sync_time')
def test_last_sync_time_called(self, storage_mock, _):
sync_clickhouse_model(ClickHouseTestModel)
storage_mock.assert_called()
self.assertEqual(2, len(storage_mock.call_args))
self.assertEqual(storage_mock.call_args[0][0], 'ClickHouseTestModel')
self.assertIsInstance(storage_mock.call_args[0][1], datetime.datetime)
@mock.patch.object(sync_clickhouse_model, 'delay')
class ClickHouseAutoSyncTest(TestCase):
@mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel])
@mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True)
def test_needs_sync_enabled(self, need_sync_mock, get_subclasses_mock, sync_delay_mock):
clickhouse_auto_sync()
sync_delay_mock.assert_called_with('tests.clickhouse_models.ClickHouseTestModel')
@mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel])
@mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=False)
def test_does_not_need_sync(self, need_sync_mock, get_subclasses_mock, sync_delay_mock):
clickhouse_auto_sync()
sync_delay_mock.assert_not_called()
@mock.patch('django_clickhouse.tasks.get_subclasses',
return_value=[ClickHouseTestModel, ClickHouseCollapseTestModel])
@mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True)
@mock.patch.object(ClickHouseCollapseTestModel, 'need_sync', return_value=True)
def test_multiple_models(self, need_sync_1_mock, need_sync_2_mock, get_subclasses_mock, sync_delay_mock):
clickhouse_auto_sync()
self.assertEqual(2, sync_delay_mock.call_count)
# Used to profile sync execution time. Disabled by default
@skip
class ProfileTest(TransactionTestCase):