From 874eae88a0bc49eabfa2a95e8564269133368b44 Mon Sep 17 00:00:00 2001 From: M1ha-Shvn Date: Tue, 30 Mar 2021 16:09:18 +0500 Subject: [PATCH] Fixed compatibility with celery JSON serializer --- .travis.yml | 4 +++ setup.py | 2 +- src/django_clickhouse/tasks.py | 14 +++++----- tests/test_models.py | 4 ++- tests/test_sync.py | 47 +++++++++++++++++++++++++++++++++- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b775fa2..da00d5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,6 +51,10 @@ env: - PG=10 DJANGO=3.0 - PG=11 DJANGO=3.0 - PG=12 DJANGO=3.0 + - PG=9.6 DJANGO=3.1 + - PG=10 DJANGO=3.1 + - PG=11 DJANGO=3.1 + - PG=12 DJANGO=3.1 before_install: # Use default PostgreSQL 11 port diff --git a/setup.py b/setup.py index b9264bc..bad7ac8 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ with open('requirements.txt') as f: setup( name='django-clickhouse', - version='1.0.2', + version='1.0.3', packages=['django_clickhouse'], package_dir={'': 'src'}, url='https://github.com/carrotquest/django-clickhouse', diff --git a/src/django_clickhouse/tasks.py b/src/django_clickhouse/tasks.py index b23baed..0d8120d 100644 --- a/src/django_clickhouse/tasks.py +++ b/src/django_clickhouse/tasks.py @@ -7,16 +7,19 @@ from infi.clickhouse_orm.utils import import_submodules from django_clickhouse.clickhouse_models import ClickHouseModel from .configuration import config -from .utils import get_subclasses +from .utils import get_subclasses, lazy_class_import @shared_task(queue=config.CELERY_QUEUE) def sync_clickhouse_model(model_cls) -> None: """ Syncs one batch of given ClickHouseModel - :param model_cls: ClickHouseModel subclass + :param model_cls: ClickHouseModel subclass or python path to it :return: None """ + model_cls = lazy_class_import(model_cls) + + # If sync will not finish it is not fatal to set up sync period here: sync will be executed next time model_cls.get_storage().set_last_sync_time(model_cls.get_import_key(), datetime.datetime.now()) model_cls.sync_batch_from_storage() @@ -37,9 +40,8 @@ def clickhouse_auto_sync(): except ImportError: pass - # Start for cls in get_subclasses(ClickHouseModel, recursive=True): if cls.need_sync(): - # Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации - # Она выполнится следующей таской через интервал. - sync_clickhouse_model.delay(cls) + # I pass class as a string in order to make it JSON serializable + cls_path = "%s.%s" % (cls.__module__, cls.__name__) + sync_clickhouse_model.delay(cls_path) diff --git a/tests/test_models.py b/tests/test_models.py index c9b39be..b89957f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,7 +2,7 @@ import datetime from unittest import skipIf import django -from django.test import TransactionTestCase +from django.test import TransactionTestCase, TestCase from django.utils.timezone import now from tests.clickhouse_models import ClickHouseTestModel, ClickHouseSecondTestModel, ClickHouseCollapseTestModel, \ @@ -25,6 +25,8 @@ class TestOperations(TransactionTestCase): fixtures = ['test_model'] django_model = TestModel clickhouse_model = ClickHouseTestModel + + databases = ['default', 'secondary'] db_alias = 'default' multi_db = True diff --git a/tests/test_sync.py b/tests/test_sync.py index 32f9337..1958e2f 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -2,15 +2,18 @@ import datetime import logging from subprocess import Popen from time import sleep -from unittest import expectedFailure, skip +from unittest import expectedFailure, skip, mock import os from django.test import TransactionTestCase +from django.test.testcases import TestCase from django.utils.timezone import now from random import randint from django_clickhouse.database import connections from django_clickhouse.migrations import migrate_app +from django_clickhouse.storages import RedisStorage +from django_clickhouse.tasks import sync_clickhouse_model, clickhouse_auto_sync from django_clickhouse.utils import int_ranges from tests.clickhouse_models import ClickHouseTestModel, ClickHouseCollapseTestModel, ClickHouseMultiTestModel from tests.models import TestModel @@ -207,6 +210,48 @@ class KillTest(TransactionTestCase): self._check_data() +@mock.patch.object(ClickHouseTestModel, 'sync_batch_from_storage') +class SyncClickHouseModelTest(TestCase): + def test_model_as_class(self, sync_mock): + sync_clickhouse_model(ClickHouseTestModel) + sync_mock.assert_called() + + def test_model_as_string(self, sync_mock): + sync_clickhouse_model('tests.clickhouse_models.ClickHouseTestModel') + sync_mock.assert_called() + + @mock.patch.object(RedisStorage, 'set_last_sync_time') + def test_last_sync_time_called(self, storage_mock, _): + sync_clickhouse_model(ClickHouseTestModel) + storage_mock.assert_called() + self.assertEqual(2, len(storage_mock.call_args)) + self.assertEqual(storage_mock.call_args[0][0], 'ClickHouseTestModel') + self.assertIsInstance(storage_mock.call_args[0][1], datetime.datetime) + + +@mock.patch.object(sync_clickhouse_model, 'delay') +class ClickHouseAutoSyncTest(TestCase): + @mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel]) + @mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True) + def test_needs_sync_enabled(self, need_sync_mock, get_subclasses_mock, sync_delay_mock): + clickhouse_auto_sync() + sync_delay_mock.assert_called_with('tests.clickhouse_models.ClickHouseTestModel') + + @mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel]) + @mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=False) + def test_does_not_need_sync(self, need_sync_mock, get_subclasses_mock, sync_delay_mock): + clickhouse_auto_sync() + sync_delay_mock.assert_not_called() + + @mock.patch('django_clickhouse.tasks.get_subclasses', + return_value=[ClickHouseTestModel, ClickHouseCollapseTestModel]) + @mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True) + @mock.patch.object(ClickHouseCollapseTestModel, 'need_sync', return_value=True) + def test_multiple_models(self, need_sync_1_mock, need_sync_2_mock, get_subclasses_mock, sync_delay_mock): + clickhouse_auto_sync() + self.assertEqual(2, sync_delay_mock.call_count) + + # Used to profile sync execution time. Disabled by default @skip class ProfileTest(TransactionTestCase):