Merge pull request #23 from carrotquest/fix-json-serializer/issue-22

Fixed compatibility with celery JSON serializer
Added django 3.1 to Travis tests

.travis.yml

@@ -51,6 +51,10 @@ env:
   - PG=10 DJANGO=3.0
   - PG=11 DJANGO=3.0
   - PG=12 DJANGO=3.0
+  - PG=9.6 DJANGO=3.1
+  - PG=10 DJANGO=3.1
+  - PG=11 DJANGO=3.1
+  - PG=12 DJANGO=3.1
 
 before_install:
   # Use default PostgreSQL 11 port

setup.py

@@ -13,7 +13,7 @@ with open('requirements.txt') as f:
 
 setup(
     name='django-clickhouse',
-    version='1.0.2',
+    version='1.0.3',
     packages=['django_clickhouse'],
     package_dir={'': 'src'},
     url='https://github.com/carrotquest/django-clickhouse',

src/django_clickhouse/tasks.py

@@ -7,16 +7,19 @@ from infi.clickhouse_orm.utils import import_submodules
 
 from django_clickhouse.clickhouse_models import ClickHouseModel
 from .configuration import config
-from .utils import get_subclasses
+from .utils import get_subclasses, lazy_class_import
 
 
 @shared_task(queue=config.CELERY_QUEUE)
 def sync_clickhouse_model(model_cls) -> None:
     """
     Syncs one batch of given ClickHouseModel
-    :param model_cls: ClickHouseModel subclass
+    :param model_cls: ClickHouseModel subclass or python path to it
     :return: None
     """
+    model_cls = lazy_class_import(model_cls)
+
+    # If sync will not finish it is not fatal to set up sync period here: sync will be executed next time
     model_cls.get_storage().set_last_sync_time(model_cls.get_import_key(), datetime.datetime.now())
     model_cls.sync_batch_from_storage()
 
@@ -37,9 +40,8 @@ def clickhouse_auto_sync():
         except ImportError:
             pass
 
-    # Start
     for cls in get_subclasses(ClickHouseModel, recursive=True):
         if cls.need_sync():
-            # Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации
-            # Она выполнится следующей таской через интервал.
-            sync_clickhouse_model.delay(cls)
+            # I pass class as a string in order to make it JSON serializable
+            cls_path = "%s.%s" % (cls.__module__, cls.__name__)
+            sync_clickhouse_model.delay(cls_path)

tests/test_models.py

@@ -2,7 +2,7 @@ import datetime
 from unittest import skipIf
 
 import django
-from django.test import TransactionTestCase
+from django.test import TransactionTestCase, TestCase
 from django.utils.timezone import now
 
 from tests.clickhouse_models import ClickHouseTestModel, ClickHouseSecondTestModel, ClickHouseCollapseTestModel, \
@@ -25,6 +25,8 @@ class TestOperations(TransactionTestCase):
     fixtures = ['test_model']
     django_model = TestModel
     clickhouse_model = ClickHouseTestModel
+
+    databases = ['default', 'secondary']
     db_alias = 'default'
     multi_db = True
 

tests/test_sync.py

@@ -2,15 +2,18 @@ import datetime
 import logging
 from subprocess import Popen
 from time import sleep
-from unittest import expectedFailure, skip
+from unittest import expectedFailure, skip, mock
 
 import os
 from django.test import TransactionTestCase
+from django.test.testcases import TestCase
 from django.utils.timezone import now
 from random import randint
 
 from django_clickhouse.database import connections
 from django_clickhouse.migrations import migrate_app
+from django_clickhouse.storages import RedisStorage
+from django_clickhouse.tasks import sync_clickhouse_model, clickhouse_auto_sync
 from django_clickhouse.utils import int_ranges
 from tests.clickhouse_models import ClickHouseTestModel, ClickHouseCollapseTestModel, ClickHouseMultiTestModel
 from tests.models import TestModel
@@ -207,6 +210,48 @@ class KillTest(TransactionTestCase):
         self._check_data()
 
 
+@mock.patch.object(ClickHouseTestModel, 'sync_batch_from_storage')
+class SyncClickHouseModelTest(TestCase):
+    def test_model_as_class(self, sync_mock):
+        sync_clickhouse_model(ClickHouseTestModel)
+        sync_mock.assert_called()
+
+    def test_model_as_string(self, sync_mock):
+        sync_clickhouse_model('tests.clickhouse_models.ClickHouseTestModel')
+        sync_mock.assert_called()
+
+    @mock.patch.object(RedisStorage, 'set_last_sync_time')
+    def test_last_sync_time_called(self, storage_mock, _):
+        sync_clickhouse_model(ClickHouseTestModel)
+        storage_mock.assert_called()
+        self.assertEqual(2, len(storage_mock.call_args))
+        self.assertEqual(storage_mock.call_args[0][0], 'ClickHouseTestModel')
+        self.assertIsInstance(storage_mock.call_args[0][1], datetime.datetime)
+
+
+@mock.patch.object(sync_clickhouse_model, 'delay')
+class ClickHouseAutoSyncTest(TestCase):
+    @mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel])
+    @mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True)
+    def test_needs_sync_enabled(self, need_sync_mock, get_subclasses_mock, sync_delay_mock):
+        clickhouse_auto_sync()
+        sync_delay_mock.assert_called_with('tests.clickhouse_models.ClickHouseTestModel')
+
+    @mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel])
+    @mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=False)
+    def test_does_not_need_sync(self, need_sync_mock, get_subclasses_mock, sync_delay_mock):
+        clickhouse_auto_sync()
+        sync_delay_mock.assert_not_called()
+
+    @mock.patch('django_clickhouse.tasks.get_subclasses',
+                return_value=[ClickHouseTestModel, ClickHouseCollapseTestModel])
+    @mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True)
+    @mock.patch.object(ClickHouseCollapseTestModel, 'need_sync', return_value=True)
+    def test_multiple_models(self, need_sync_1_mock, need_sync_2_mock, get_subclasses_mock, sync_delay_mock):
+        clickhouse_auto_sync()
+        self.assertEqual(2, sync_delay_mock.call_count)
+
+
 # Used to profile sync execution time. Disabled by default
 @skip
 class ProfileTest(TransactionTestCase):