mirror of
https://github.com/carrotquest/django-clickhouse.git
synced 2024-11-10 19:36:38 +03:00
Merge pull request #23 from carrotquest/fix-json-serializer/issue-22
Fixed compatibility with celery JSON serializer Added django 3.1 to Travis tests
This commit is contained in:
commit
803626d883
|
@ -51,6 +51,10 @@ env:
|
|||
- PG=10 DJANGO=3.0
|
||||
- PG=11 DJANGO=3.0
|
||||
- PG=12 DJANGO=3.0
|
||||
- PG=9.6 DJANGO=3.1
|
||||
- PG=10 DJANGO=3.1
|
||||
- PG=11 DJANGO=3.1
|
||||
- PG=12 DJANGO=3.1
|
||||
|
||||
before_install:
|
||||
# Use default PostgreSQL 11 port
|
||||
|
|
2
setup.py
2
setup.py
|
@ -13,7 +13,7 @@ with open('requirements.txt') as f:
|
|||
|
||||
setup(
|
||||
name='django-clickhouse',
|
||||
version='1.0.2',
|
||||
version='1.0.3',
|
||||
packages=['django_clickhouse'],
|
||||
package_dir={'': 'src'},
|
||||
url='https://github.com/carrotquest/django-clickhouse',
|
||||
|
|
|
@ -7,16 +7,19 @@ from infi.clickhouse_orm.utils import import_submodules
|
|||
|
||||
from django_clickhouse.clickhouse_models import ClickHouseModel
|
||||
from .configuration import config
|
||||
from .utils import get_subclasses
|
||||
from .utils import get_subclasses, lazy_class_import
|
||||
|
||||
|
||||
@shared_task(queue=config.CELERY_QUEUE)
|
||||
def sync_clickhouse_model(model_cls) -> None:
|
||||
"""
|
||||
Syncs one batch of given ClickHouseModel
|
||||
:param model_cls: ClickHouseModel subclass
|
||||
:param model_cls: ClickHouseModel subclass or python path to it
|
||||
:return: None
|
||||
"""
|
||||
model_cls = lazy_class_import(model_cls)
|
||||
|
||||
# If sync will not finish it is not fatal to set up sync period here: sync will be executed next time
|
||||
model_cls.get_storage().set_last_sync_time(model_cls.get_import_key(), datetime.datetime.now())
|
||||
model_cls.sync_batch_from_storage()
|
||||
|
||||
|
@ -37,9 +40,8 @@ def clickhouse_auto_sync():
|
|||
except ImportError:
|
||||
pass
|
||||
|
||||
# Start
|
||||
for cls in get_subclasses(ClickHouseModel, recursive=True):
|
||||
if cls.need_sync():
|
||||
# Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации
|
||||
# Она выполнится следующей таской через интервал.
|
||||
sync_clickhouse_model.delay(cls)
|
||||
# I pass class as a string in order to make it JSON serializable
|
||||
cls_path = "%s.%s" % (cls.__module__, cls.__name__)
|
||||
sync_clickhouse_model.delay(cls_path)
|
||||
|
|
|
@ -2,7 +2,7 @@ import datetime
|
|||
from unittest import skipIf
|
||||
|
||||
import django
|
||||
from django.test import TransactionTestCase
|
||||
from django.test import TransactionTestCase, TestCase
|
||||
from django.utils.timezone import now
|
||||
|
||||
from tests.clickhouse_models import ClickHouseTestModel, ClickHouseSecondTestModel, ClickHouseCollapseTestModel, \
|
||||
|
@ -25,6 +25,8 @@ class TestOperations(TransactionTestCase):
|
|||
fixtures = ['test_model']
|
||||
django_model = TestModel
|
||||
clickhouse_model = ClickHouseTestModel
|
||||
|
||||
databases = ['default', 'secondary']
|
||||
db_alias = 'default'
|
||||
multi_db = True
|
||||
|
||||
|
|
|
@ -2,15 +2,18 @@ import datetime
|
|||
import logging
|
||||
from subprocess import Popen
|
||||
from time import sleep
|
||||
from unittest import expectedFailure, skip
|
||||
from unittest import expectedFailure, skip, mock
|
||||
|
||||
import os
|
||||
from django.test import TransactionTestCase
|
||||
from django.test.testcases import TestCase
|
||||
from django.utils.timezone import now
|
||||
from random import randint
|
||||
|
||||
from django_clickhouse.database import connections
|
||||
from django_clickhouse.migrations import migrate_app
|
||||
from django_clickhouse.storages import RedisStorage
|
||||
from django_clickhouse.tasks import sync_clickhouse_model, clickhouse_auto_sync
|
||||
from django_clickhouse.utils import int_ranges
|
||||
from tests.clickhouse_models import ClickHouseTestModel, ClickHouseCollapseTestModel, ClickHouseMultiTestModel
|
||||
from tests.models import TestModel
|
||||
|
@ -207,6 +210,48 @@ class KillTest(TransactionTestCase):
|
|||
self._check_data()
|
||||
|
||||
|
||||
@mock.patch.object(ClickHouseTestModel, 'sync_batch_from_storage')
|
||||
class SyncClickHouseModelTest(TestCase):
|
||||
def test_model_as_class(self, sync_mock):
|
||||
sync_clickhouse_model(ClickHouseTestModel)
|
||||
sync_mock.assert_called()
|
||||
|
||||
def test_model_as_string(self, sync_mock):
|
||||
sync_clickhouse_model('tests.clickhouse_models.ClickHouseTestModel')
|
||||
sync_mock.assert_called()
|
||||
|
||||
@mock.patch.object(RedisStorage, 'set_last_sync_time')
|
||||
def test_last_sync_time_called(self, storage_mock, _):
|
||||
sync_clickhouse_model(ClickHouseTestModel)
|
||||
storage_mock.assert_called()
|
||||
self.assertEqual(2, len(storage_mock.call_args))
|
||||
self.assertEqual(storage_mock.call_args[0][0], 'ClickHouseTestModel')
|
||||
self.assertIsInstance(storage_mock.call_args[0][1], datetime.datetime)
|
||||
|
||||
|
||||
@mock.patch.object(sync_clickhouse_model, 'delay')
|
||||
class ClickHouseAutoSyncTest(TestCase):
|
||||
@mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel])
|
||||
@mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True)
|
||||
def test_needs_sync_enabled(self, need_sync_mock, get_subclasses_mock, sync_delay_mock):
|
||||
clickhouse_auto_sync()
|
||||
sync_delay_mock.assert_called_with('tests.clickhouse_models.ClickHouseTestModel')
|
||||
|
||||
@mock.patch('django_clickhouse.tasks.get_subclasses', return_value=[ClickHouseTestModel])
|
||||
@mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=False)
|
||||
def test_does_not_need_sync(self, need_sync_mock, get_subclasses_mock, sync_delay_mock):
|
||||
clickhouse_auto_sync()
|
||||
sync_delay_mock.assert_not_called()
|
||||
|
||||
@mock.patch('django_clickhouse.tasks.get_subclasses',
|
||||
return_value=[ClickHouseTestModel, ClickHouseCollapseTestModel])
|
||||
@mock.patch.object(ClickHouseTestModel, 'need_sync', return_value=True)
|
||||
@mock.patch.object(ClickHouseCollapseTestModel, 'need_sync', return_value=True)
|
||||
def test_multiple_models(self, need_sync_1_mock, need_sync_2_mock, get_subclasses_mock, sync_delay_mock):
|
||||
clickhouse_auto_sync()
|
||||
self.assertEqual(2, sync_delay_mock.call_count)
|
||||
|
||||
|
||||
# Used to profile sync execution time. Disabled by default
|
||||
@skip
|
||||
class ProfileTest(TransactionTestCase):
|
||||
|
|
Loading…
Reference in New Issue
Block a user