diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index c5d5240..b6a8af3 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -11,12 +11,42 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] - postgres-version: ["9.6", "10", "11", "12", "13"] - django-version: ["2.1", "2.2", "3.0", "3.1", "3.2"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + postgres-version: ["9.6", "10", "11", "12", "13", "14", "15"] + django-version: ["3.1", "3.2", "4.0", "4.1", "4.2", "5.0"] clickhouse-version: ["latest"] redis-version: ["latest"] - + exclude: + # Django 4.0+ doesn't support PostgreSQL 9.6 + - django-version: "4.0" + postgres-version: "9.6" + - django-version: "4.1" + postgres-version: "9.6" + - django-version: "4.2" + postgres-version: "9.6" + - django-version: "5.0" + postgres-version: "9.6" + + # Django 4.1+ doesn't support PostgreSQL 10 + - django-version: "4.1" + postgres-version: "10" + - django-version: "4.2" + postgres-version: "10" + - django-version: "5.0" + postgres-version: "10" + + # Django 4.2+ doesn't support PostgreSQL 11 + - django-version: "4.2" + postgres-version: "11" + - django-version: "5.0" + postgres-version: "11" + + # Django 5.0+ does not support python 3.8, 3.9 + - django-version: "5.0" + python-version: "3.8" + - django-version: "5.0" + python-version: "3.9" + services: postgres: image: postgres:${{ matrix.postgres-version }} diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py index bd009e7..8b048a1 100644 --- a/src/django_clickhouse/clickhouse_models.py +++ b/src/django_clickhouse/clickhouse_models.py @@ -9,6 +9,7 @@ from itertools import chain from typing import List, Tuple, Iterable, Set, Any, Optional from django.db.models import Model as DjangoModel, QuerySet as DjangoQuerySet +from django.utils.timezone import now from infi.clickhouse_orm.engines import CollapsingMergeTree from infi.clickhouse_orm.models import Model as InfiModel, ModelBase as InfiModelBase from statsd.defaults.django import statsd @@ -290,7 +291,7 @@ class ClickHouseModel(InfiModel, metaclass=ClickHouseModelMeta): res = (datetime.datetime.now() - last_sync_time).total_seconds() >= cls.get_sync_delay() logger.debug('django-clickhouse: need_sync returned %s for class %s as no last sync found' ' (now: %s, last: %s, delay: %d)' - % (res, cls.__name__, datetime.datetime.now().isoformat(), last_sync_time.isoformat(), + % (res, cls.__name__, now().isoformat(), last_sync_time.isoformat(), cls.get_sync_delay())) return res diff --git a/src/django_clickhouse/storages.py b/src/django_clickhouse/storages.py index 63115ae..9502024 100644 --- a/src/django_clickhouse/storages.py +++ b/src/django_clickhouse/storages.py @@ -11,6 +11,8 @@ import logging from typing import Any, Optional, List, Tuple import os + +from django.utils.timezone import now from statsd.defaults.django import statsd from .configuration import config @@ -186,8 +188,7 @@ class RedisStorage(Storage, metaclass=SingletonMeta): def get_operations(self, import_key, count, **kwargs): ops_key = self.REDIS_KEY_OPS_TEMPLATE.format(import_key=import_key) - res = self._redis.zrangebyscore(ops_key, '-inf', datetime.datetime.now().timestamp(), start=0, num=count, - withscores=True) + res = self._redis.zrangebyscore(ops_key, '-inf', now().timestamp(), start=0, num=count, withscores=True) if res: ops, scores = zip(*res) diff --git a/tests/fixtures/test_model.json b/tests/fixtures/test_model.json index aaf7400..f5d7150 100644 --- a/tests/fixtures/test_model.json +++ b/tests/fixtures/test_model.json @@ -5,7 +5,7 @@ "fields": { "value": 100, "created_date": "2018-01-01", - "created": "2018-01-01 00:00:00" + "created": "2018-01-01 00:00:00+0000" } }, { @@ -14,7 +14,7 @@ "fields": { "value": 200, "created_date": "2018-02-01", - "created": "2018-02-01 00:00:00" + "created": "2018-02-01 00:00:00+0000" } }, { @@ -23,7 +23,7 @@ "fields": { "value": 300, "created_date": "2018-03-01", - "created": "2018-03-01 00:00:00" + "created": "2018-03-01 00:00:00+0000" } }, { @@ -32,7 +32,7 @@ "fields": { "value": 400, "created_date": "2018-04-01", - "created": "2018-04-01 00:00:00" + "created": "2018-04-01 00:00:00+0000" } }, { @@ -41,7 +41,7 @@ "fields": { "value": 500, "created_date": "2018-05-01", - "created": "2018-05-01 00:00:00" + "created": "2018-05-01 00:00:00+0000" } } ] \ No newline at end of file diff --git a/tests/fixtures/test_secondary_model.json b/tests/fixtures/test_secondary_model.json index 03a04b4..88fdfd8 100644 --- a/tests/fixtures/test_secondary_model.json +++ b/tests/fixtures/test_secondary_model.json @@ -5,7 +5,7 @@ "fields": { "value": 100, "created_date": "2018-01-01", - "created": "2018-02-01 00:00:00" + "created": "2018-02-01 00:00:00+0000" } }, { @@ -14,7 +14,7 @@ "fields": { "value": 200, "created_date": "2018-02-01", - "created": "2018-02-01 00:00:00" + "created": "2018-02-01 00:00:00+0000" } } ] \ No newline at end of file diff --git a/tests/kill_test_sub_process.py b/tests/kill_test_sub_process.py index 36f9385..52ef0cd 100644 --- a/tests/kill_test_sub_process.py +++ b/tests/kill_test_sub_process.py @@ -8,6 +8,7 @@ from time import sleep import datetime +from django.utils.timezone import now # set Django environment sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -25,7 +26,7 @@ logger = logging.getLogger('django-clickhouse') def create(batch_size=1000, test_time=60, period=1, **kwargs): for iteration in range(int(test_time / period)): res = TestModel.objects.db_manager('test_db').bulk_create([ - TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=iteration * batch_size + i) + TestModel(created=now(), created_date='2018-01-01', value=iteration * batch_size + i) for i in range(batch_size) ]) logger.info('django-clickhouse: test created %d records' % len(res)) @@ -54,8 +55,8 @@ def sync(period=1, test_time=60, **kwargs): if kwargs['once']: ClickHouseCollapseTestModel.sync_batch_from_storage() else: - start = datetime.datetime.now() - while (datetime.datetime.now() - start).total_seconds() < test_time: + start = now() + while (now() - start).total_seconds() < test_time: ClickHouseCollapseTestModel.sync_batch_from_storage() sleep(period) diff --git a/tests/settings.py b/tests/settings.py index 4d83af1..3668be9 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -4,6 +4,7 @@ This file contains django settings to run tests with runtests.py from os import environ SECRET_KEY = 'fake-key' +USE_TZ = True DATABASES = { 'default': { diff --git a/tests/test_clickhouse_models.py b/tests/test_clickhouse_models.py index 860ebc6..b2f720a 100644 --- a/tests/test_clickhouse_models.py +++ b/tests/test_clickhouse_models.py @@ -1,6 +1,7 @@ import datetime from django.test import TestCase +from django.utils.timezone import now from tests.clickhouse_models import ClickHouseTestModel @@ -20,11 +21,11 @@ class ClickHouseModelTest(TestCase): self.assertTrue(ClickHouseTestModel.need_sync()) # Time hasn't passed - no sync - self.storage.set_last_sync_time(ClickHouseTestModel.get_import_key(), datetime.datetime.now()) + self.storage.set_last_sync_time(ClickHouseTestModel.get_import_key(), now()) self.assertFalse(ClickHouseTestModel.need_sync()) # Time has passed sync_delay = ClickHouseTestModel.get_sync_delay() self.storage.set_last_sync_time(ClickHouseTestModel.get_import_key(), - datetime.datetime.now() - datetime.timedelta(seconds=sync_delay + 1)) + now() - datetime.timedelta(seconds=sync_delay + 1)) self.assertTrue(ClickHouseTestModel.need_sync()) diff --git a/tests/test_models.py b/tests/test_models.py index 56580bd..f556223 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -40,7 +40,7 @@ class TestOperations(TransactionTestCase): def test_save(self): # INSERT operation - instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2) + instance = self.django_model(created_date=datetime.date.today(), created=now(), value=2) instance.save() self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) @@ -52,13 +52,13 @@ class TestOperations(TransactionTestCase): self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_create(self): - instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), - created=datetime.datetime.now(), value=2) + instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), created=now(), + value=2) self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_bulk_create(self): - items = [self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i) + items = [self.django_model(created_date=datetime.date.today(), created=now(), value=i) for i in range(5)] items = self.django_model.objects.bulk_create(items) self.assertEqual(5, len(items)) @@ -187,7 +187,7 @@ class TestOperations(TransactionTestCase): def test_get_or_create(self): instance, created = self.django_model.objects. \ - get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': datetime.datetime.now(), + get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': now(), 'value': 2}) self.assertTrue(created) @@ -203,8 +203,7 @@ class TestOperations(TransactionTestCase): def test_update_or_create(self): instance, created = self.django_model.objects. \ - update_or_create(pk=100, defaults={'created_date': datetime.date.today(), - 'created': datetime.datetime.now(), 'value': 2}) + update_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': now(), 'value': 2}) self.assertTrue(created) self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) @@ -229,7 +228,7 @@ class TestOperations(TransactionTestCase): def test_bulk_create_returning(self): items = [ - self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i) + self.django_model(created_date=datetime.date.today(), created=now(), value=i) for i in range(5) ] items = self.django_model.objects.bulk_create_returning(items) @@ -260,7 +259,7 @@ class TestOperations(TransactionTestCase): def test_save_returning(self): # INSERT operation - instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2) + instance = self.django_model(created_date=datetime.date.today(), created=now(), value=2) instance.save_returning() self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) @@ -284,6 +283,7 @@ class TestOperations(TransactionTestCase): class TestSecondaryOperations(TestOperations): + # from django.db.models.fields import * fixtures = ['test_secondary_model'] django_model = SecondaryTestModel clickhouse_model = ClickHouseSecondTestModel diff --git a/tests/test_sync.py b/tests/test_sync.py index 1958e2f..ecd5b57 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -30,7 +30,7 @@ class SyncTest(TransactionTestCase): ClickHouseTestModel.get_storage().flush() def test_simple(self): - obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today()) ClickHouseTestModel.sync_batch_from_storage() synced_data = list(ClickHouseTestModel.objects.all()) @@ -40,7 +40,7 @@ class SyncTest(TransactionTestCase): self.assertEqual(obj.id, synced_data[0].id) def test_collapsing_update_by_final(self): - obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today()) obj.value = 2 obj.save() ClickHouseCollapseTestModel.sync_batch_from_storage() @@ -63,7 +63,7 @@ class SyncTest(TransactionTestCase): def test_collapsing_update_by_version(self): ClickHouseCollapseTestModel.engine.version_col = 'version' - obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today()) obj.value = 2 obj.save() ClickHouseCollapseTestModel.sync_batch_from_storage() @@ -97,7 +97,7 @@ class SyncTest(TransactionTestCase): self.assertEqual(0, len(synced_data)) def test_multi_model(self): - obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today()) obj.value = 2 obj.save() ClickHouseMultiTestModel.sync_batch_from_storage() @@ -268,7 +268,7 @@ class ProfileTest(TransactionTestCase): ClickHouseTestModel.sync_enabled = False TestModel.objects.bulk_create([ - TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=i) + TestModel(created=now(), created_date='2018-01-01', value=i) for i in range(self.BATCH_SIZE) ])