carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2024-11-21 16:46:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added latest software versions to testing in GitHub actions (#48)

Browse Source 
1. Added latest software versions to testing in GitHub actions
2. Fixed timezone mistakes in tests
3. Fixed naive datetime warnings
This commit is contained in:
M1ha Shvn 2024-01-05 15:20:10 +05:00 committed by GitHub
parent 0ac4615649
commit 244b4fbd37
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 68 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.github/workflows/python-tests.yml vendored
View File

@ -11,12 +11,42 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
postgres-version: ["9.6", "10", "11", "12", "13"]
django-version: ["2.1", "2.2", "3.0", "3.1", "3.2"]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
postgres-version: ["9.6", "10", "11", "12", "13", "14", "15"]
django-version: ["3.1", "3.2", "4.0", "4.1", "4.2", "5.0"]
clickhouse-version: ["latest"]
redis-version: ["latest"]
exclude:
# Django 4.0+ doesn't support PostgreSQL 9.6
- django-version: "4.0"
postgres-version: "9.6"
- django-version: "4.1"
postgres-version: "9.6"
- django-version: "4.2"
postgres-version: "9.6"
- django-version: "5.0"
postgres-version: "9.6"
# Django 4.1+ doesn't support PostgreSQL 10
- django-version: "4.1"
postgres-version: "10"
- django-version: "4.2"
postgres-version: "10"
- django-version: "5.0"
postgres-version: "10"
# Django 4.2+ doesn't support PostgreSQL 11
- django-version: "4.2"
postgres-version: "11"
- django-version: "5.0"
postgres-version: "11"
# Django 5.0+ does not support python 3.8, 3.9
- django-version: "5.0"
python-version: "3.8"
- django-version: "5.0"
python-version: "3.9"
services:
postgres:
image: postgres:${{ matrix.postgres-version }}

3
src/django_clickhouse/clickhouse_models.py
View File

@ -9,6 +9,7 @@ from itertools import chain
from typing import List, Tuple, Iterable, Set, Any, Optional
from django.db.models import Model as DjangoModel, QuerySet as DjangoQuerySet
from django.utils.timezone import now
from infi.clickhouse_orm.engines import CollapsingMergeTree
from infi.clickhouse_orm.models import Model as InfiModel, ModelBase as InfiModelBase
from statsd.defaults.django import statsd
@ -290,7 +291,7 @@ class ClickHouseModel(InfiModel, metaclass=ClickHouseModelMeta):
res = (datetime.datetime.now() - last_sync_time).total_seconds() >= cls.get_sync_delay()
logger.debug('django-clickhouse: need_sync returned %s for class %s as no last sync found'
' (now: %s, last: %s, delay: %d)'
% (res, cls.__name__, datetime.datetime.now().isoformat(), last_sync_time.isoformat(),
% (res, cls.__name__, now().isoformat(), last_sync_time.isoformat(),
cls.get_sync_delay()))
return res

5
src/django_clickhouse/storages.py
View File

@ -11,6 +11,8 @@ import logging
from typing import Any, Optional, List, Tuple
import os
from django.utils.timezone import now
from statsd.defaults.django import statsd
from .configuration import config
@ -186,8 +188,7 @@ class RedisStorage(Storage, metaclass=SingletonMeta):
def get_operations(self, import_key, count, **kwargs):
ops_key = self.REDIS_KEY_OPS_TEMPLATE.format(import_key=import_key)
res = self._redis.zrangebyscore(ops_key, '-inf', datetime.datetime.now().timestamp(), start=0, num=count,
withscores=True)
res = self._redis.zrangebyscore(ops_key, '-inf', now().timestamp(), start=0, num=count, withscores=True)
if res:
ops, scores = zip(*res)

10
tests/fixtures/test_model.json vendored
View File

@ -5,7 +5,7 @@
"fields": {
"value": 100,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00"
"created": "2018-01-01 00:00:00+0000"
}
},
{
@ -14,7 +14,7 @@
"fields": {
"value": 200,
"created_date": "2018-02-01",
"created": "2018-02-01 00:00:00"
"created": "2018-02-01 00:00:00+0000"
}
},
{
@ -23,7 +23,7 @@
"fields": {
"value": 300,
"created_date": "2018-03-01",
"created": "2018-03-01 00:00:00"
"created": "2018-03-01 00:00:00+0000"
}
},
{
@ -32,7 +32,7 @@
"fields": {
"value": 400,
"created_date": "2018-04-01",
"created": "2018-04-01 00:00:00"
"created": "2018-04-01 00:00:00+0000"
}
},
{
@ -41,7 +41,7 @@
"fields": {
"value": 500,
"created_date": "2018-05-01",
"created": "2018-05-01 00:00:00"
"created": "2018-05-01 00:00:00+0000"
}
}
]

4
tests/fixtures/test_secondary_model.json vendored
View File

@ -5,7 +5,7 @@
"fields": {
"value": 100,
"created_date": "2018-01-01",
"created": "2018-02-01 00:00:00"
"created": "2018-02-01 00:00:00+0000"
}
},
{
@ -14,7 +14,7 @@
"fields": {
"value": 200,
"created_date": "2018-02-01",
"created": "2018-02-01 00:00:00"
"created": "2018-02-01 00:00:00+0000"
}
}
]

7
tests/kill_test_sub_process.py
View File

@ -8,6 +8,7 @@ from time import sleep
import datetime
from django.utils.timezone import now
# set Django environment
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@ -25,7 +26,7 @@ logger = logging.getLogger('django-clickhouse')
def create(batch_size=1000, test_time=60, period=1, **kwargs):
for iteration in range(int(test_time / period)):
res = TestModel.objects.db_manager('test_db').bulk_create([
TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=iteration * batch_size + i)
TestModel(created=now(), created_date='2018-01-01', value=iteration * batch_size + i)
for i in range(batch_size)
])
logger.info('django-clickhouse: test created %d records' % len(res))
@ -54,8 +55,8 @@ def sync(period=1, test_time=60, **kwargs):
if kwargs['once']:
ClickHouseCollapseTestModel.sync_batch_from_storage()
else:
start = datetime.datetime.now()
while (datetime.datetime.now() - start).total_seconds() < test_time:
start = now()
while (now() - start).total_seconds() < test_time:
ClickHouseCollapseTestModel.sync_batch_from_storage()
sleep(period)

1
tests/settings.py
View File

@ -4,6 +4,7 @@ This file contains django settings to run tests with runtests.py
from os import environ
SECRET_KEY = 'fake-key'
USE_TZ = True
DATABASES = {
'default': {

5
tests/test_clickhouse_models.py
View File

@ -1,6 +1,7 @@
import datetime
from django.test import TestCase
from django.utils.timezone import now
from tests.clickhouse_models import ClickHouseTestModel
@ -20,11 +21,11 @@ class ClickHouseModelTest(TestCase):
self.assertTrue(ClickHouseTestModel.need_sync())
# Time hasn't passed - no sync
self.storage.set_last_sync_time(ClickHouseTestModel.get_import_key(), datetime.datetime.now())
self.storage.set_last_sync_time(ClickHouseTestModel.get_import_key(), now())
self.assertFalse(ClickHouseTestModel.need_sync())
# Time has passed
sync_delay = ClickHouseTestModel.get_sync_delay()
self.storage.set_last_sync_time(ClickHouseTestModel.get_import_key(),
datetime.datetime.now() - datetime.timedelta(seconds=sync_delay + 1))
now() - datetime.timedelta(seconds=sync_delay + 1))
self.assertTrue(ClickHouseTestModel.need_sync())

18
tests/test_models.py
View File

@ -40,7 +40,7 @@ class TestOperations(TransactionTestCase):
def test_save(self):
# INSERT operation
instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2)
instance = self.django_model(created_date=datetime.date.today(), created=now(), value=2)
instance.save()
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
@ -52,13 +52,13 @@ class TestOperations(TransactionTestCase):
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_create(self):
instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(),
created=datetime.datetime.now(), value=2)
instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), created=now(),
value=2)
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_bulk_create(self):
items = [self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i)
items = [self.django_model(created_date=datetime.date.today(), created=now(), value=i)
for i in range(5)]
items = self.django_model.objects.bulk_create(items)
self.assertEqual(5, len(items))
@ -187,7 +187,7 @@ class TestOperations(TransactionTestCase):
def test_get_or_create(self):
instance, created = self.django_model.objects. \
get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': datetime.datetime.now(),
get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': now(),
'value': 2})
self.assertTrue(created)
@ -203,8 +203,7 @@ class TestOperations(TransactionTestCase):
def test_update_or_create(self):
instance, created = self.django_model.objects. \
update_or_create(pk=100, defaults={'created_date': datetime.date.today(),
'created': datetime.datetime.now(), 'value': 2})
update_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': now(), 'value': 2})
self.assertTrue(created)
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
@ -229,7 +228,7 @@ class TestOperations(TransactionTestCase):
def test_bulk_create_returning(self):
items = [
self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i)
self.django_model(created_date=datetime.date.today(), created=now(), value=i)
for i in range(5)
]
items = self.django_model.objects.bulk_create_returning(items)
@ -260,7 +259,7 @@ class TestOperations(TransactionTestCase):
def test_save_returning(self):
# INSERT operation
instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2)
instance = self.django_model(created_date=datetime.date.today(), created=now(), value=2)
instance.save_returning()
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
@ -284,6 +283,7 @@ class TestOperations(TransactionTestCase):
class TestSecondaryOperations(TestOperations):
# from django.db.models.fields import *
fixtures = ['test_secondary_model']
django_model = SecondaryTestModel
clickhouse_model = ClickHouseSecondTestModel

10
tests/test_sync.py
View File

@ -30,7 +30,7 @@ class SyncTest(TransactionTestCase):
ClickHouseTestModel.get_storage().flush()
def test_simple(self):
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today())
ClickHouseTestModel.sync_batch_from_storage()
synced_data = list(ClickHouseTestModel.objects.all())
@ -40,7 +40,7 @@ class SyncTest(TransactionTestCase):
self.assertEqual(obj.id, synced_data[0].id)
def test_collapsing_update_by_final(self):
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today())
obj.value = 2
obj.save()
ClickHouseCollapseTestModel.sync_batch_from_storage()
@ -63,7 +63,7 @@ class SyncTest(TransactionTestCase):
def test_collapsing_update_by_version(self):
ClickHouseCollapseTestModel.engine.version_col = 'version'
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today())
obj.value = 2
obj.save()
ClickHouseCollapseTestModel.sync_batch_from_storage()
@ -97,7 +97,7 @@ class SyncTest(TransactionTestCase):
self.assertEqual(0, len(synced_data))
def test_multi_model(self):
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=now(), created_date=datetime.date.today())
obj.value = 2
obj.save()
ClickHouseMultiTestModel.sync_batch_from_storage()
@ -268,7 +268,7 @@ class ProfileTest(TransactionTestCase):
ClickHouseTestModel.sync_enabled = False
TestModel.objects.bulk_create([
TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=i)
TestModel(created=now(), created_date='2018-01-01', value=i)
for i in range(self.BATCH_SIZE)
])