carrotquest/django-clickhouse
1
1
Fork 0
mirror of https://github.com/carrotquest/django-clickhouse.git synced 2024-11-24 18:13:46 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added tests for get_final_versions on date and datetime

Browse Source
This commit is contained in:
M1ha 2019-01-14 12:45:46 +05:00
parent ec72e31986
commit 43815723df
10 changed files with 74 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/django_clickhouse/engines.py
View File

@ -2,7 +2,7 @@
This file contains wrappers for infi.clckhouse_orm engines to use in django-clickhouse
"""
import datetime
from typing import List, TypeVar, Type
from typing import List, TypeVar, Type, Union
from django.db.models import Model as DjangoModel
from infi.clickhouse_orm import engines as infi_engines
@ -74,7 +74,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
qs = connections[db_alias].select(query, model_class=model_cls)
return list(qs)
def get_final_versions(self, model_cls, objects):
def get_final_versions(self, model_cls, objects, date_col=None):
"""
Get objects, that are currently stored in ClickHouse.
Depending on the partition key this can be different for different models.
@ -84,9 +84,19 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
:param objects: Objects for which final versions are searched
:return: A list of model objects
"""
def _dt_to_str(dt): # type: (Union[datetime.date, datetime.datetime]) -> str
if isinstance(dt, datetime.datetime):
return format_datetime(dt, 0, db_alias=db_alias)
elif isinstance(dt, datetime.date):
return dt.isoformat()
else:
raise Exception('Invalid date or datetime object: `%s`' % dt)
if not objects:
return []
date_col = date_col or self.date_col
min_date, max_date = None, None
for obj in objects:
obj_date = getattr(obj, self.date_col)
@ -101,15 +111,8 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
db_alias = model_cls.get_database_alias()
if isinstance(min_date, datetime.date):
min_date = min_date.isoformat()
else:
min_date = format_datetime(min_date, 0, db_alias=db_alias)
if isinstance(max_date, datetime.date):
max_date = max_date.isoformat()
else:
max_date = format_datetime(max_date, 0, day_end=True, db_alias=db_alias)
min_date = _dt_to_str(min_date)
max_date = _dt_to_str(max_date)
if self.version_col:
return self._get_final_versions_by_version(db_alias, model_cls, min_date, max_date, object_pks)

3
tests/clickhouse_models.py
View File

@ -25,7 +25,8 @@ class ClickHouseCollapseTestModel(ClickHouseModel):
sync_enabled = True
id = fields.Int32Field()
created_date = fields.DateField()
created_date = fields.DateField(materialized='toDate(created)')
created = fields.DateTimeField()
value = fields.Int32Field()
sign = fields.Int8Field()
version = fields.Int8Field(default=1)

6
tests/fixtures/test_model.json vendored
View File

@ -4,7 +4,8 @@
"pk": 1,
"fields": {
"value": 100,
"created_date": "2018-01-01"
"created_date": "2018-01-01",
"created": "2018-02-01 00:00:00"
}
},
{
@ -12,7 +13,8 @@
"pk": 2,
"fields": {
"value": 200,
"created_date": "2018-02-01"
"created_date": "2018-02-01",
"created": "2018-02-01 00:00:00"
}
}
]

6
tests/fixtures/test_secondary_model.json vendored
View File

@ -4,7 +4,8 @@
"pk": 1,
"fields": {
"value": 100,
"created_date": "2018-01-01"
"created_date": "2018-01-01",
"created": "2018-02-01 00:00:00"
}
},
{
@ -12,7 +13,8 @@
"pk": 2,
"fields": {
"value": 200,
"created_date": "2018-02-01"
"created_date": "2018-02-01",
"created": "2018-02-01 00:00:00"
}
}
]

3
tests/kill_test_sub_process.py
View File

@ -25,7 +25,8 @@ logger = logging.getLogger('django-clickhouse')
def create(batch_size=1000, test_time=60, period=1, **kwargs):
for iteration in range(int(test_time / period)):
res = TestModel.objects.db_manager('test_db').bulk_create([
TestModel(created_date='2018-01-01', value=iteration * batch_size + i) for i in range(batch_size)
TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=iteration * batch_size + i)
for i in range(batch_size)
])
logger.info('django-clickhouse: test created %d records' % len(res))
sleep(period)

2
tests/migrations/0001_initial.py
View File

@ -18,6 +18,7 @@ class Migration(migrations.Migration):
('id', models.AutoField()),
('value', models.IntegerField()),
('created_date', models.DateField()),
('created', models.DateTimeField()),
],
options={
'abstract': False,
@ -29,6 +30,7 @@ class Migration(migrations.Migration):
('id', models.AutoField()),
('value', models.IntegerField()),
('created_date', models.DateField()),
('created', models.DateTimeField()),
],
options={
'abstract': False,

2
tests/models.py
View File

@ -9,8 +9,10 @@ from django_clickhouse.models import ClickHouseSyncModel
class TestModel(ClickHouseSyncModel):
value = models.IntegerField()
created_date = models.DateField()
created = models.DateTimeField()
class SecondaryTestModel(ClickHouseSyncModel):
value = models.IntegerField()
created_date = models.DateField()
created = models.DateTimeField()

44
tests/test_engines.py
View File

@ -1,5 +1,6 @@
import datetime
import pytz
from django.test import TestCase
from django_clickhouse.migrations import migrate_app
@ -14,37 +15,37 @@ class CollapsingMergeTreeTest(TestCase):
collapse_fixture = [{
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": 1,
"version": 1
}, {
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": -1,
"version": 1
}, {
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": 1,
"version": 2
}, {
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": -1,
"version": 2
}, {
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": 1,
"version": 3
}, {
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": -1,
"version": 3
}, {
"id": 1,
"created_date": "2018-01-01",
"created": "2018-01-01 00:00:00",
"sign": 1,
"version": 4
}]
@ -61,19 +62,36 @@ class CollapsingMergeTreeTest(TestCase):
])
self.objects = TestModel.objects.filter(id=1)
def test_get_final_versions_by_final(self):
def test_get_final_versions_by_final_date(self):
final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
self.objects)
self.assertEqual(1, len(final_versions))
self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0, 'created_date': datetime.date(2018, 1, 1)},
final_versions[0].to_dict())
self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))
def test_get_final_versions_by_version(self):
def test_get_final_versions_by_version_date(self):
ClickHouseCollapseTestModel.engine.version_col = 'version'
final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
self.objects)
self.assertEqual(1, len(final_versions))
self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0, 'created_date': datetime.date(2018, 1, 1)},
final_versions[0].to_dict())
self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))
def test_get_final_versions_by_final_datetime(self):
final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
self.objects, date_col='created')
self.assertEqual(1, len(final_versions))
self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))
def test_get_final_versions_by_version_datetime(self):
ClickHouseCollapseTestModel.engine.version_col = 'version'
final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
self.objects, date_col='created')
self.assertEqual(1, len(final_versions))
self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))

14
tests/test_models.py
View File

@ -34,7 +34,7 @@ class TestOperations(TransactionTestCase):
def test_save(self):
# INSERT operation
instance = self.django_model(created_date=datetime.date.today(), value=2)
instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2)
instance.save()
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
@ -46,12 +46,14 @@ class TestOperations(TransactionTestCase):
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_create(self):
instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), value=2)
instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(),
created=datetime.datetime.now(), value=2)
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_bulk_create(self):
items = [self.django_model(created_date=datetime.date.today(), value=i) for i in range(5)]
items = [self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i)
for i in range(5)]
items = self.django_model.objects.bulk_create(items)
self.assertEqual(5, len(items))
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items],
@ -59,7 +61,8 @@ class TestOperations(TransactionTestCase):
def test_get_or_create(self):
instance, created = self.django_model.objects. \
get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'value': 2})
get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': datetime.datetime.now(),
'value': 2})
self.assertTrue(created)
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
@ -74,7 +77,8 @@ class TestOperations(TransactionTestCase):
def test_update_or_create(self):
instance, created = self.django_model.objects. \
update_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'value': 2})
update_or_create(pk=100, defaults={'created_date': datetime.date.today(),
'created': datetime.datetime.now(), 'value': 2})
self.assertTrue(created)
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))

11
tests/test_sync.py
View File

@ -27,7 +27,7 @@ class SyncTest(TransactionTestCase):
ClickHouseTestModel.get_storage().flush()
def test_simple(self):
obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
ClickHouseTestModel.sync_batch_from_storage()
synced_data = list(ClickHouseTestModel.objects.all())
@ -37,7 +37,7 @@ class SyncTest(TransactionTestCase):
self.assertEqual(obj.id, synced_data[0].id)
def test_collapsing_update_by_final(self):
obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj.value = 2
obj.save()
ClickHouseCollapseTestModel.sync_batch_from_storage()
@ -55,14 +55,13 @@ class SyncTest(TransactionTestCase):
synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel))
self.assertGreaterEqual(1, len(synced_data))
self.assertEqual(obj.created_date, synced_data[0].created_date)
self.assertEqual(obj.value, synced_data[0].value)
self.assertEqual(obj.id, synced_data[0].id)
def test_collapsing_update_by_version(self):
ClickHouseCollapseTestModel.engine.version_col = 'version'
obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj.value = 2
obj.save()
ClickHouseCollapseTestModel.sync_batch_from_storage()
@ -80,7 +79,6 @@ class SyncTest(TransactionTestCase):
synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel))
self.assertGreaterEqual(1, len(synced_data))
self.assertEqual(obj.created_date, synced_data[0].created_date)
self.assertEqual(obj.value, synced_data[0].value)
self.assertEqual(obj.id, synced_data[0].id)
@ -98,7 +96,7 @@ class SyncTest(TransactionTestCase):
self.assertEqual(0, len(synced_data))
def test_multi_model(self):
obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
obj.value = 2
obj.save()
ClickHouseMultiTestModel.sync_batch_from_storage()
@ -122,7 +120,6 @@ class SyncTest(TransactionTestCase):
synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel))
self.assertGreaterEqual(1, len(synced_data))
self.assertEqual(obj.created_date, synced_data[0].created_date)
self.assertEqual(obj.value, synced_data[0].value)
self.assertEqual(obj.id, synced_data[0].id)