diff --git a/src/django_clickhouse/engines.py b/src/django_clickhouse/engines.py index 90beb84..5438897 100644 --- a/src/django_clickhouse/engines.py +++ b/src/django_clickhouse/engines.py @@ -2,7 +2,7 @@ This file contains wrappers for infi.clckhouse_orm engines to use in django-clickhouse """ import datetime -from typing import List, TypeVar, Type +from typing import List, TypeVar, Type, Union from django.db.models import Model as DjangoModel from infi.clickhouse_orm import engines as infi_engines @@ -74,7 +74,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre qs = connections[db_alias].select(query, model_class=model_cls) return list(qs) - def get_final_versions(self, model_cls, objects): + def get_final_versions(self, model_cls, objects, date_col=None): """ Get objects, that are currently stored in ClickHouse. Depending on the partition key this can be different for different models. @@ -84,9 +84,19 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre :param objects: Objects for which final versions are searched :return: A list of model objects """ + + def _dt_to_str(dt): # type: (Union[datetime.date, datetime.datetime]) -> str + if isinstance(dt, datetime.datetime): + return format_datetime(dt, 0, db_alias=db_alias) + elif isinstance(dt, datetime.date): + return dt.isoformat() + else: + raise Exception('Invalid date or datetime object: `%s`' % dt) + if not objects: return [] + date_col = date_col or self.date_col min_date, max_date = None, None for obj in objects: obj_date = getattr(obj, self.date_col) @@ -101,15 +111,8 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre db_alias = model_cls.get_database_alias() - if isinstance(min_date, datetime.date): - min_date = min_date.isoformat() - else: - min_date = format_datetime(min_date, 0, db_alias=db_alias) - - if isinstance(max_date, datetime.date): - max_date = max_date.isoformat() - else: - max_date = format_datetime(max_date, 0, day_end=True, db_alias=db_alias) + min_date = _dt_to_str(min_date) + max_date = _dt_to_str(max_date) if self.version_col: return self._get_final_versions_by_version(db_alias, model_cls, min_date, max_date, object_pks) diff --git a/tests/clickhouse_models.py b/tests/clickhouse_models.py index 8012272..6b30990 100644 --- a/tests/clickhouse_models.py +++ b/tests/clickhouse_models.py @@ -25,7 +25,8 @@ class ClickHouseCollapseTestModel(ClickHouseModel): sync_enabled = True id = fields.Int32Field() - created_date = fields.DateField() + created_date = fields.DateField(materialized='toDate(created)') + created = fields.DateTimeField() value = fields.Int32Field() sign = fields.Int8Field() version = fields.Int8Field(default=1) diff --git a/tests/fixtures/test_model.json b/tests/fixtures/test_model.json index 70cf33b..a8ab0f5 100644 --- a/tests/fixtures/test_model.json +++ b/tests/fixtures/test_model.json @@ -4,7 +4,8 @@ "pk": 1, "fields": { "value": 100, - "created_date": "2018-01-01" + "created_date": "2018-01-01", + "created": "2018-02-01 00:00:00" } }, { @@ -12,7 +13,8 @@ "pk": 2, "fields": { "value": 200, - "created_date": "2018-02-01" + "created_date": "2018-02-01", + "created": "2018-02-01 00:00:00" } } ] \ No newline at end of file diff --git a/tests/fixtures/test_secondary_model.json b/tests/fixtures/test_secondary_model.json index 1a37bf5..03a04b4 100644 --- a/tests/fixtures/test_secondary_model.json +++ b/tests/fixtures/test_secondary_model.json @@ -4,7 +4,8 @@ "pk": 1, "fields": { "value": 100, - "created_date": "2018-01-01" + "created_date": "2018-01-01", + "created": "2018-02-01 00:00:00" } }, { @@ -12,7 +13,8 @@ "pk": 2, "fields": { "value": 200, - "created_date": "2018-02-01" + "created_date": "2018-02-01", + "created": "2018-02-01 00:00:00" } } ] \ No newline at end of file diff --git a/tests/kill_test_sub_process.py b/tests/kill_test_sub_process.py index fa872d6..96d0825 100644 --- a/tests/kill_test_sub_process.py +++ b/tests/kill_test_sub_process.py @@ -25,7 +25,8 @@ logger = logging.getLogger('django-clickhouse') def create(batch_size=1000, test_time=60, period=1, **kwargs): for iteration in range(int(test_time / period)): res = TestModel.objects.db_manager('test_db').bulk_create([ - TestModel(created_date='2018-01-01', value=iteration * batch_size + i) for i in range(batch_size) + TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=iteration * batch_size + i) + for i in range(batch_size) ]) logger.info('django-clickhouse: test created %d records' % len(res)) sleep(period) diff --git a/tests/migrations/0001_initial.py b/tests/migrations/0001_initial.py index 41b0a33..0b1beb0 100644 --- a/tests/migrations/0001_initial.py +++ b/tests/migrations/0001_initial.py @@ -18,6 +18,7 @@ class Migration(migrations.Migration): ('id', models.AutoField()), ('value', models.IntegerField()), ('created_date', models.DateField()), + ('created', models.DateTimeField()), ], options={ 'abstract': False, @@ -29,6 +30,7 @@ class Migration(migrations.Migration): ('id', models.AutoField()), ('value', models.IntegerField()), ('created_date', models.DateField()), + ('created', models.DateTimeField()), ], options={ 'abstract': False, diff --git a/tests/models.py b/tests/models.py index 03f8931..b27b271 100644 --- a/tests/models.py +++ b/tests/models.py @@ -9,8 +9,10 @@ from django_clickhouse.models import ClickHouseSyncModel class TestModel(ClickHouseSyncModel): value = models.IntegerField() created_date = models.DateField() + created = models.DateTimeField() class SecondaryTestModel(ClickHouseSyncModel): value = models.IntegerField() created_date = models.DateField() + created = models.DateTimeField() diff --git a/tests/test_engines.py b/tests/test_engines.py index 555a08f..6fed6ae 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -1,5 +1,6 @@ import datetime +import pytz from django.test import TestCase from django_clickhouse.migrations import migrate_app @@ -14,37 +15,37 @@ class CollapsingMergeTreeTest(TestCase): collapse_fixture = [{ "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": 1, "version": 1 }, { "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": -1, "version": 1 }, { "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": 1, "version": 2 }, { "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": -1, "version": 2 }, { "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": 1, "version": 3 }, { "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": -1, "version": 3 }, { "id": 1, - "created_date": "2018-01-01", + "created": "2018-01-01 00:00:00", "sign": 1, "version": 4 }] @@ -61,19 +62,36 @@ class CollapsingMergeTreeTest(TestCase): ]) self.objects = TestModel.objects.filter(id=1) - def test_get_final_versions_by_final(self): + def test_get_final_versions_by_final_date(self): final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel, self.objects) self.assertEqual(1, len(final_versions)) - self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0, 'created_date': datetime.date(2018, 1, 1)}, - final_versions[0].to_dict()) + self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0}, + final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version'))) - def test_get_final_versions_by_version(self): + def test_get_final_versions_by_version_date(self): ClickHouseCollapseTestModel.engine.version_col = 'version' final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel, self.objects) self.assertEqual(1, len(final_versions)) - self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0, 'created_date': datetime.date(2018, 1, 1)}, - final_versions[0].to_dict()) + self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0}, + final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version'))) + + def test_get_final_versions_by_final_datetime(self): + final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel, + self.objects, date_col='created') + + self.assertEqual(1, len(final_versions)) + self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0}, + final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version'))) + + def test_get_final_versions_by_version_datetime(self): + ClickHouseCollapseTestModel.engine.version_col = 'version' + final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel, + self.objects, date_col='created') + + self.assertEqual(1, len(final_versions)) + self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0}, + final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version'))) diff --git a/tests/test_models.py b/tests/test_models.py index c9bfd05..4202a9d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -34,7 +34,7 @@ class TestOperations(TransactionTestCase): def test_save(self): # INSERT operation - instance = self.django_model(created_date=datetime.date.today(), value=2) + instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2) instance.save() self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) @@ -46,12 +46,14 @@ class TestOperations(TransactionTestCase): self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_create(self): - instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), value=2) + instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), + created=datetime.datetime.now(), value=2) self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_bulk_create(self): - items = [self.django_model(created_date=datetime.date.today(), value=i) for i in range(5)] + items = [self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i) + for i in range(5)] items = self.django_model.objects.bulk_create(items) self.assertEqual(5, len(items)) self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items], @@ -59,7 +61,8 @@ class TestOperations(TransactionTestCase): def test_get_or_create(self): instance, created = self.django_model.objects. \ - get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'value': 2}) + get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': datetime.datetime.now(), + 'value': 2}) self.assertTrue(created) self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], @@ -74,7 +77,8 @@ class TestOperations(TransactionTestCase): def test_update_or_create(self): instance, created = self.django_model.objects. \ - update_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'value': 2}) + update_or_create(pk=100, defaults={'created_date': datetime.date.today(), + 'created': datetime.datetime.now(), 'value': 2}) self.assertTrue(created) self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) diff --git a/tests/test_sync.py b/tests/test_sync.py index 4112de6..c88f2f5 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -27,7 +27,7 @@ class SyncTest(TransactionTestCase): ClickHouseTestModel.get_storage().flush() def test_simple(self): - obj = TestModel.objects.create(value=1, created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) ClickHouseTestModel.sync_batch_from_storage() synced_data = list(ClickHouseTestModel.objects.all()) @@ -37,7 +37,7 @@ class SyncTest(TransactionTestCase): self.assertEqual(obj.id, synced_data[0].id) def test_collapsing_update_by_final(self): - obj = TestModel.objects.create(value=1, created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) obj.value = 2 obj.save() ClickHouseCollapseTestModel.sync_batch_from_storage() @@ -55,14 +55,13 @@ class SyncTest(TransactionTestCase): synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel)) self.assertGreaterEqual(1, len(synced_data)) - self.assertEqual(obj.created_date, synced_data[0].created_date) self.assertEqual(obj.value, synced_data[0].value) self.assertEqual(obj.id, synced_data[0].id) def test_collapsing_update_by_version(self): ClickHouseCollapseTestModel.engine.version_col = 'version' - obj = TestModel.objects.create(value=1, created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) obj.value = 2 obj.save() ClickHouseCollapseTestModel.sync_batch_from_storage() @@ -80,7 +79,6 @@ class SyncTest(TransactionTestCase): synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel)) self.assertGreaterEqual(1, len(synced_data)) - self.assertEqual(obj.created_date, synced_data[0].created_date) self.assertEqual(obj.value, synced_data[0].value) self.assertEqual(obj.id, synced_data[0].id) @@ -98,7 +96,7 @@ class SyncTest(TransactionTestCase): self.assertEqual(0, len(synced_data)) def test_multi_model(self): - obj = TestModel.objects.create(value=1, created_date=datetime.date.today()) + obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today()) obj.value = 2 obj.save() ClickHouseMultiTestModel.sync_batch_from_storage() @@ -122,7 +120,6 @@ class SyncTest(TransactionTestCase): synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel)) self.assertGreaterEqual(1, len(synced_data)) - self.assertEqual(obj.created_date, synced_data[0].created_date) self.assertEqual(obj.value, synced_data[0].value) self.assertEqual(obj.id, synced_data[0].id)