Added tests for get_final_versions on date and datetime

src/django_clickhouse/engines.py

@@ -2,7 +2,7 @@
 This file contains wrappers for infi.clckhouse_orm engines to use in django-clickhouse
 """
 import datetime
-from typing import List, TypeVar, Type
+from typing import List, TypeVar, Type, Union
 
 from django.db.models import Model as DjangoModel
 from infi.clickhouse_orm import engines as infi_engines
@@ -74,7 +74,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
         qs = connections[db_alias].select(query, model_class=model_cls)
         return list(qs)
 
-    def get_final_versions(self, model_cls, objects):
+    def get_final_versions(self, model_cls, objects, date_col=None):
         """
         Get objects, that are currently stored in ClickHouse.
         Depending on the partition key this can be different for different models.
@@ -84,9 +84,19 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
         :param objects: Objects for which final versions are searched
         :return: A list of model objects
         """
+
+        def _dt_to_str(dt):  # type: (Union[datetime.date, datetime.datetime]) -> str
+            if isinstance(dt, datetime.datetime):
+                return format_datetime(dt, 0, db_alias=db_alias)
+            elif isinstance(dt, datetime.date):
+                return dt.isoformat()
+            else:
+                raise Exception('Invalid date or datetime object: `%s`' % dt)
+
         if not objects:
             return []
 
+        date_col = date_col or self.date_col
         min_date, max_date = None, None
         for obj in objects:
             obj_date = getattr(obj, self.date_col)
@@ -101,15 +111,8 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
 
         db_alias = model_cls.get_database_alias()
 
-        if isinstance(min_date, datetime.date):
+        min_date = _dt_to_str(min_date)
-            min_date = min_date.isoformat()
+        max_date = _dt_to_str(max_date)
-        else:
-            min_date = format_datetime(min_date, 0, db_alias=db_alias)
-
-        if isinstance(max_date, datetime.date):
-            max_date = max_date.isoformat()
-        else:
-            max_date = format_datetime(max_date, 0, day_end=True, db_alias=db_alias)
 
         if self.version_col:
             return self._get_final_versions_by_version(db_alias, model_cls, min_date, max_date, object_pks)

tests/clickhouse_models.py

@@ -25,7 +25,8 @@ class ClickHouseCollapseTestModel(ClickHouseModel):
     sync_enabled = True
 
     id = fields.Int32Field()
-    created_date = fields.DateField()
+    created_date = fields.DateField(materialized='toDate(created)')
+    created = fields.DateTimeField()
     value = fields.Int32Field()
     sign = fields.Int8Field()
     version = fields.Int8Field(default=1)

tests/fixtures/test_model.json

@@ -4,7 +4,8 @@
     "pk": 1,
     "fields": {
       "value": 100,
-      "created_date": "2018-01-01"
+      "created_date": "2018-01-01",
+      "created": "2018-02-01 00:00:00"
     }
   },
   {
@@ -12,7 +13,8 @@
     "pk": 2,
     "fields": {
       "value": 200,
-      "created_date": "2018-02-01"
+      "created_date": "2018-02-01",
+      "created": "2018-02-01 00:00:00"
     }
   }
 ]

tests/fixtures/test_secondary_model.json

@@ -4,7 +4,8 @@
     "pk": 1,
     "fields": {
       "value": 100,
-      "created_date": "2018-01-01"
+      "created_date": "2018-01-01",
+      "created": "2018-02-01 00:00:00"
     }
   },
   {
@@ -12,7 +13,8 @@
     "pk": 2,
     "fields": {
       "value": 200,
-      "created_date": "2018-02-01"
+      "created_date": "2018-02-01",
+      "created": "2018-02-01 00:00:00"
     }
   }
 ]

tests/kill_test_sub_process.py

@@ -25,7 +25,8 @@ logger = logging.getLogger('django-clickhouse')
 def create(batch_size=1000, test_time=60, period=1, **kwargs):
     for iteration in range(int(test_time / period)):
         res = TestModel.objects.db_manager('test_db').bulk_create([
-            TestModel(created_date='2018-01-01', value=iteration * batch_size + i) for i in range(batch_size)
+            TestModel(created=datetime.datetime.now(), created_date='2018-01-01', value=iteration * batch_size + i)
+            for i in range(batch_size)
         ])
         logger.info('django-clickhouse: test created %d records' % len(res))
         sleep(period)

tests/migrations/0001_initial.py

@@ -18,6 +18,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField()),
                 ('value', models.IntegerField()),
                 ('created_date', models.DateField()),
+                ('created', models.DateTimeField()),
             ],
             options={
                 'abstract': False,
@@ -29,6 +30,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField()),
                 ('value', models.IntegerField()),
                 ('created_date', models.DateField()),
+                ('created', models.DateTimeField()),
             ],
             options={
                 'abstract': False,

tests/models.py

@@ -9,8 +9,10 @@ from django_clickhouse.models import ClickHouseSyncModel
 class TestModel(ClickHouseSyncModel):
     value = models.IntegerField()
     created_date = models.DateField()
+    created = models.DateTimeField()
 
 
 class SecondaryTestModel(ClickHouseSyncModel):
     value = models.IntegerField()
     created_date = models.DateField()
+    created = models.DateTimeField()

tests/test_engines.py

@@ -1,5 +1,6 @@
 import datetime
 
+import pytz
 from django.test import TestCase
 from django_clickhouse.migrations import migrate_app
 
@@ -14,37 +15,37 @@ class CollapsingMergeTreeTest(TestCase):
 
     collapse_fixture = [{
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": 1,
         "version": 1
     }, {
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": -1,
         "version": 1
     }, {
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": 1,
         "version": 2
     }, {
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": -1,
         "version": 2
     }, {
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": 1,
         "version": 3
     }, {
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": -1,
         "version": 3
     }, {
         "id": 1,
-        "created_date": "2018-01-01",
+        "created": "2018-01-01 00:00:00",
         "sign": 1,
         "version": 4
     }]
@@ -61,19 +62,36 @@ class CollapsingMergeTreeTest(TestCase):
         ])
         self.objects = TestModel.objects.filter(id=1)
 
-    def test_get_final_versions_by_final(self):
+    def test_get_final_versions_by_final_date(self):
         final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
                                                                                self.objects)
 
         self.assertEqual(1, len(final_versions))
-        self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0, 'created_date': datetime.date(2018, 1, 1)},
+        self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
-                             final_versions[0].to_dict())
+                             final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))
 
-    def test_get_final_versions_by_version(self):
+    def test_get_final_versions_by_version_date(self):
         ClickHouseCollapseTestModel.engine.version_col = 'version'
         final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
                                                                                self.objects)
 
         self.assertEqual(1, len(final_versions))
-        self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0, 'created_date': datetime.date(2018, 1, 1)},
+        self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
-                             final_versions[0].to_dict())
+                             final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))
+
+    def test_get_final_versions_by_final_datetime(self):
+        final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
+                                                                               self.objects, date_col='created')
+
+        self.assertEqual(1, len(final_versions))
+        self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
+                             final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))
+
+    def test_get_final_versions_by_version_datetime(self):
+        ClickHouseCollapseTestModel.engine.version_col = 'version'
+        final_versions = ClickHouseCollapseTestModel.engine.get_final_versions(ClickHouseCollapseTestModel,
+                                                                               self.objects, date_col='created')
+
+        self.assertEqual(1, len(final_versions))
+        self.assertDictEqual({'id': 1, 'sign': 1, 'version': 4, 'value': 0},
+                             final_versions[0].to_dict(field_names=('id', 'sign', 'value', 'version')))

tests/test_models.py

@@ -34,7 +34,7 @@ class TestOperations(TransactionTestCase):
 
     def test_save(self):
         # INSERT operation
-        instance = self.django_model(created_date=datetime.date.today(), value=2)
+        instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2)
         instance.save()
         self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
@@ -46,12 +46,14 @@ class TestOperations(TransactionTestCase):
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_create(self):
-        instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(), value=2)
+        instance = self.django_model.objects.create(pk=100555, created_date=datetime.date.today(),
+                                                    created=datetime.datetime.now(), value=2)
         self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_bulk_create(self):
-        items = [self.django_model(created_date=datetime.date.today(), value=i) for i in range(5)]
+        items = [self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i)
+                 for i in range(5)]
         items = self.django_model.objects.bulk_create(items)
         self.assertEqual(5, len(items))
         self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items],
@@ -59,7 +61,8 @@ class TestOperations(TransactionTestCase):
 
     def test_get_or_create(self):
         instance, created = self.django_model.objects. \
-            get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'value': 2})
+            get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': datetime.datetime.now(),
+                                            'value': 2})
 
         self.assertTrue(created)
         self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
@@ -74,7 +77,8 @@ class TestOperations(TransactionTestCase):
 
     def test_update_or_create(self):
         instance, created = self.django_model.objects. \
-            update_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'value': 2})
+            update_or_create(pk=100, defaults={'created_date': datetime.date.today(),
+                                               'created': datetime.datetime.now(), 'value': 2})
         self.assertTrue(created)
         self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))

tests/test_sync.py

@@ -27,7 +27,7 @@ class SyncTest(TransactionTestCase):
         ClickHouseTestModel.get_storage().flush()
 
     def test_simple(self):
-        obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
+        obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
         ClickHouseTestModel.sync_batch_from_storage()
 
         synced_data = list(ClickHouseTestModel.objects.all())
@@ -37,7 +37,7 @@ class SyncTest(TransactionTestCase):
         self.assertEqual(obj.id, synced_data[0].id)
 
     def test_collapsing_update_by_final(self):
-        obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
+        obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
         obj.value = 2
         obj.save()
         ClickHouseCollapseTestModel.sync_batch_from_storage()
@@ -55,14 +55,13 @@ class SyncTest(TransactionTestCase):
 
         synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel))
         self.assertGreaterEqual(1, len(synced_data))
-        self.assertEqual(obj.created_date, synced_data[0].created_date)
         self.assertEqual(obj.value, synced_data[0].value)
         self.assertEqual(obj.id, synced_data[0].id)
 
     def test_collapsing_update_by_version(self):
         ClickHouseCollapseTestModel.engine.version_col = 'version'
 
-        obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
+        obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
         obj.value = 2
         obj.save()
         ClickHouseCollapseTestModel.sync_batch_from_storage()
@@ -80,7 +79,6 @@ class SyncTest(TransactionTestCase):
 
         synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel))
         self.assertGreaterEqual(1, len(synced_data))
-        self.assertEqual(obj.created_date, synced_data[0].created_date)
         self.assertEqual(obj.value, synced_data[0].value)
         self.assertEqual(obj.id, synced_data[0].id)
 
@@ -98,7 +96,7 @@ class SyncTest(TransactionTestCase):
         self.assertEqual(0, len(synced_data))
 
     def test_multi_model(self):
-        obj = TestModel.objects.create(value=1, created_date=datetime.date.today())
+        obj = TestModel.objects.create(value=1, created=datetime.datetime.now(), created_date=datetime.date.today())
         obj.value = 2
         obj.save()
         ClickHouseMultiTestModel.sync_batch_from_storage()
@@ -122,7 +120,6 @@ class SyncTest(TransactionTestCase):
 
         synced_data = list(self.db.select('SELECT * FROM $table FINAL', model_class=ClickHouseCollapseTestModel))
         self.assertGreaterEqual(1, len(synced_data))
-        self.assertEqual(obj.created_date, synced_data[0].created_date)
         self.assertEqual(obj.value, synced_data[0].value)
         self.assertEqual(obj.id, synced_data[0].id)