Fixed python 3.7 compatibility issues

src/django_clickhouse/clickhouse_models.py

@@ -66,9 +66,6 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
     def get_tuple_class(cls, field_names=None, defaults=None):
         field_names = field_names or cls.fields(writable=False).keys()
 
-        # Strange, but sometimes the columns are in different order...
-        field_names = sorted(field_names)
-
         if defaults:
             defaults_new = deepcopy(cls._defaults)
             defaults_new.update(defaults)
@@ -78,9 +75,12 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
         # defaults should be rightmost arguments
         required_field_names = tuple(name for name in field_names if name not in defaults_new)
 
-        default_field_names, default_values = zip(*defaults_new.items())
+        default_field_names, default_values = zip(*sorted(defaults_new.items(), key=lambda t: t[0]))
 
-        return namedtuple("%sTuple" % cls.__name__, required_field_names + default_field_names, defaults=default_values)
+        # Strange, but sometimes the columns are in different order...
+        field_names = tuple(sorted(required_field_names)) + default_field_names
+
+        return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values)
 
     @classmethod
     def objects_in(cls, database):  # type: (Database) -> QuerySet

src/django_clickhouse/database.py

@@ -88,11 +88,13 @@ class Database(InfiDatabase):
 
         fields_list = ','.join('`%s`' % name for name in first_tuple._fields)
         fields_dict = model_class.fields(writable=True)
-        fields = [fields_dict[name] for name in first_tuple._fields]
         statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__)
 
         def tuple_to_csv(tup):
-            return '\t'.join(field.to_db_string(val, quote=False) for field, val in zip(fields, tup)) + '\n'
+            return '\t'.join(
+                fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
+                for field_name in first_tuple._fields
+            ) + '\n'
 
         def gen():
             buf = BytesIO()

tests/clickhouse_models.py

@@ -12,7 +12,7 @@ class ClickHouseTestModel(ClickHouseModel):
 
     id = fields.Int32Field()
     created_date = fields.DateField()
-    value = fields.Int32Field()
+    value = fields.Int32Field(default=100500)
     str_field = fields.StringField()
 
     engine = ReplacingMergeTree('created_date', ('id',))

tests/fixtures/test_model.json

@@ -16,5 +16,32 @@
       "created_date": "2018-02-01",
       "created": "2018-02-01 00:00:00"
     }
+  },
+  {
+    "model": "tests.TestModel",
+    "pk": 3,
+    "fields": {
+      "value": 300,
+      "created_date": "2018-03-01",
+      "created": "2018-03-01 00:00:00"
+    }
+  },
+  {
+    "model": "tests.TestModel",
+    "pk": 4,
+    "fields": {
+      "value": 400,
+      "created_date": "2018-04-01",
+      "created": "2018-04-01 00:00:00"
+    }
+  },
+  {
+    "model": "tests.TestModel",
+    "pk": 5,
+    "fields": {
+      "value": 500,
+      "created_date": "2018-05-01",
+      "created": "2018-05-01 00:00:00"
+    }
   }
 ]

tests/test_database.py

@@ -31,6 +31,22 @@ class CollapsingMergeTreeTest(TestCase):
             'str_field': str(i)
         } for i in range(10)], [item.to_dict() for item in qs])
 
+    def test_insert_tuples_defaults(self):
+        tuple_class = ClickHouseTestModel.get_tuple_class(defaults={'created_date': date.today()})
+        data = [
+            tuple_class(id=i, str_field=str(i))
+            for i in range(10)
+        ]
+        self.db.insert_tuples(ClickHouseTestModel, data)
+
+        qs = ClickHouseTestModel.objects.order_by('id').all()
+        self.assertListEqual([{
+            'id': i,
+            'created_date': date.today(),
+            'value': 100500,
+            'str_field': str(i)
+        } for i in range(10)], [item.to_dict() for item in qs])
+
     def test_insert_tuples_batch_size(self):
         tuple_class = ClickHouseTestModel.get_tuple_class()
         data = [

tests/test_models.py

@@ -28,6 +28,7 @@ class TestOperations(TransactionTestCase):
     def setUp(self):
         self.storage = self.django_model.get_clickhouse_storage()
         self.storage.flush()
+        self.before_op_items = list(self.django_model.objects.all())
 
     def tearDown(self):
         self.storage.flush()
@@ -56,8 +57,8 @@ class TestOperations(TransactionTestCase):
                  for i in range(5)]
         items = self.django_model.objects.bulk_create(items)
         self.assertEqual(5, len(items))
-        self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items],
+        self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items},
-                             self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
+                            set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)))
 
     def test_get_or_create(self):
         instance, created = self.django_model.objects. \
@@ -96,10 +97,9 @@ class TestOperations(TransactionTestCase):
         self.assertListEqual([('update', "%s.1" % self.db_alias)],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
-        # Update, after which updated element will not suit update conditions
         self.django_model.objects.filter(created_date__lt=datetime.date.today()). \
             update(created_date=datetime.date.today())
-        self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)],
+        self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_qs_update_returning(self):
@@ -110,7 +110,7 @@ class TestOperations(TransactionTestCase):
         # Update, after which updated element will not suit update conditions
         self.django_model.objects.filter(created_date__lt=datetime.date.today()). \
             update_returning(created_date=datetime.date.today())
-        self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)],
+        self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_qs_delete_returning(self):
@@ -118,9 +118,9 @@ class TestOperations(TransactionTestCase):
         self.assertListEqual([('delete', "%s.1" % self.db_alias)],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
-        # Update, после которого исходный фильтр уже не сработает
+        # Delete, after which updated element will not suit update conditions
         self.django_model.objects.filter(created_date__lt=datetime.date.today()).delete_returning()
-        self.assertListEqual([('delete', "%s.1" % self.db_alias), ('delete', "%s.2" % self.db_alias)],
+        self.assertListEqual([('delete', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_delete(self):