From fafcbe4c9336463ae86eb501292ad17c0868b49f Mon Sep 17 00:00:00 2001 From: M1ha Date: Wed, 26 Jun 2019 12:24:46 +0500 Subject: [PATCH] Fixed python 3.7 compatibility issues --- src/django_clickhouse/clickhouse_models.py | 10 ++++---- src/django_clickhouse/database.py | 6 +++-- tests/clickhouse_models.py | 2 +- tests/fixtures/test_model.json | 27 ++++++++++++++++++++++ tests/test_database.py | 16 +++++++++++++ tests/test_models.py | 14 +++++------ 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py index 80b0446..967b59f 100644 --- a/src/django_clickhouse/clickhouse_models.py +++ b/src/django_clickhouse/clickhouse_models.py @@ -66,9 +66,6 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)): def get_tuple_class(cls, field_names=None, defaults=None): field_names = field_names or cls.fields(writable=False).keys() - # Strange, but sometimes the columns are in different order... - field_names = sorted(field_names) - if defaults: defaults_new = deepcopy(cls._defaults) defaults_new.update(defaults) @@ -78,9 +75,12 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)): # defaults should be rightmost arguments required_field_names = tuple(name for name in field_names if name not in defaults_new) - default_field_names, default_values = zip(*defaults_new.items()) + default_field_names, default_values = zip(*sorted(defaults_new.items(), key=lambda t: t[0])) - return namedtuple("%sTuple" % cls.__name__, required_field_names + default_field_names, defaults=default_values) + # Strange, but sometimes the columns are in different order... + field_names = tuple(sorted(required_field_names)) + default_field_names + + return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values) @classmethod def objects_in(cls, database): # type: (Database) -> QuerySet diff --git a/src/django_clickhouse/database.py b/src/django_clickhouse/database.py index ee96995..8c6c930 100644 --- a/src/django_clickhouse/database.py +++ b/src/django_clickhouse/database.py @@ -88,11 +88,13 @@ class Database(InfiDatabase): fields_list = ','.join('`%s`' % name for name in first_tuple._fields) fields_dict = model_class.fields(writable=True) - fields = [fields_dict[name] for name in first_tuple._fields] statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__) def tuple_to_csv(tup): - return '\t'.join(field.to_db_string(val, quote=False) for field, val in zip(fields, tup)) + '\n' + return '\t'.join( + fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False) + for field_name in first_tuple._fields + ) + '\n' def gen(): buf = BytesIO() diff --git a/tests/clickhouse_models.py b/tests/clickhouse_models.py index 541f33d..77893a3 100644 --- a/tests/clickhouse_models.py +++ b/tests/clickhouse_models.py @@ -12,7 +12,7 @@ class ClickHouseTestModel(ClickHouseModel): id = fields.Int32Field() created_date = fields.DateField() - value = fields.Int32Field() + value = fields.Int32Field(default=100500) str_field = fields.StringField() engine = ReplacingMergeTree('created_date', ('id',)) diff --git a/tests/fixtures/test_model.json b/tests/fixtures/test_model.json index 35a1145..aaf7400 100644 --- a/tests/fixtures/test_model.json +++ b/tests/fixtures/test_model.json @@ -16,5 +16,32 @@ "created_date": "2018-02-01", "created": "2018-02-01 00:00:00" } + }, + { + "model": "tests.TestModel", + "pk": 3, + "fields": { + "value": 300, + "created_date": "2018-03-01", + "created": "2018-03-01 00:00:00" + } + }, + { + "model": "tests.TestModel", + "pk": 4, + "fields": { + "value": 400, + "created_date": "2018-04-01", + "created": "2018-04-01 00:00:00" + } + }, + { + "model": "tests.TestModel", + "pk": 5, + "fields": { + "value": 500, + "created_date": "2018-05-01", + "created": "2018-05-01 00:00:00" + } } ] \ No newline at end of file diff --git a/tests/test_database.py b/tests/test_database.py index 4549131..120a3b8 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -31,6 +31,22 @@ class CollapsingMergeTreeTest(TestCase): 'str_field': str(i) } for i in range(10)], [item.to_dict() for item in qs]) + def test_insert_tuples_defaults(self): + tuple_class = ClickHouseTestModel.get_tuple_class(defaults={'created_date': date.today()}) + data = [ + tuple_class(id=i, str_field=str(i)) + for i in range(10) + ] + self.db.insert_tuples(ClickHouseTestModel, data) + + qs = ClickHouseTestModel.objects.order_by('id').all() + self.assertListEqual([{ + 'id': i, + 'created_date': date.today(), + 'value': 100500, + 'str_field': str(i) + } for i in range(10)], [item.to_dict() for item in qs]) + def test_insert_tuples_batch_size(self): tuple_class = ClickHouseTestModel.get_tuple_class() data = [ diff --git a/tests/test_models.py b/tests/test_models.py index 4202a9d..0de5c92 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -28,6 +28,7 @@ class TestOperations(TransactionTestCase): def setUp(self): self.storage = self.django_model.get_clickhouse_storage() self.storage.flush() + self.before_op_items = list(self.django_model.objects.all()) def tearDown(self): self.storage.flush() @@ -56,8 +57,8 @@ class TestOperations(TransactionTestCase): for i in range(5)] items = self.django_model.objects.bulk_create(items) self.assertEqual(5, len(items)) - self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items], - self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) + self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items}, + set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) def test_get_or_create(self): instance, created = self.django_model.objects. \ @@ -96,10 +97,9 @@ class TestOperations(TransactionTestCase): self.assertListEqual([('update', "%s.1" % self.db_alias)], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) - # Update, after which updated element will not suit update conditions self.django_model.objects.filter(created_date__lt=datetime.date.today()). \ update(created_date=datetime.date.today()) - self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)], + self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_qs_update_returning(self): @@ -110,7 +110,7 @@ class TestOperations(TransactionTestCase): # Update, after which updated element will not suit update conditions self.django_model.objects.filter(created_date__lt=datetime.date.today()). \ update_returning(created_date=datetime.date.today()) - self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)], + self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_qs_delete_returning(self): @@ -118,9 +118,9 @@ class TestOperations(TransactionTestCase): self.assertListEqual([('delete', "%s.1" % self.db_alias)], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) - # Update, после которого исходный фильтр уже не сработает + # Delete, after which updated element will not suit update conditions self.django_model.objects.filter(created_date__lt=datetime.date.today()).delete_returning() - self.assertListEqual([('delete', "%s.1" % self.db_alias), ('delete', "%s.2" % self.db_alias)], + self.assertListEqual([('delete', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items], self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)) def test_delete(self):