Fixed python 3.7 compatibility issues

This commit is contained in:
M1ha 2019-06-26 12:24:46 +05:00
parent ae253e0587
commit fafcbe4c93
6 changed files with 60 additions and 15 deletions

View File

@ -66,9 +66,6 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
def get_tuple_class(cls, field_names=None, defaults=None):
field_names = field_names or cls.fields(writable=False).keys()
# Strange, but sometimes the columns are in different order...
field_names = sorted(field_names)
if defaults:
defaults_new = deepcopy(cls._defaults)
defaults_new.update(defaults)
@ -78,9 +75,12 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
# defaults should be rightmost arguments
required_field_names = tuple(name for name in field_names if name not in defaults_new)
default_field_names, default_values = zip(*defaults_new.items())
default_field_names, default_values = zip(*sorted(defaults_new.items(), key=lambda t: t[0]))
return namedtuple("%sTuple" % cls.__name__, required_field_names + default_field_names, defaults=default_values)
# Strange, but sometimes the columns are in different order...
field_names = tuple(sorted(required_field_names)) + default_field_names
return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values)
@classmethod
def objects_in(cls, database): # type: (Database) -> QuerySet

View File

@ -88,11 +88,13 @@ class Database(InfiDatabase):
fields_list = ','.join('`%s`' % name for name in first_tuple._fields)
fields_dict = model_class.fields(writable=True)
fields = [fields_dict[name] for name in first_tuple._fields]
statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__)
def tuple_to_csv(tup):
return '\t'.join(field.to_db_string(val, quote=False) for field, val in zip(fields, tup)) + '\n'
return '\t'.join(
fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
for field_name in first_tuple._fields
) + '\n'
def gen():
buf = BytesIO()

View File

@ -12,7 +12,7 @@ class ClickHouseTestModel(ClickHouseModel):
id = fields.Int32Field()
created_date = fields.DateField()
value = fields.Int32Field()
value = fields.Int32Field(default=100500)
str_field = fields.StringField()
engine = ReplacingMergeTree('created_date', ('id',))

View File

@ -16,5 +16,32 @@
"created_date": "2018-02-01",
"created": "2018-02-01 00:00:00"
}
},
{
"model": "tests.TestModel",
"pk": 3,
"fields": {
"value": 300,
"created_date": "2018-03-01",
"created": "2018-03-01 00:00:00"
}
},
{
"model": "tests.TestModel",
"pk": 4,
"fields": {
"value": 400,
"created_date": "2018-04-01",
"created": "2018-04-01 00:00:00"
}
},
{
"model": "tests.TestModel",
"pk": 5,
"fields": {
"value": 500,
"created_date": "2018-05-01",
"created": "2018-05-01 00:00:00"
}
}
]

View File

@ -31,6 +31,22 @@ class CollapsingMergeTreeTest(TestCase):
'str_field': str(i)
} for i in range(10)], [item.to_dict() for item in qs])
def test_insert_tuples_defaults(self):
tuple_class = ClickHouseTestModel.get_tuple_class(defaults={'created_date': date.today()})
data = [
tuple_class(id=i, str_field=str(i))
for i in range(10)
]
self.db.insert_tuples(ClickHouseTestModel, data)
qs = ClickHouseTestModel.objects.order_by('id').all()
self.assertListEqual([{
'id': i,
'created_date': date.today(),
'value': 100500,
'str_field': str(i)
} for i in range(10)], [item.to_dict() for item in qs])
def test_insert_tuples_batch_size(self):
tuple_class = ClickHouseTestModel.get_tuple_class()
data = [

View File

@ -28,6 +28,7 @@ class TestOperations(TransactionTestCase):
def setUp(self):
self.storage = self.django_model.get_clickhouse_storage()
self.storage.flush()
self.before_op_items = list(self.django_model.objects.all())
def tearDown(self):
self.storage.flush()
@ -56,8 +57,8 @@ class TestOperations(TransactionTestCase):
for i in range(5)]
items = self.django_model.objects.bulk_create(items)
self.assertEqual(5, len(items))
self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items},
set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)))
def test_get_or_create(self):
instance, created = self.django_model.objects. \
@ -96,10 +97,9 @@ class TestOperations(TransactionTestCase):
self.assertListEqual([('update', "%s.1" % self.db_alias)],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
# Update, after which updated element will not suit update conditions
self.django_model.objects.filter(created_date__lt=datetime.date.today()). \
update(created_date=datetime.date.today())
self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)],
self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_qs_update_returning(self):
@ -110,7 +110,7 @@ class TestOperations(TransactionTestCase):
# Update, after which updated element will not suit update conditions
self.django_model.objects.filter(created_date__lt=datetime.date.today()). \
update_returning(created_date=datetime.date.today())
self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)],
self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_qs_delete_returning(self):
@ -118,9 +118,9 @@ class TestOperations(TransactionTestCase):
self.assertListEqual([('delete', "%s.1" % self.db_alias)],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
# Update, после которого исходный фильтр уже не сработает
# Delete, after which updated element will not suit update conditions
self.django_model.objects.filter(created_date__lt=datetime.date.today()).delete_returning()
self.assertListEqual([('delete', "%s.1" % self.db_alias), ('delete', "%s.2" % self.db_alias)],
self.assertListEqual([('delete', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
def test_delete(self):