From ac8d34c261df37b3dfd281b7e0ef3ab431a08fd7 Mon Sep 17 00:00:00 2001 From: "M1ha.Shvn" Date: Wed, 19 Oct 2022 12:24:53 +0500 Subject: [PATCH 1/3] Added ability to add hints to RunPython and RunSQL migration operations --- docs/migrations.md | 6 ++++++ setup.py | 2 +- src/django_clickhouse/migrations.py | 13 +++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/migrations.md b/docs/migrations.md index 72dc84e..3635588 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -75,6 +75,12 @@ Parameters * `--verbosity: Optional[int] = 1` - Level of debug output. See [here](https://docs.djangoproject.com/en/3.2/ref/django-admin/#cmdoption-verbosity) for more details. * `--help` - Print help + +## Migration operations enhancements +* `RunSQL`, `RunPython` + Can accept `hints: dict = {}` parameter in order to set migration database alias (`force_migrate_on_databases: List[str]` key) or model (`model: Union[str, Type[ClickHouseModel]]` key) + + ## Migration algorithm - Get a list of databases from `CLICKHOUSE_DATABASES` setting. Migrate them one by one. - Find all django apps from `INSTALLED_APPS` setting, which have no `readonly=True` attribute and have `migrate=True` attribute. Migrate them one by one. diff --git a/setup.py b/setup.py index 261d418..02a22a3 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ with open('requirements.txt') as f: setup( name='django-clickhouse', - version='1.1.2', + version='1.2.0', packages=['django_clickhouse', 'django_clickhouse.management.commands'], package_dir={'': 'src'}, url='https://github.com/carrotquest/django-clickhouse', diff --git a/src/django_clickhouse/migrations.py b/src/django_clickhouse/migrations.py index edc4f03..0f00719 100644 --- a/src/django_clickhouse/migrations.py +++ b/src/django_clickhouse/migrations.py @@ -10,6 +10,7 @@ from django.dispatch import receiver # In order to support all operations import here from infi.clickhouse_orm.migrations import * # noqa F401, F403 +from infi.clickhouse_orm.migrations import RunSQL as LibRunSQL, RunPython as LibRunPython from infi.clickhouse_orm.database import ServerError, DatabaseException from infi.clickhouse_orm.fields import StringField, DateField @@ -176,3 +177,15 @@ class MigrationHistory(ClickHouseModel): @classmethod def table_name(cls): return 'django_clickhouse_migrations' + + +class RunSQL(LibRunSQL): + def __init__(self, *args, hints: Optional[dict] = None, **kwargs): + super().__init__(*args, **kwargs) + self.hints = hints or {} + + +class RunPython(LibRunSQL): + def __init__(self, *args, hints: Optional[dict] = None, **kwargs): + super().__init__(*args, **kwargs) + self.hints = hints or {} From 9fb93060ef64cb693753a2b6fe8b8ffd1c8a4f4c Mon Sep 17 00:00:00 2001 From: "M1ha.Shvn" Date: Wed, 19 Oct 2022 12:32:30 +0500 Subject: [PATCH 2/3] Bug fix --- src/django_clickhouse/migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django_clickhouse/migrations.py b/src/django_clickhouse/migrations.py index 0f00719..38c785c 100644 --- a/src/django_clickhouse/migrations.py +++ b/src/django_clickhouse/migrations.py @@ -185,7 +185,7 @@ class RunSQL(LibRunSQL): self.hints = hints or {} -class RunPython(LibRunSQL): +class RunPython(LibRunPython): def __init__(self, *args, hints: Optional[dict] = None, **kwargs): super().__init__(*args, **kwargs) self.hints = hints or {} From fc362e852f11671253917b1cce95234bf7270c99 Mon Sep 17 00:00:00 2001 From: M1ha Shvn Date: Wed, 19 Oct 2022 11:09:10 +0300 Subject: [PATCH 3/3] Added tests on issue-42 (#45) Fixed issue https://github.com/carrotquest/django-clickhouse/issues/42 --- src/django_clickhouse/models.py | 2 +- tests/test_models.py | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/django_clickhouse/models.py b/src/django_clickhouse/models.py index cd7c8ec..9f997da 100644 --- a/src/django_clickhouse/models.py +++ b/src/django_clickhouse/models.py @@ -64,7 +64,7 @@ class ClickHouseSyncBulkUpdateQuerySetMixin(ClickHouseSyncRegisterMixin, BulkUpd if returning is None: returning = pk_name elif isinstance(returning, str): - returning = [pk_name, returning] + returning = [pk_name, returning] if returning != '*' else '*' else: returning = list(returning) + [pk_name] diff --git a/tests/test_models.py b/tests/test_models.py index cf378f2..56580bd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -99,6 +99,22 @@ class TestOperations(TransactionTestCase): self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items}, set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) + def test_pg_bulk_create_returning(self): + now_dt = now() + res = self.django_model.objects.pg_bulk_create([ + {'value': i, 'created': now_dt, 'created_date': now_dt.date()} + for i in range(5) + ], returning='*') + + self.assertEqual(5, len(res)) + for i, instance in enumerate(res): + self.assertEqual(instance.created, now_dt) + self.assertEqual(instance.created_date, now_dt.date()) + self.assertEqual(i, instance.value) + + self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in res}, + set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) + def test_pg_bulk_update(self): items = list(self.django_model.objects.filter(pk__in={1, 2})) @@ -115,6 +131,21 @@ class TestOperations(TransactionTestCase): self.assertSetEqual({('update', "%s.%d" % (self.db_alias, instance.pk)) for instance in items}, set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) + def test_pg_bulk_update_returning(self): + items = list(self.django_model.objects.filter(pk__in={1, 2})) + + res = self.django_model.objects.pg_bulk_update([ + {'id': instance.pk, 'value': instance.pk * 10} + for instance in items + ], returning='*') + + self.assertEqual(2, len(res)) + for instance in res: + self.assertEqual(instance.value, instance.pk * 10) + + self.assertSetEqual({('update', "%s.%d" % (self.db_alias, instance.pk)) for instance in items}, + set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) + def test_pg_bulk_update_or_create(self): items = list(self.django_model.objects.filter(pk__in={1, 2})) @@ -135,6 +166,25 @@ class TestOperations(TransactionTestCase): self.assertSetEqual({('update', "%s.%d" % (self.db_alias, instance.pk)) for instance in items}, set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) + def test_pg_bulk_update_or_create_returning(self): + items = list(self.django_model.objects.filter(pk__in={1, 2})) + + data = [{ + 'id': instance.pk, + 'value': instance.pk * 10, + 'created_date': instance.created_date, + 'created': instance.created + } for instance in items] + [{'id': 11, 'value': 110, 'created_date': datetime.date.today(), 'created': now()}] + + res = self.django_model.objects.pg_bulk_update_or_create(data, returning='*') + + self.assertEqual(3, len(res)) + for instance in res: + self.assertEqual(instance.value, instance.pk * 10) + + self.assertSetEqual({('update', "%s.%d" % (self.db_alias, instance.pk)) for instance in res}, + set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))) + def test_get_or_create(self): instance, created = self.django_model.objects. \ get_or_create(pk=100, defaults={'created_date': datetime.date.today(), 'created': datetime.datetime.now(),