From 0a7f0c121929d54d63e3246649ef9d1685ee9bcf Mon Sep 17 00:00:00 2001 From: M1ha Shvn Date: Sat, 25 Sep 2021 23:00:28 +0500 Subject: [PATCH] Implemented `clickhouse_migrate` management command (#33) 1. Implemented `clickhouse_migrate` management command 2. Ability to print more verbose output when running `manage.py migrate` django command --- docs/migrations.md | 18 +++ setup.py | 4 +- src/django_clickhouse/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/clickhouse_migrate.py | 45 +++++++ src/django_clickhouse/migrations.py | 55 ++++++--- tests/test_migrations.py | 110 +++++++++++++++++- 7 files changed, 210 insertions(+), 22 deletions(-) create mode 100644 src/django_clickhouse/management/__init__.py create mode 100644 src/django_clickhouse/management/commands/__init__.py create mode 100644 src/django_clickhouse/management/commands/clickhouse_migrate.py diff --git a/docs/migrations.md b/docs/migrations.md index f71d77b..72dc84e 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -57,6 +57,24 @@ By default migrations are applied to all [CLICKHOUSE_DATABASES](configuration.md Note: migrations are only applied, with django `default` database. So if you call `python manage.py migrate --database=secondary` they wouldn't be applied. +## Admin migration command +In order to make migrations separately from django's `manage.py migrate` command, +this library implements custom `manage.py` command `clickhouse_migrate`. + +Usage: +```bash +python manage.py clickhouse_migrate [--help] [--database ] [--verbosity {0,1,2,3}] [app_label] [migration_number] +``` + +Parameters +* `app_label: Optional[str]` - If set, migrates only given django application +* `migration_number: Optional[int]` - If set, migrate django app with `app_label` to migration with this number + **Important note**: Library currently does not support unapplying migrations. + If already applied migration is given - it will do noting. +* `--database: Optional[str]` - If set, migrates only this database alias from [CLICKHOUSE_DATABASES config parameter](configuration.md#clickhouse_databases) +* `--verbosity: Optional[int] = 1` - Level of debug output. See [here](https://docs.djangoproject.com/en/3.2/ref/django-admin/#cmdoption-verbosity) for more details. +* `--help` - Print help + ## Migration algorithm - Get a list of databases from `CLICKHOUSE_DATABASES` setting. Migrate them one by one. - Find all django apps from `INSTALLED_APPS` setting, which have no `readonly=True` attribute and have `migrate=True` attribute. Migrate them one by one. diff --git a/setup.py b/setup.py index 1ca5fc3..27c53ff 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ with open('requirements.txt') as f: setup( name='django-clickhouse', - version='1.0.4', - packages=['django_clickhouse'], + version='1.1.0', + packages=['django_clickhouse', 'django_clickhouse.management.commands'], package_dir={'': 'src'}, url='https://github.com/carrotquest/django-clickhouse', license='BSD 3-clause "New" or "Revised" License', diff --git a/src/django_clickhouse/management/__init__.py b/src/django_clickhouse/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/django_clickhouse/management/commands/__init__.py b/src/django_clickhouse/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/django_clickhouse/management/commands/clickhouse_migrate.py b/src/django_clickhouse/management/commands/clickhouse_migrate.py new file mode 100644 index 0000000..eef9aa5 --- /dev/null +++ b/src/django_clickhouse/management/commands/clickhouse_migrate.py @@ -0,0 +1,45 @@ +""" +Django command that applies migrations for ClickHouse database +""" +import json + +from django.conf import settings +from django.core.management import BaseCommand, CommandParser + +from ...configuration import config +from ...migrations import migrate_app + + +class Command(BaseCommand): + help = 'Migrates ClickHouse databases' + requires_migrations_checks = False + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument('app_label', nargs='?', type=str, + help='Django App name to migrate. By default all found apps are migrated.') + + parser.add_argument('migration_number', nargs='?', type=int, + help='Migration number in selected django app to migrate to.' + ' By default all available migrations are applied.' + ' Note that library currently have no ability rollback migrations') + + parser.add_argument('--database', '-d', nargs='?', type=str, required=False, choices=config.DATABASES.keys(), + help='ClickHouse database alias key from CLICKHOUSE_DATABASES django setting.' + ' By defaults migrations are applied to all databases.') + + def handle(self, *args, **options) -> None: + apps = [options['app_label']] if options['app_label'] else list(settings.INSTALLED_APPS) + databases = [options['database']] if options['database'] else list(config.DATABASES.keys()) + kwargs = {'up_to': options['migration_number']} if options['migration_number'] else {} + + self.stdout.write(self.style.MIGRATE_HEADING( + "Applying ClickHouse migrations for apps %s in databases %s" % (json.dumps(apps), json.dumps(databases)))) + + any_migrations_applied = False + for app_label in apps: + for db_alias in databases: + res = migrate_app(app_label, db_alias, verbosity=options['verbosity'], **kwargs) + any_migrations_applied = any_migrations_applied or res + + if not any_migrations_applied: + self.stdout.write("No migrations to apply") diff --git a/src/django_clickhouse/migrations.py b/src/django_clickhouse/migrations.py index a5bcd5f..f38c261 100644 --- a/src/django_clickhouse/migrations.py +++ b/src/django_clickhouse/migrations.py @@ -46,42 +46,63 @@ class Migration: op.apply(database) -def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None) -> None: +def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None, + verbosity: int = 1) -> bool: """ Migrates given django app :param app_label: App label to migrate :param db_alias: Database alias to migrate :param up_to: Migration number to migrate to :param database: Sometimes I want to pass db object directly for testing purposes - :return: None + :param verbosity: 0-4, уровень verbosity вывода + :return: True if any migration has been applied """ # Can't migrate such connection, just skip it if config.DATABASES[db_alias].get('readonly', False): - return + if verbosity > 1: + print('Skipping database "%s": marked as readonly' % db_alias) + return False # Ignore force not migrated databases if not config.DATABASES[db_alias].get('migrate', True): - return + if verbosity > 1: + print('Skipping database "%s": migrations are restricted in configuration' % db_alias) + return False migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE) - if module_exists(migrations_package): - database = database or connections[db_alias] - migration_history_model = lazy_class_import(config.MIGRATION_HISTORY_MODEL) + if not module_exists(migrations_package): + if verbosity > 1: + print('Skipping migrations for app "%s": no migration_package "%s"' % (app_label, migrations_package)) + return False - applied_migrations = migration_history_model.get_applied_migrations(db_alias, migrations_package) - modules = import_submodules(migrations_package) - unapplied_migrations = set(modules.keys()) - applied_migrations + database = database or connections[db_alias] + migration_history_model = lazy_class_import(config.MIGRATION_HISTORY_MODEL) - for name in sorted(unapplied_migrations): + applied_migrations = migration_history_model.get_applied_migrations(db_alias, migrations_package) + modules = import_submodules(migrations_package) + unapplied_migrations = set(modules.keys()) - applied_migrations + + any_applied = False + for name in sorted(unapplied_migrations): + if int(name[:4]) > up_to: + break + + if verbosity > 0: print('Applying ClickHouse migration %s for app %s in database %s' % (name, app_label, db_alias)) - migration = modules[name].Migration() - migration.apply(db_alias, database=database) - migration_history_model.set_migration_applied(db_alias, migrations_package, name) + migration = modules[name].Migration() + migration.apply(db_alias, database=database) - if int(name[:4]) >= up_to: - break + migration_history_model.set_migration_applied(db_alias, migrations_package, name) + any_applied = True + + if not any_applied: + if verbosity > 1: + print('No migrations to apply for app "%s" does not exist' % app_label) + return False + + return True @receiver(post_migrate) @@ -97,7 +118,7 @@ def clickhouse_migrate(sender, **kwargs): app_name = kwargs['app_config'].name for db_alias in config.DATABASES: - migrate_app(app_name, db_alias) + migrate_app(app_name, db_alias, verbosity=kwargs.get('verbosity', 1)) class MigrationHistory(ClickHouseModel): diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 4577806..70a8c27 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,8 +1,13 @@ -from django.test import TestCase, override_settings -from django_clickhouse.migrations import MigrationHistory +from typing import List, Dict, Any +from unittest import mock +from django.conf import settings +from django.test import TestCase, override_settings + +from django_clickhouse.configuration import config from django_clickhouse.database import connections -from django_clickhouse.migrations import migrate_app +from django_clickhouse.management.commands.clickhouse_migrate import Command +from django_clickhouse.migrations import MigrationHistory, migrate_app from django_clickhouse.routers import DefaultRouter from tests.clickhouse_models import ClickHouseTestModel @@ -53,3 +58,102 @@ class MigrateAppTest(TestCase): def test_readonly_connections(self): migrate_app('tests', 'readonly') self.assertFalse(table_exists(connections['readonly'], ClickHouseTestModel)) + + +@override_settings(CLICKHOUSE_MIGRATE_WITH_DEFAULT_DB=False) +@mock.patch('django_clickhouse.management.commands.clickhouse_migrate.migrate_app', return_value=True) +class MigrateDjangoCommandTest(TestCase): + def setUp(self) -> None: + self.cmd = Command() + + def test_handle_all(self, migrate_app_mock): + self.cmd.handle(verbosity=3, app_label=None, database=None, migration_number=None) + + self.assertEqual(len(config.DATABASES.keys()) * len(settings.INSTALLED_APPS), migrate_app_mock.call_count) + for db_alias in config.DATABASES.keys(): + for app_label in settings.INSTALLED_APPS: + migrate_app_mock.assert_any_call(app_label, db_alias, verbosity=3) + + def test_handle_app(self, migrate_app_mock): + self.cmd.handle(verbosity=3, app_label='tests', database=None, migration_number=None) + + self.assertEqual(len(config.DATABASES.keys()), migrate_app_mock.call_count) + for db_alias in config.DATABASES.keys(): + migrate_app_mock.assert_any_call('tests', db_alias, verbosity=3) + + def test_handle_database(self, migrate_app_mock): + self.cmd.handle(verbosity=3, database='default', app_label=None, migration_number=None) + + self.assertEqual(len(settings.INSTALLED_APPS), migrate_app_mock.call_count) + for app_label in settings.INSTALLED_APPS: + migrate_app_mock.assert_any_call(app_label, 'default', verbosity=3) + + def test_handle_app_and_database(self, migrate_app_mock): + self.cmd.handle(verbosity=3, app_label='tests', database='default', migration_number=None) + + migrate_app_mock.assert_called_with('tests', 'default', verbosity=3) + + def test_handle_migration_number(self, migrate_app_mock): + self.cmd.handle(verbosity=3, database='default', app_label='tests', migration_number=1) + + migrate_app_mock.assert_called_with('tests', 'default', up_to=1, verbosity=3) + + def _test_parser_results(self, argv: List[str], expected: Dict[str, Any]) -> None: + """ + Tests if parser process input correctly. + Checks only expected parameters, ignores others. + :param argv: List of string arguments from command line + :param expected: Dictionary of expected results + :return: None + :raises AssertionError: If expected result is incorrect + """ + parser = self.cmd.create_parser('./manage.py', 'clickhouse_migrate') + + options = parser.parse_args(argv) + + # Copied from django.core.management.base.BaseCommand.run_from_argv('...') + cmd_options = vars(options) + cmd_options.pop('args', ()) + + self.assertDictEqual(expected, {opt: cmd_options[opt] for opt in expected.keys()}) + + def test_parser(self, _): + with self.subTest('Simple'): + self._test_parser_results([], { + 'app_label': None, + 'database': None, + 'migration_number': None, + 'verbosity': 1 + }) + + with self.subTest('App label'): + self._test_parser_results(['tests'], { + 'app_label': 'tests', + 'database': None, + 'migration_number': None, + 'verbosity': 1 + }) + + with self.subTest('App label and migration number'): + self._test_parser_results(['tests', '123'], { + 'app_label': 'tests', + 'database': None, + 'migration_number': 123, + 'verbosity': 1 + }) + + with self.subTest('Database'): + self._test_parser_results(['--database', 'default'], { + 'app_label': None, + 'database': 'default', + 'migration_number': None, + 'verbosity': 1 + }) + + with self.subTest('Verbosity'): + self._test_parser_results(['--verbosity', '2'], { + 'app_label': None, + 'database': None, + 'migration_number': None, + 'verbosity': 2 + })