Implemented clickhouse_migrate management command (#33)

1. Implemented `clickhouse_migrate` management command
2. Ability to print more verbose output when running `manage.py migrate` django command

docs/migrations.md

@@ -57,6 +57,24 @@ By default migrations are applied to all [CLICKHOUSE_DATABASES](configuration.md
 Note: migrations are only applied, with django `default` database.  
 So if you call `python manage.py migrate --database=secondary` they wouldn't be applied.
 
+## Admin migration command
+In order to make migrations separately from django's `manage.py migrate` command,
+this library implements custom `manage.py` command `clickhouse_migrate`.
+
+Usage:
+```bash
+python manage.py clickhouse_migrate [--help] [--database <db_alias>] [--verbosity {0,1,2,3}] [app_label] [migration_number]
+```
+
+Parameters
+* `app_label: Optional[str]` - If set, migrates only given django application  
+* `migration_number: Optional[int]` - If set, migrate django app with `app_label` to migration with this number   
+    **Important note**: Library currently does not support unapplying migrations. 
+      If already applied migration is given - it will do noting.
+* `--database: Optional[str]` - If set, migrates only this database alias from [CLICKHOUSE_DATABASES config parameter](configuration.md#clickhouse_databases)
+* `--verbosity: Optional[int] = 1` - Level of debug output. See [here](https://docs.djangoproject.com/en/3.2/ref/django-admin/#cmdoption-verbosity) for more details.
+* `--help` - Print help
+
 ## Migration algorithm
 - Get a list of databases from `CLICKHOUSE_DATABASES` setting. Migrate them one by one.  
   - Find all django apps from `INSTALLED_APPS` setting, which have no `readonly=True` attribute and have `migrate=True` attribute. Migrate them one by one.  

setup.py

@@ -13,8 +13,8 @@ with open('requirements.txt') as f:
 
 setup(
     name='django-clickhouse',
-    version='1.0.4',
+    version='1.1.0',
-    packages=['django_clickhouse'],
+    packages=['django_clickhouse', 'django_clickhouse.management.commands'],
     package_dir={'': 'src'},
     url='https://github.com/carrotquest/django-clickhouse',
     license='BSD 3-clause "New" or "Revised" License',

src/django_clickhouse/management/commands/clickhouse_migrate.py

@@ -0,0 +1,45 @@
+"""
+Django command that applies migrations for ClickHouse database
+"""
+import json
+
+from django.conf import settings
+from django.core.management import BaseCommand, CommandParser
+
+from ...configuration import config
+from ...migrations import migrate_app
+
+
+class Command(BaseCommand):
+    help = 'Migrates ClickHouse databases'
+    requires_migrations_checks = False
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument('app_label', nargs='?', type=str,
+                            help='Django App name to migrate. By default all found apps are migrated.')
+
+        parser.add_argument('migration_number', nargs='?', type=int,
+                            help='Migration number in selected django app to migrate to.'
+                                 ' By default all available migrations are applied.'
+                                 ' Note that library currently have no ability rollback migrations')
+
+        parser.add_argument('--database', '-d', nargs='?', type=str, required=False, choices=config.DATABASES.keys(),
+                            help='ClickHouse database alias key from CLICKHOUSE_DATABASES django setting.'
+                                 ' By defaults migrations are applied to all databases.')
+
+    def handle(self, *args, **options) -> None:
+        apps = [options['app_label']] if options['app_label'] else list(settings.INSTALLED_APPS)
+        databases = [options['database']] if options['database'] else list(config.DATABASES.keys())
+        kwargs = {'up_to': options['migration_number']} if options['migration_number'] else {}
+
+        self.stdout.write(self.style.MIGRATE_HEADING(
+            "Applying ClickHouse migrations for apps %s in databases %s" % (json.dumps(apps), json.dumps(databases))))
+
+        any_migrations_applied = False
+        for app_label in apps:
+            for db_alias in databases:
+                res = migrate_app(app_label, db_alias, verbosity=options['verbosity'], **kwargs)
+                any_migrations_applied = any_migrations_applied or res
+
+        if not any_migrations_applied:
+            self.stdout.write("No migrations to apply")

src/django_clickhouse/migrations.py

@@ -46,42 +46,63 @@ class Migration:
                 op.apply(database)
 
 
-def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None) -> None:
+def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None,
+                verbosity: int = 1) -> bool:
     """
     Migrates given django app
     :param app_label: App label to migrate
     :param db_alias: Database alias to migrate
     :param up_to: Migration number to migrate to
     :param database: Sometimes I want to pass db object directly for testing purposes
-    :return: None
+    :param verbosity: 0-4, уровень verbosity вывода
+    :return: True if any migration has been applied
     """
     # Can't migrate such connection, just skip it
     if config.DATABASES[db_alias].get('readonly', False):
-        return
+        if verbosity > 1:
+            print('Skipping database "%s": marked as readonly' % db_alias)
+        return False
 
     # Ignore force not migrated databases
     if not config.DATABASES[db_alias].get('migrate', True):
-        return
+        if verbosity > 1:
+            print('Skipping database "%s": migrations are restricted in configuration' % db_alias)
+        return False
 
     migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE)
 
-    if module_exists(migrations_package):
+    if not module_exists(migrations_package):
-        database = database or connections[db_alias]
+        if verbosity > 1:
-        migration_history_model = lazy_class_import(config.MIGRATION_HISTORY_MODEL)
+            print('Skipping migrations for app "%s": no migration_package "%s"' % (app_label, migrations_package))
+        return False
 
-        applied_migrations = migration_history_model.get_applied_migrations(db_alias, migrations_package)
+    database = database or connections[db_alias]
-        modules = import_submodules(migrations_package)
+    migration_history_model = lazy_class_import(config.MIGRATION_HISTORY_MODEL)
-        unapplied_migrations = set(modules.keys()) - applied_migrations
 
-        for name in sorted(unapplied_migrations):
+    applied_migrations = migration_history_model.get_applied_migrations(db_alias, migrations_package)
+    modules = import_submodules(migrations_package)
+    unapplied_migrations = set(modules.keys()) - applied_migrations
+
+    any_applied = False
+    for name in sorted(unapplied_migrations):
+        if int(name[:4]) > up_to:
+            break
+
+        if verbosity > 0:
             print('Applying ClickHouse migration %s for app %s in database %s' % (name, app_label, db_alias))
-            migration = modules[name].Migration()
-            migration.apply(db_alias, database=database)
 
-            migration_history_model.set_migration_applied(db_alias, migrations_package, name)
+        migration = modules[name].Migration()
+        migration.apply(db_alias, database=database)
 
-            if int(name[:4]) >= up_to:
+        migration_history_model.set_migration_applied(db_alias, migrations_package, name)
-                break
+        any_applied = True
+
+    if not any_applied:
+        if verbosity > 1:
+            print('No migrations to apply for app "%s" does not exist' % app_label)
+        return False
+
+    return True
 
 
 @receiver(post_migrate)
@@ -97,7 +118,7 @@ def clickhouse_migrate(sender, **kwargs):
     app_name = kwargs['app_config'].name
 
     for db_alias in config.DATABASES:
-        migrate_app(app_name, db_alias)
+        migrate_app(app_name, db_alias, verbosity=kwargs.get('verbosity', 1))
 
 
 class MigrationHistory(ClickHouseModel):

tests/test_migrations.py

@@ -1,8 +1,13 @@
-from django.test import TestCase, override_settings
+from typing import List, Dict, Any
-from django_clickhouse.migrations import MigrationHistory
+from unittest import mock
 
+from django.conf import settings
+from django.test import TestCase, override_settings
+
+from django_clickhouse.configuration import config
 from django_clickhouse.database import connections
-from django_clickhouse.migrations import migrate_app
+from django_clickhouse.management.commands.clickhouse_migrate import Command
+from django_clickhouse.migrations import MigrationHistory, migrate_app
 from django_clickhouse.routers import DefaultRouter
 from tests.clickhouse_models import ClickHouseTestModel
 
@@ -53,3 +58,102 @@ class MigrateAppTest(TestCase):
     def test_readonly_connections(self):
         migrate_app('tests', 'readonly')
         self.assertFalse(table_exists(connections['readonly'], ClickHouseTestModel))
+
+
+@override_settings(CLICKHOUSE_MIGRATE_WITH_DEFAULT_DB=False)
+@mock.patch('django_clickhouse.management.commands.clickhouse_migrate.migrate_app', return_value=True)
+class MigrateDjangoCommandTest(TestCase):
+    def setUp(self) -> None:
+        self.cmd = Command()
+
+    def test_handle_all(self, migrate_app_mock):
+        self.cmd.handle(verbosity=3, app_label=None, database=None, migration_number=None)
+
+        self.assertEqual(len(config.DATABASES.keys()) * len(settings.INSTALLED_APPS), migrate_app_mock.call_count)
+        for db_alias in config.DATABASES.keys():
+            for app_label in settings.INSTALLED_APPS:
+                migrate_app_mock.assert_any_call(app_label, db_alias, verbosity=3)
+
+    def test_handle_app(self, migrate_app_mock):
+        self.cmd.handle(verbosity=3, app_label='tests', database=None, migration_number=None)
+
+        self.assertEqual(len(config.DATABASES.keys()), migrate_app_mock.call_count)
+        for db_alias in config.DATABASES.keys():
+            migrate_app_mock.assert_any_call('tests', db_alias, verbosity=3)
+
+    def test_handle_database(self, migrate_app_mock):
+        self.cmd.handle(verbosity=3, database='default', app_label=None, migration_number=None)
+
+        self.assertEqual(len(settings.INSTALLED_APPS), migrate_app_mock.call_count)
+        for app_label in settings.INSTALLED_APPS:
+            migrate_app_mock.assert_any_call(app_label, 'default', verbosity=3)
+
+    def test_handle_app_and_database(self, migrate_app_mock):
+        self.cmd.handle(verbosity=3, app_label='tests', database='default', migration_number=None)
+
+        migrate_app_mock.assert_called_with('tests', 'default', verbosity=3)
+
+    def test_handle_migration_number(self, migrate_app_mock):
+        self.cmd.handle(verbosity=3, database='default', app_label='tests', migration_number=1)
+
+        migrate_app_mock.assert_called_with('tests', 'default', up_to=1, verbosity=3)
+
+    def _test_parser_results(self, argv: List[str], expected: Dict[str, Any]) -> None:
+        """
+        Tests if parser process input correctly.
+        Checks only expected parameters, ignores others.
+        :param argv: List of string arguments from command line
+        :param expected: Dictionary of expected results
+        :return: None
+        :raises AssertionError: If expected result is incorrect
+        """
+        parser = self.cmd.create_parser('./manage.py', 'clickhouse_migrate')
+
+        options = parser.parse_args(argv)
+
+        # Copied from django.core.management.base.BaseCommand.run_from_argv('...')
+        cmd_options = vars(options)
+        cmd_options.pop('args', ())
+
+        self.assertDictEqual(expected, {opt: cmd_options[opt] for opt in expected.keys()})
+
+    def test_parser(self, _):
+        with self.subTest('Simple'):
+            self._test_parser_results([], {
+                'app_label': None,
+                'database': None,
+                'migration_number': None,
+                'verbosity': 1
+            })
+
+        with self.subTest('App label'):
+            self._test_parser_results(['tests'], {
+                'app_label': 'tests',
+                'database': None,
+                'migration_number': None,
+                'verbosity': 1
+            })
+
+        with self.subTest('App label and migration number'):
+            self._test_parser_results(['tests', '123'], {
+                'app_label': 'tests',
+                'database': None,
+                'migration_number': 123,
+                'verbosity': 1
+            })
+
+        with self.subTest('Database'):
+            self._test_parser_results(['--database', 'default'], {
+                'app_label': None,
+                'database': 'default',
+                'migration_number': None,
+                'verbosity': 1
+            })
+
+        with self.subTest('Verbosity'):
+            self._test_parser_results(['--verbosity', '2'], {
+                'app_label': None,
+                'database': None,
+                'migration_number': None,
+                'verbosity': 2
+            })