diff --git a/CHANGELOG.md b/CHANGELOG.md index d302dc4..1bfa0fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ Change Log ========== +Unreleased +---------- +- Add `AlterTableWithBuffer` migration operation + v0.9.6 ------ - Fix python3 compatibility (TvoroG) diff --git a/docs/schema_migrations.md b/docs/schema_migrations.md index e4647bf..ce56b04 100644 --- a/docs/schema_migrations.md +++ b/docs/schema_migrations.md @@ -34,11 +34,13 @@ The following operations are supported: **CreateTable** -A migration operation that creates a table for a given model class. +A migration operation that creates a table for a given model class. If the table already exists, the operation does nothing. + +In case the model class is a `BufferModel`, the operation first creates the underlying on-disk table, and then creates the buffer table. **DropTable** -A migration operation that drops the table of a given model class. +A migration operation that drops the table of a given model class. If the table does not exist, the operation does nothing. **AlterTable** @@ -50,6 +52,13 @@ A migration operation that compares the table of a given model class to the mode Default values are not altered by this operation. +**AlterTableWithBuffer** + +A compound migration operation for altering a buffer table and its underlying on-disk table. The buffer table is dropped, the on-disk table is altered, and then the buffer table is re-created. This is the procedure recommended in the ClickHouse documentation for handling scenarios in which the underlying table needs to be modified. + +Applying this migration operation to a regular table has the same effect as an `AlterTable` operation. + + Running Migrations ------------------ diff --git a/src/infi/clickhouse_orm/migrations.py b/src/infi/clickhouse_orm/migrations.py index 1a82a5b..a7843a7 100644 --- a/src/infi/clickhouse_orm/migrations.py +++ b/src/infi/clickhouse_orm/migrations.py @@ -82,6 +82,25 @@ class AlterTable(Operation): self._alter_table(database, 'MODIFY COLUMN %s %s' % model_field) +class AlterTableWithBuffer(Operation): + ''' + A migration operation for altering a buffer table and its underlying on-disk table. + The buffer table is dropped, the on-disk table is altered, and then the buffer table + is re-created. + ''' + + def __init__(self, model_class): + self.model_class = model_class + + def apply(self, database): + if issubclass(self.model_class, BufferModel): + DropTable(self.model_class).apply(database) + AlterTable(self.model_class.engine.main_model).apply(database) + CreateTable(self.model_class).apply(database) + else: + AlterTable(self.model_class).apply(database) + + class DropTable(Operation): ''' A migration operation that drops the table of a given model class. @@ -91,7 +110,7 @@ class DropTable(Operation): self.model_class = model_class def apply(self, database): - logger.info(' Drop table %s', self.model_class.__name__) + logger.info(' Drop table %s', self.model_class.table_name()) database.drop_table(self.model_class) diff --git a/tests/sample_migrations/0010.py b/tests/sample_migrations/0010.py new file mode 100644 index 0000000..3892583 --- /dev/null +++ b/tests/sample_migrations/0010.py @@ -0,0 +1,6 @@ +from infi.clickhouse_orm import migrations +from ..test_migrations import * + +operations = [ + migrations.CreateTable(Model4Buffer) +] diff --git a/tests/sample_migrations/0011.py b/tests/sample_migrations/0011.py new file mode 100644 index 0000000..dd9d09e --- /dev/null +++ b/tests/sample_migrations/0011.py @@ -0,0 +1,6 @@ +from infi.clickhouse_orm import migrations +from ..test_migrations import * + +operations = [ + migrations.AlterTableWithBuffer(Model4Buffer_changed) +] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 3478f9f..7e31c84 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.models import Model, BufferModel from infi.clickhouse_orm.fields import * from infi.clickhouse_orm.engines import * from infi.clickhouse_orm.migrations import MigrationHistory @@ -61,6 +61,7 @@ class MigrationsTestCase(unittest.TestCase): self.assertTrue(self.tableExists(EnumModel1)) self.assertEquals(self.getTableFields(EnumModel2), [('date', 'Date'), ('f1', "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")]) + # Materialized fields and alias fields self.database.migrate('tests.sample_migrations', 8) self.assertTrue(self.tableExists(MaterializedModel)) self.assertEquals(self.getTableFields(MaterializedModel), @@ -69,6 +70,15 @@ class MigrationsTestCase(unittest.TestCase): self.assertTrue(self.tableExists(AliasModel)) self.assertEquals(self.getTableFields(AliasModel), [('date', 'Date'), ('date_alias', "Date")]) + # Buffer models creation and alteration + self.database.migrate('tests.sample_migrations', 10) + self.assertTrue(self.tableExists(Model4)) + self.assertTrue(self.tableExists(Model4Buffer)) + self.assertEquals(self.getTableFields(Model4), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')]) + self.assertEquals(self.getTableFields(Model4Buffer), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')]) + self.database.migrate('tests.sample_migrations', 11) + self.assertEquals(self.getTableFields(Model4), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')]) + self.assertEquals(self.getTableFields(Model4Buffer), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')]) # Several different models with the same table name, to simulate a table that changes over time @@ -159,3 +169,47 @@ class AliasModel(Model): @classmethod def table_name(cls): return 'alias_date' + + +class Model4(Model): + + date = DateField() + f1 = Int32Field() + f2 = StringField() + + engine = MergeTree('date', ('date',)) + + @classmethod + def table_name(cls): + return 'model4' + + +class Model4Buffer(BufferModel, Model4): + + engine = Buffer(Model4) + + @classmethod + def table_name(cls): + return 'model4buffer' + + +class Model4_changed(Model): + + date = DateField() + f3 = DateTimeField() + f2 = StringField() + + engine = MergeTree('date', ('date',)) + + @classmethod + def table_name(cls): + return 'model4' + + +class Model4Buffer_changed(BufferModel, Model4_changed): + + engine = Buffer(Model4_changed) + + @classmethod + def table_name(cls): + return 'model4buffer'