Add AlterTableWithBuffer migration operation

This commit is contained in:
Itai Shirav 2017-09-10 15:46:55 +03:00
parent 430872b958
commit 7bbcae574a
6 changed files with 102 additions and 4 deletions

View File

@ -1,6 +1,10 @@
Change Log
==========
Unreleased
----------
- Add `AlterTableWithBuffer` migration operation
v0.9.6
------
- Fix python3 compatibility (TvoroG)

View File

@ -34,11 +34,13 @@ The following operations are supported:
**CreateTable**
A migration operation that creates a table for a given model class.
A migration operation that creates a table for a given model class. If the table already exists, the operation does nothing.
In case the model class is a `BufferModel`, the operation first creates the underlying on-disk table, and then creates the buffer table.
**DropTable**
A migration operation that drops the table of a given model class.
A migration operation that drops the table of a given model class. If the table does not exist, the operation does nothing.
**AlterTable**
@ -50,6 +52,13 @@ A migration operation that compares the table of a given model class to the mode
Default values are not altered by this operation.
**AlterTableWithBuffer**
A compound migration operation for altering a buffer table and its underlying on-disk table. The buffer table is dropped, the on-disk table is altered, and then the buffer table is re-created. This is the procedure recommended in the ClickHouse documentation for handling scenarios in which the underlying table needs to be modified.
Applying this migration operation to a regular table has the same effect as an `AlterTable` operation.
Running Migrations
------------------

View File

@ -82,6 +82,25 @@ class AlterTable(Operation):
self._alter_table(database, 'MODIFY COLUMN %s %s' % model_field)
class AlterTableWithBuffer(Operation):
'''
A migration operation for altering a buffer table and its underlying on-disk table.
The buffer table is dropped, the on-disk table is altered, and then the buffer table
is re-created.
'''
def __init__(self, model_class):
self.model_class = model_class
def apply(self, database):
if issubclass(self.model_class, BufferModel):
DropTable(self.model_class).apply(database)
AlterTable(self.model_class.engine.main_model).apply(database)
CreateTable(self.model_class).apply(database)
else:
AlterTable(self.model_class).apply(database)
class DropTable(Operation):
'''
A migration operation that drops the table of a given model class.
@ -91,7 +110,7 @@ class DropTable(Operation):
self.model_class = model_class
def apply(self, database):
logger.info(' Drop table %s', self.model_class.__name__)
logger.info(' Drop table %s', self.model_class.table_name())
database.drop_table(self.model_class)

View File

@ -0,0 +1,6 @@
from infi.clickhouse_orm import migrations
from ..test_migrations import *
operations = [
migrations.CreateTable(Model4Buffer)
]

View File

@ -0,0 +1,6 @@
from infi.clickhouse_orm import migrations
from ..test_migrations import *
operations = [
migrations.AlterTableWithBuffer(Model4Buffer_changed)
]

View File

@ -2,7 +2,7 @@ from __future__ import unicode_literals
import unittest
from infi.clickhouse_orm.database import Database
from infi.clickhouse_orm.models import Model
from infi.clickhouse_orm.models import Model, BufferModel
from infi.clickhouse_orm.fields import *
from infi.clickhouse_orm.engines import *
from infi.clickhouse_orm.migrations import MigrationHistory
@ -61,6 +61,7 @@ class MigrationsTestCase(unittest.TestCase):
self.assertTrue(self.tableExists(EnumModel1))
self.assertEquals(self.getTableFields(EnumModel2),
[('date', 'Date'), ('f1', "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")])
# Materialized fields and alias fields
self.database.migrate('tests.sample_migrations', 8)
self.assertTrue(self.tableExists(MaterializedModel))
self.assertEquals(self.getTableFields(MaterializedModel),
@ -69,6 +70,15 @@ class MigrationsTestCase(unittest.TestCase):
self.assertTrue(self.tableExists(AliasModel))
self.assertEquals(self.getTableFields(AliasModel),
[('date', 'Date'), ('date_alias', "Date")])
# Buffer models creation and alteration
self.database.migrate('tests.sample_migrations', 10)
self.assertTrue(self.tableExists(Model4))
self.assertTrue(self.tableExists(Model4Buffer))
self.assertEquals(self.getTableFields(Model4), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')])
self.assertEquals(self.getTableFields(Model4Buffer), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')])
self.database.migrate('tests.sample_migrations', 11)
self.assertEquals(self.getTableFields(Model4), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')])
self.assertEquals(self.getTableFields(Model4Buffer), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')])
# Several different models with the same table name, to simulate a table that changes over time
@ -159,3 +169,47 @@ class AliasModel(Model):
@classmethod
def table_name(cls):
return 'alias_date'
class Model4(Model):
date = DateField()
f1 = Int32Field()
f2 = StringField()
engine = MergeTree('date', ('date',))
@classmethod
def table_name(cls):
return 'model4'
class Model4Buffer(BufferModel, Model4):
engine = Buffer(Model4)
@classmethod
def table_name(cls):
return 'model4buffer'
class Model4_changed(Model):
date = DateField()
f3 = DateTimeField()
f2 = StringField()
engine = MergeTree('date', ('date',))
@classmethod
def table_name(cls):
return 'model4'
class Model4Buffer_changed(BufferModel, Model4_changed):
engine = Buffer(Model4_changed)
@classmethod
def table_name(cls):
return 'model4buffer'