Merge branch 'carrotquest-raw_migrations' into develop

CHANGELOG.md

@@ -1,6 +1,11 @@
 Change Log
 ==========
 
+Unreleased
+----------
+- Bug fix: add field names list explicitly to Database.insert method (anci)
+- Added RunPython and RunSQL migrations (M1hacka)
+
 v0.9.7
 ------
 - Add `distinct` method to querysets

docs/schema_migrations.md

@@ -32,16 +32,19 @@ Each migration file is expected to contain a list of `operations`, for example:
 
 The following operations are supported:
 
+
 **CreateTable**
 
 A migration operation that creates a table for a given model class. If the table already exists, the operation does nothing.
 
 In case the model class is a `BufferModel`, the operation first creates the underlying on-disk table, and then creates the buffer table.
 
+
 **DropTable**
 
 A migration operation that drops the table of a given model class. If the table does not exist, the operation does nothing.
 
+
 **AlterTable**
 
 A migration operation that compares the table of a given model class to the model’s fields, and alters the table to match the model. The operation can:
@@ -52,6 +55,7 @@ A migration operation that compares the table of a given model class to the mode
 
 Default values are not altered by this operation.
 
+
 **AlterTableWithBuffer**
 
 A compound migration operation for altering a buffer table and its underlying on-disk table. The buffer table is dropped, the on-disk table is altered, and then the buffer table is re-created. This is the procedure recommended in the ClickHouse documentation for handling scenarios in which the underlying table needs to be modified.
@@ -59,6 +63,35 @@ A compound migration operation for altering a buffer table and its underlying on
 Applying this migration operation to a regular table has the same effect as an `AlterTable` operation.
 
 
+**RunPython**
+
+A migration operation that runs a Python function. The function receives the `Database` instance to operate on.
+
+    def forward(database):
+        database.insert([
+             TestModel(field=1)
+        ])
+
+    operations = [
+        migrations.RunPython(forward),
+    ]
+
+
+**RunSQL**
+
+A migration operation that runs raw SQL queries. It expects a string containing an SQL query, or an array of SQL-query strings.
+
+Example:
+
+    operations = [
+        RunSQL('INSERT INTO `test_table` (field) VALUES (1)'),
+        RunSQL([
+          'INSERT INTO `test_table` (field) VALUES (2)',
+          'INSERT INTO `test_table` (field) VALUES (3)'
+         ])
+    ]
+
+
 Running Migrations
 ------------------
 

src/infi/clickhouse_orm/migrations.py

@@ -1,3 +1,5 @@
+import six
+
 from .models import Model, BufferModel
 from .fields import DateField, StringField
 from .engines import MergeTree
@@ -114,6 +116,37 @@ class DropTable(Operation):
         database.drop_table(self.model_class)
 
 
+class RunPython(Operation):
+    '''
+    A migration operation that executes given python function on database
+    '''
+    def __init__(self, func):
+        assert callable(func), "'func' parameter must be function"
+        self._func = func
+
+    def apply(self, database):
+        logger.info('    Executing python operation %s', self._func.__name__)
+        self._func(database)
+
+
+class RunSQL(Operation):
+    '''
+    A migration operation that executes given SQL on database
+    '''
+
+    def __init__(self, sql):
+        if isinstance(sql, six.string_types):
+            sql = [sql]
+
+        assert isinstance(sql, list), "'sql' parameter must be string or list of strings"
+        self._sql = sql
+
+    def apply(self, database):
+        logger.info('    Executing raw SQL operations')
+        for item in self._sql:
+            database.raw(item)
+
+
 class MigrationHistory(Model):
     '''
     A model for storing which migrations were already applied to the containing database.

tests/sample_migrations/0012.py

@@ -0,0 +1,9 @@
+from infi.clickhouse_orm import migrations
+
+operations = [
+    migrations.RunSQL("INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-01', 1, 1, 'test') "),
+    migrations.RunSQL([
+        "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-02', 2, 2, 'test2') ",
+        "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-03', 3, 3, 'test3') ",
+    ])
+]

tests/sample_migrations/0013.py

@@ -0,0 +1,15 @@
+import datetime
+
+from infi.clickhouse_orm import migrations
+from test_migrations import Model3
+
+
+def forward(database):
+    database.insert([
+        Model3(date=datetime.date(2016, 1, 4), f1=4, f3=1, f4='test4')
+    ])
+
+
+operations = [
+    migrations.RunPython(forward)
+]

tests/test_migrations.py

@@ -80,6 +80,16 @@ class MigrationsTestCase(unittest.TestCase):
         self.assertEquals(self.getTableFields(Model4), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')])
         self.assertEquals(self.getTableFields(Model4Buffer), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')])
 
+        self.database.migrate('tests.sample_migrations', 12)
+        self.assertEqual(self.database.count(Model3), 3)
+        data = [item.f1 for item in self.database.select('SELECT f1 FROM $table ORDER BY f1', model_class=Model3)]
+        self.assertListEqual(data, [1, 2, 3])
+
+        self.database.migrate('tests.sample_migrations', 13)
+        self.assertEqual(self.database.count(Model3), 4)
+        data = [item.f1 for item in self.database.select('SELECT f1 FROM $table ORDER BY f1', model_class=Model3)]
+        self.assertListEqual(data, [1, 2, 3, 4])
+
 
 # Several different models with the same table name, to simulate a table that changes over time