From 52e9d30c5cd46505d3ace92731b0fd5f93b1ac04 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 10 Jul 2016 16:40:47 +0300 Subject: [PATCH 1/2] stream only SELECTs, to prevent "connection reset by peer" errors --- src/infi/clickhouse_orm/database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 1761469..342c870 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -58,7 +58,7 @@ class Database(object): def select(self, query, model_class=None, settings=None): query += ' FORMAT TabSeparatedWithNamesAndTypes' - r = self._send(query, settings) + r = self._send(query, settings, True) lines = r.iter_lines() field_names = parse_tsv(next(lines)) field_types = parse_tsv(next(lines)) @@ -103,9 +103,9 @@ class Database(object): query = "SELECT module_name from `%s`.`%s` WHERE package_name = '%s'" % (self.db_name, MigrationHistory.table_name(), migrations_package_name) return set(obj.module_name for obj in self.select(query)) - def _send(self, data, settings=None): + def _send(self, data, settings=None, stream=False): params = self._build_params(settings) - r = requests.post(self.db_url, params=params, data=data, stream=True) + r = requests.post(self.db_url, params=params, data=data, stream=stream) if r.status_code != 200: raise DatabaseException(r.text) return r From 5906d90b1fcdb24c85b2cdb35b298665c1f0e273 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Mon, 11 Jul 2016 13:04:48 +0300 Subject: [PATCH 2/2] migrations support - documentation --- MIGRATIONS.rst | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 9 +++++++ 2 files changed, 74 insertions(+) create mode 100644 MIGRATIONS.rst diff --git a/MIGRATIONS.rst b/MIGRATIONS.rst new file mode 100644 index 0000000..0f0d5e2 --- /dev/null +++ b/MIGRATIONS.rst @@ -0,0 +1,65 @@ +Migrations +========== + +Over time, the ORM models in your application may change. Migrations provide a way to modify the database +tables according to the changes in your models, without writing raw SQL. + +The migrations that were applied to the database are recorded in the ``infi_clickhouse_orm_migrations`` table, +so migrating the database will only apply any missing migrations. + +Writing Migrations +------------------ + +To write migrations, create a Python package. Then create a python file for the initial migration. The migration +files must begin with a four-digit number, and will be applied in sequence. For example:: + + analytics + | + +-- analytics_migrations + | + +-- __init__.py + | + +-- 0001_initial.py + | + +-- 0002_add_user_agents_table.py + +Each migration file is expected to contain a list of ``operations``, for example:: + + from infi.clickhouse_orm import migrations + from analytics import models + + operations = [ + migrations.CreateTable(models.Visits), + migrations.CreateTable(models.Visitors) + ] + +The following operations are supported: + +**CreateTable** + +A migration operation that creates a table for a given model class. + +**DropTable** + +A migration operation that drops the table of a given model class. + +**AlterTable** + +A migration operation that compares the table of a given model class to +the model's fields, and alters the table to match the model. The operation can: + +- add new columns +- drop obsolete columns +- modify column types + +Default values are not altered by this operation. + +Running Migrations +------------------ + +To migrate a database, create a ``Database`` instance and call its ``migrate`` method with the package +name containing your migrations:: + + Database('analytics_db').migrate('analytics.analytics_migrations') + +Note that you may have more than one migrations package. \ No newline at end of file diff --git a/README.rst b/README.rst index 7070323..e70e585 100644 --- a/README.rst +++ b/README.rst @@ -142,6 +142,15 @@ You can optionally pass conditions to the query:: Note that ``order_by`` must be chosen so that the ordering is unique, otherwise there might be inconsistencies in the pagination (such as an instance that appears on two different pages). +Schema Migrations +----------------- + +Over time, your models may change and the database will have to be modified accordingly. +Migrations allow you to describe these changes succinctly using Python, and to apply them +to the database. A migrations table automatically keeps track of which migrations were already applied. + +For details please refer to the MIGRATIONS.rst document. + Field Types -----------