From 1ff82a57e157a6aebd59c30fc5a65bcedf44ab72 Mon Sep 17 00:00:00 2001 From: Marsel Date: Sun, 14 May 2017 23:11:58 +0300 Subject: [PATCH 1/5] Fix "NameError: name 'unicode' is not defined" in python3 --- tests/test_readonly.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_readonly.py b/tests/test_readonly.py index 371fdcb..a192b62 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -2,6 +2,7 @@ import unittest +import six from infi.clickhouse_orm.database import Database, DatabaseException from infi.clickhouse_orm.models import Model from infi.clickhouse_orm.fields import * @@ -25,7 +26,7 @@ class ReadonlyTestCase(TestCaseWithData): with self.assertRaises(DatabaseException): self.database.drop_database() except DatabaseException as e: - if 'Unknown user' in unicode(e): + if 'Unknown user' in six.text_type(e): raise unittest.SkipTest('Database user "%s" is not defined' % username) else: raise @@ -56,4 +57,3 @@ class ReadOnlyModel(Model): readonly = True name = StringField() - From e6dba1f89f2bc4451bb53b11c271dd99a9f37026 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Mon, 15 May 2017 08:35:29 +0300 Subject: [PATCH 2/5] TRIVIAL --- scripts/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index aaf2f27..b666b6c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -35,10 +35,9 @@ Usage: generate_all ------------ Does everything: - - - Generates the class reference using generate_ref - - Generates the table of contents using generate_toc - - Converts to HTML for visual inspection using docs2html +- Generates the class reference using generate_ref +- Generates the table of contents using generate_toc +- Converts to HTML for visual inspection using docs2html Usage: From c388f543d22bd015e12c035971bcf42453340f42 Mon Sep 17 00:00:00 2001 From: Marsel Date: Mon, 15 May 2017 13:11:20 +0300 Subject: [PATCH 3/5] ipython<6: closes #32 --- buildout.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildout.cfg b/buildout.cfg index 0078704..a8c2fec 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -43,7 +43,7 @@ output = ${project:version_file} dependent-scripts = true recipe = infi.recipe.console_scripts eggs = ${project:name} - ipython + ipython<6 nose coverage enum34 From fcb8196d3db5bc9eb5ada835acdddeb0c4b31774 Mon Sep 17 00:00:00 2001 From: Ivan Ladelshchikov Date: Tue, 6 Jun 2017 20:00:15 +0500 Subject: [PATCH 4/5] fix unicode params for Py2 --- src/infi/clickhouse_orm/fields.py | 4 ++-- src/infi/clickhouse_orm/system_models.py | 4 +++- tests/test_alias_fields.py | 2 +- tests/test_materialized_fields.py | 2 +- tests/test_system_models.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index a136601..1d2d59a 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -18,9 +18,9 @@ class Field(object): def __init__(self, default=None, alias=None, materialized=None): assert (None, None) in {(default, alias), (alias, materialized), (default, materialized)}, \ "Only one of default, alias and materialized parameters can be given" - assert alias is None or isinstance(alias, str) and alias != "",\ + assert alias is None or isinstance(alias, string_types) and alias != "",\ "Alias field must be string field name, if given" - assert materialized is None or isinstance(materialized, str) and alias != "",\ + assert materialized is None or isinstance(materialized, string_types) and alias != "",\ "Materialized field must be string, if given" self.creation_counter = Field.creation_counter diff --git a/src/infi/clickhouse_orm/system_models.py b/src/infi/clickhouse_orm/system_models.py index 8c66550..c151302 100644 --- a/src/infi/clickhouse_orm/system_models.py +++ b/src/infi/clickhouse_orm/system_models.py @@ -2,6 +2,8 @@ This file contains system readonly models that can be got from database https://clickhouse.yandex/reference_en.html#System tables """ +from six import string_types + from .database import Database from .fields import * from .models import Model @@ -115,7 +117,7 @@ class SystemPart(Model): :return: A list of SystemPart objects """ assert isinstance(database, Database), "database must be database.Database class instance" - assert isinstance(conditions, str), "conditions must be a string" + assert isinstance(conditions, string_types), "conditions must be a string" if conditions: conditions += " AND" field_names = ','.join([f[0] for f in cls._fields]) diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index af7bbc8..e8d896f 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -60,7 +60,7 @@ class ModelWithAliasFields(Model): date_field = DateField() str_field = StringField() - alias_str = StringField(alias='str_field') + alias_str = StringField(alias=u'str_field') alias_int = Int32Field(alias='int_field') alias_date = DateField(alias='date_field') diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py index 3151dc3..f877116 100644 --- a/tests/test_materialized_fields.py +++ b/tests/test_materialized_fields.py @@ -62,7 +62,7 @@ class ModelWithMaterializedFields(Model): mat_str = StringField(materialized='lower(str_field)') mat_int = Int32Field(materialized='abs(int_field)') - mat_date = DateField(materialized='toDate(date_time_field)') + mat_date = DateField(materialized=u'toDate(date_time_field)') engine = MergeTree('mat_date', ('mat_date',)) diff --git a/tests/test_system_models.py b/tests/test_system_models.py index 544713b..3e48b0c 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -40,7 +40,7 @@ class SystemPartTest(unittest.TestCase): def test_get_conditions(self): parts = list(SystemPart.get(self.database, conditions="table='testtable'")) self.assertEqual(len(parts), 1) - parts = list(SystemPart.get(self.database, conditions="table='othertable'")) + parts = list(SystemPart.get(self.database, conditions=u"table='othertable'")) self.assertEqual(len(parts), 0) def test_attach_detach(self): From d02d6b14eb4e3237c38d7a14fcb48293d9ce0876 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 15 Jun 2017 11:19:56 +0300 Subject: [PATCH 5/5] - Added `ne` and `not_in` queryset operators - Querysets no longer have a default order when `order_by` is not called - Added `autocreate` flag to database initializer - Fix for SELECT FROM JOIN (#37) --- CHANGELOG.md | 9 ++++++ docs/class_reference.md | 3 +- docs/models_and_databases.md | 8 ++---- docs/querysets.md | 6 +++- scripts/docs2html.sh | 3 ++ src/infi/clickhouse_orm/database.py | 11 ++++++-- src/infi/clickhouse_orm/models.py | 4 +-- src/infi/clickhouse_orm/query.py | 22 +++++++++++++-- tests/test_database.py | 11 +++++++- tests/test_join.py | 44 +++++++++++++++++++++++++++++ tests/test_querysets.py | 8 ++++++ 11 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 tests/test_join.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8da6ee6..d48f361 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ Change Log ========== +Unreleased +---------- +- Added `ne` and `not_in` queryset operators +- Querysets no longer have a default order unless `order_by` is called +- Added `autocreate` flag to database initializer +- Fix some Python 2/3 incompatibilities (TvoroG, tsionyx) +- To work around a JOIN bug in ClickHouse, `$table` now inserts only the table name, + and the database name is sent in the query params instead + v0.9.0 ------ - Major new feature: building model queries using QuerySets diff --git a/docs/class_reference.md b/docs/class_reference.md index 0f4f57f..0c53eb3 100644 --- a/docs/class_reference.md +++ b/docs/class_reference.md @@ -10,7 +10,7 @@ infi.clickhouse_orm.database Database instances connect to a specific ClickHouse database for running queries, inserting data and other operations. -#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False) +#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False, autocreate=True) Initializes a database instance. Unless it's readonly, the database will be @@ -21,6 +21,7 @@ created on the ClickHouse server if it does not already exist. - `username`: optional connection credentials. - `password`: optional connection credentials. - `readonly`: use a read-only connection. +- `autocreate`: automatically create the database if does not exist (unless in readonly mode). #### count(model_class, conditions=None) diff --git a/docs/models_and_databases.md b/docs/models_and_databases.md index 2879c32..ffeb54e 100644 --- a/docs/models_and_databases.md +++ b/docs/models_and_databases.md @@ -117,7 +117,7 @@ This is a very convenient feature that saves you the need to define a model for SQL Placeholders ---------------- -There are a couple of special placeholders that you can use inside the SQL to make it easier to write: `$db` and `$table`. The first one is replaced by the database name, and the second is replaced by the database name plus table name (but is available only when the model is specified). +There are a couple of special placeholders that you can use inside the SQL to make it easier to write: `$db` and `$table`. The first one is replaced by the database name, and the second is replaced by the table name (but is available only when the model is specified). So instead of this: @@ -125,11 +125,9 @@ So instead of this: you can use: - db.select("SELECT * FROM $db.person", model_class=Person) + db.select("SELECT * FROM $db.$table", model_class=Person) -or even: - - db.select("SELECT * FROM $table", model_class=Person) +Note: normally it is not necessary to specify the database name, since it's already sent in the query parameters to ClickHouse. It is enough to specify the table name. Counting -------- diff --git a/docs/querysets.md b/docs/querysets.md index fd7e253..2990a77 100644 --- a/docs/querysets.md +++ b/docs/querysets.md @@ -31,11 +31,13 @@ There are different operators that can be used, by passing `__ value` | | | `gte` | `field >= value` | | | `lt` | `field < value` | | | `lte` | `field <= value` | | | `in` | `field IN (values)` | See below | +| `not_in` | `field NOT IN (values)` | See below | | `contains` | `field LIKE '%value%'` | For string fields only | | `startswith` | `field LIKE 'value%'` | For string fields only | | `endswith` | `field LIKE '%value'` | For string fields only | @@ -46,7 +48,7 @@ There are different operators that can be used, by passing `__')) register_operator('gte', SimpleOperator('>=')) register_operator('lt', SimpleOperator('<')) register_operator('lte', SimpleOperator('<=')) register_operator('in', InOperator()) +register_operator('not_in', NotOperator(InOperator())) register_operator('contains', LikeOperator('%{}%')) register_operator('startswith', LikeOperator('{}%')) register_operator('endswith', LikeOperator('%{}')) @@ -165,7 +180,7 @@ class QuerySet(object): """ self._model_cls = model_cls self._database = database - self._order_by = [f[0] for f in model_cls._fields] + self._order_by = [] self._q = [] self._fields = [] @@ -194,8 +209,9 @@ class QuerySet(object): fields = '*' if self._fields: fields = ', '.join('`%s`' % field for field in self._fields) - params = (fields, self._database.db_name, self._model_cls.table_name(), self.conditions_as_sql(), self.order_by_as_sql()) - return u'SELECT %s\nFROM `%s`.`%s`\nWHERE %s\nORDER BY %s' % params + ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else '' + params = (fields, self._database.db_name, self._model_cls.table_name(), self.conditions_as_sql(), ordering) + return u'SELECT %s\nFROM `%s`.`%s`\nWHERE %s%s' % params def order_by_as_sql(self): """ diff --git a/tests/test_database.py b/tests/test_database.py index 8f94230..fddf383 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -128,4 +128,13 @@ class DatabaseTestCase(TestCaseWithData): def test_invalid_user(self): with self.assertRaises(DatabaseException): - Database(self.database.db_name, username='default', password='wrong') \ No newline at end of file + Database(self.database.db_name, username='default', password='wrong') + + def test_nonexisting_db(self): + db = Database('db_not_here', autocreate=False) + with self.assertRaises(DatabaseException): + db.create_table(Person) + + def test_preexisting_db(self): + db = Database(self.database.db_name, autocreate=False) + db.count(Person) diff --git a/tests/test_join.py b/tests/test_join.py new file mode 100644 index 0000000..7f3e2df --- /dev/null +++ b/tests/test_join.py @@ -0,0 +1,44 @@ +from __future__ import unicode_literals, print_function + +import unittest +import json + +from infi.clickhouse_orm import database, engines, fields, models + + +class JoinTest(unittest.TestCase): + + def setUp(self): + self.database = database.Database('test-db') + self.database.create_table(Foo) + self.database.create_table(Bar) + self.database.insert([Foo(id=i) for i in range(3)]) + self.database.insert([Bar(id=i, b=i * i) for i in range(3)]) + + def print_res(self, query): + print(query) + print(json.dumps([row.to_dict() for row in self.database.select(query)])) + + def test_without_db_name(self): + self.print_res("SELECT * FROM {}".format(Foo.table_name())) + self.print_res("SELECT * FROM {}".format(Bar.table_name())) + self.print_res("SELECT b FROM {} ALL LEFT JOIN {} USING id".format(Foo.table_name(), Bar.table_name())) + + @unittest.skip('ClickHouse issue - https://github.com/yandex/ClickHouse/issues/635') + def test_with_db_name(self): + self.print_res("SELECT * FROM $db.{}".format(Foo.table_name())) + self.print_res("SELECT * FROM $db.{}".format(Bar.table_name())) + self.print_res("SELECT b FROM $db.{} ALL LEFT JOIN $db.{} USING id".format(Foo.table_name(), Bar.table_name())) + + def test_with_subquery(self): + self.print_res("SELECT b FROM {} ALL LEFT JOIN (SELECT * from {}) USING id".format(Foo.table_name(), Bar.table_name())) + self.print_res("SELECT b FROM $db.{} ALL LEFT JOIN (SELECT * from $db.{}) USING id".format(Foo.table_name(), Bar.table_name())) + + +class Foo(models.Model): + id = fields.UInt8Field() + engine = engines.Memory() + + +class Bar(Foo): + b = fields.UInt8Field() diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 27e6c5c..50fd1ba 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -46,6 +46,7 @@ class QuerySetTestCase(TestCaseWithData): self._test_qs(qs.filter(first_name__in=('Connor', 'Courtney')), 3) # in tuple self._test_qs(qs.filter(first_name__in=['Connor', 'Courtney']), 3) # in list self._test_qs(qs.filter(first_name__in="'Connor', 'Courtney'"), 3) # in string + self._test_qs(qs.filter(first_name__not_in="'Connor', 'Courtney'"), 97) self._test_qs(qs.filter(first_name__contains='sh'), 3) # case sensitive self._test_qs(qs.filter(first_name__icontains='sh'), 6) # case insensitive self._test_qs(qs.filter(first_name__startswith='le'), 0) # case sensitive @@ -74,6 +75,8 @@ class QuerySetTestCase(TestCaseWithData): def test_filter_date_field(self): qs = Person.objects_in(self.database) self._test_qs(qs.filter(birthday='1970-12-02'), 1) + self._test_qs(qs.filter(birthday__eq='1970-12-02'), 1) + self._test_qs(qs.filter(birthday__ne='1970-12-02'), 99) self._test_qs(qs.filter(birthday=date(1970, 12, 2)), 1) self._test_qs(qs.filter(birthday__lte=date(1970, 12, 2)), 3) @@ -87,6 +90,8 @@ class QuerySetTestCase(TestCaseWithData): def test_order_by(self): qs = Person.objects_in(self.database) + self.assertFalse('ORDER BY' in qs.as_sql()) + self.assertFalse(qs.order_by_as_sql()) person = list(qs.order_by('first_name', 'last_name'))[0] self.assertEquals(person.first_name, 'Abdul') person = list(qs.order_by('-first_name', '-last_name'))[0] @@ -100,6 +105,7 @@ class QuerySetTestCase(TestCaseWithData): qs = Person.objects_in(self.database) self._test_qs(qs.filter(height__in='SELECT max(height) FROM $table'), 2) self._test_qs(qs.filter(first_name__in=qs.only('last_name')), 2) + self._test_qs(qs.filter(first_name__not_in=qs.only('last_name')), 98) def _insert_sample_model(self): self.database.create_table(SampleModel) @@ -125,6 +131,8 @@ class QuerySetTestCase(TestCaseWithData): self._insert_sample_model() qs = SampleModel.objects_in(self.database) self._test_qs(qs.filter(num=1), 1) + self._test_qs(qs.filter(num__eq=1), 1) + self._test_qs(qs.filter(num__ne=1), 3) self._test_qs(qs.filter(num__gt=1), 3) self._test_qs(qs.filter(num__gte=1), 4) self._test_qs(qs.filter(num__in=(1, 2, 3)), 3)