diff --git a/CHANGELOG.md b/CHANGELOG.md index c11cd37..d302dc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ Change Log ========== +v0.9.6 +------ +- Fix python3 compatibility (TvoroG) +- Nullable arrays not supported in latest ClickHouse version +- system.parts table no longer includes "replicated" column in latest ClickHouse version + v0.9.5 ------ - Added `QuerySet.paginate()` diff --git a/buildout.cfg b/buildout.cfg index 004a6fd..5ea6da5 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -3,7 +3,7 @@ prefer-final = false newest = false download-cache = .cache develop = . -parts = +parts = [project] name = infi.clickhouse_orm @@ -29,7 +29,7 @@ homepage = https://github.com/Infinidat/infi.clickhouse_orm [isolated-python] recipe = infi.recipe.python -version = v2.7.9.4 +version = v2.7.12.4 [setup.py] recipe = infi.recipe.template.version diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index ae73059..5c816d6 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import requests from collections import namedtuple from .models import ModelBase @@ -24,11 +26,11 @@ class DatabaseException(Exception): class Database(object): ''' - Database instances connect to a specific ClickHouse database for running queries, + Database instances connect to a specific ClickHouse database for running queries, inserting data and other operations. ''' - def __init__(self, db_name, db_url='http://localhost:8123/', + def __init__(self, db_name, db_url='http://localhost:8123/', username=None, password=None, readonly=False, autocreate=True): ''' Initializes a database instance. Unless it's readonly, the database will be @@ -186,7 +188,7 @@ class Database(object): - `conditions`: optional SQL conditions (contents of the WHERE clause). - `settings`: query settings to send as HTTP GET parameters - The result is a namedtuple containing `objects` (list), `number_of_objects`, + The result is a namedtuple containing `objects` (list), `number_of_objects`, `pages_total`, `number` (of the current page), and `page_size`. ''' count = self.count(model_class, conditions) @@ -214,7 +216,7 @@ class Database(object): ''' Executes schema migrations. - - `migrations_package_name` - fully qualified name of the Python package + - `migrations_package_name` - fully qualified name of the Python package containing the migrations. - `up_to` - number of the last migration to apply. ''' diff --git a/src/infi/clickhouse_orm/engines.py b/src/infi/clickhouse_orm/engines.py index b28fedd..caa05c7 100644 --- a/src/infi/clickhouse_orm/engines.py +++ b/src/infi/clickhouse_orm/engines.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals from .utils import comma_join diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index e6e100a..a57314e 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals from six import string_types, text_type, binary_type import datetime import pytz diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index b7f4fa5..8714447 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals from logging import getLogger from six import with_metaclass @@ -18,7 +19,7 @@ class ModelBase(type): ad_hoc_model_cache = {} def __new__(cls, name, bases, attrs): - new_cls = super(ModelBase, cls).__new__(cls, name, bases, attrs) + new_cls = super(ModelBase, cls).__new__(cls, str(name), bases, attrs) # Collect fields from parent classes base_fields = [] for base in bases: @@ -76,7 +77,7 @@ class ModelBase(type): class Model(with_metaclass(ModelBase)): ''' A base class for ORM models. Each model class represent a ClickHouse table. For example: - + class CPUStats(Model): timestamp = DateTimeField() cpu_id = UInt16Field() @@ -123,7 +124,7 @@ class Model(with_metaclass(ModelBase)): def set_database(self, db): ''' - Sets the `Database` that this model instance belongs to. + Sets the `Database` that this model instance belongs to. This is done automatically when the instance is read from the database or written to it. ''' # This can not be imported globally due to circular import @@ -133,7 +134,7 @@ class Model(with_metaclass(ModelBase)): def get_database(self): ''' - Gets the `Database` that this model instance belongs to. + Gets the `Database` that this model instance belongs to. Returns `None` unless the instance was read from the database or written to it. ''' return self._database @@ -214,7 +215,7 @@ class Model(with_metaclass(ModelBase)): def to_dict(self, include_readonly=True, field_names=None): ''' Returns the instance's column values as a dict. - + - `include_readonly`: if false, returns only fields that can be inserted into database. - `field_names`: an iterable of field names to return (optional) ''' @@ -233,7 +234,7 @@ class Model(with_metaclass(ModelBase)): ''' return QuerySet(cls, database) - + class BufferModel(Model): @classmethod diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py index 217fb68..c1c1dd0 100644 --- a/src/infi/clickhouse_orm/query.py +++ b/src/infi/clickhouse_orm/query.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import six import pytz from copy import copy @@ -167,6 +168,7 @@ class Q(object): return q +@six.python_2_unicode_compatible class QuerySet(object): """ A queryset is an object that represents a database query using a specific `Model`. @@ -190,7 +192,6 @@ class QuerySet(object): """ Iterates over the model instances matching this queryset """ - print self.as_sql() return self._database.select(self.as_sql(), self._model_cls) def __bool__(self): @@ -202,7 +203,7 @@ class QuerySet(object): def __nonzero__(self): # Python 2 compatibility return type(self).__bool__(self) - def __unicode__(self): + def __str__(self): return self.as_sql() def __getitem__(self, s): @@ -211,7 +212,7 @@ class QuerySet(object): assert s >= 0, 'negative indexes are not supported' qs = copy(self) qs._limits = (s, 1) - return iter(qs).next() + return six.next(iter(qs)) else: # Slice assert s.step in (None, 1), 'step is not supported in slices' diff --git a/src/infi/clickhouse_orm/system_models.py b/src/infi/clickhouse_orm/system_models.py index b1e1bb7..49247d9 100644 --- a/src/infi/clickhouse_orm/system_models.py +++ b/src/infi/clickhouse_orm/system_models.py @@ -2,6 +2,7 @@ This file contains system readonly models that can be got from database https://clickhouse.yandex/reference_en.html#System tables """ +from __future__ import unicode_literals from six import string_types from .database import Database @@ -25,7 +26,6 @@ class SystemPart(Model): engine = StringField() # Name of the table engine, without parameters. partition = StringField() # Name of the partition, in the format YYYYMM. name = StringField() # Name of the part. - replicated = UInt8Field() # Whether the part belongs to replicated data. # Whether the part is used in a table, or is no longer needed and will be deleted soon. # Inactive parts remain after merging. diff --git a/src/infi/clickhouse_orm/utils.py b/src/infi/clickhouse_orm/utils.py index 5a8a17a..e3eb4bb 100644 --- a/src/infi/clickhouse_orm/utils.py +++ b/src/infi/clickhouse_orm/utils.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals from six import string_types, binary_type, text_type, PY3 import codecs import re @@ -43,7 +44,7 @@ def parse_tsv(line): line = line.decode() if line and line[-1] == '\n': line = line[:-1] - return [unescape(value) for value in line.split('\t')] + return [unescape(value) for value in line.split(str('\t'))] def parse_array(array_string): diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index d50d311..352d3d3 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index e8d896f..1a17c73 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from datetime import date @@ -65,5 +66,3 @@ class ModelWithAliasFields(Model): alias_date = DateField(alias='date_field') engine = MergeTree('date_field', ('date_field',)) - - diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py index ba4e94b..5676f04 100644 --- a/tests/test_array_fields.py +++ b/tests/test_array_fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from datetime import date @@ -18,8 +19,8 @@ class ArrayFieldsTest(unittest.TestCase): def test_insert_and_select(self): instance = ModelWithArrays( - date_field='2016-08-30', - arr_str=['goodbye,', 'cruel', 'world', 'special chars: ,"\\\'` \n\t\\[]'], + date_field='2016-08-30', + arr_str=['goodbye,', 'cruel', 'world', 'special chars: ,"\\\'` \n\t\\[]'], arr_date=['2010-01-01'] ) self.database.insert([instance]) @@ -52,11 +53,11 @@ class ArrayFieldsTest(unittest.TestCase): self.assertEquals(parse_array("[1, 2, 395, -44]"), ["1", "2", "395", "-44"]) self.assertEquals(parse_array("['big','mouse','','!']"), ["big", "mouse", "", "!"]) self.assertEquals(parse_array(unescape("['\\r\\n\\0\\t\\b']")), ["\r\n\0\t\b"]) - for s in ("", - "[", - "]", - "[1, 2", - "3, 4]", + for s in ("", + "[", + "]", + "[1, 2", + "3, 4]", "['aaa', 'aaa]"): with self.assertRaises(ValueError): parse_array(s) @@ -70,4 +71,3 @@ class ModelWithArrays(Model): arr_date = ArrayField(DateField()) engine = MergeTree('date_field', ('date_field',)) - diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 91bf656..744b6cf 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.models import BufferModel @@ -23,7 +23,5 @@ class BufferTestCase(TestCaseWithData): class PersonBuffer(BufferModel, Person): - - engine = Buffer(Person) - + engine = Buffer(Person) diff --git a/tests/test_database.py b/tests/test_database.py index fddf383..2eed732 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database, DatabaseException @@ -99,7 +99,7 @@ class DatabaseTestCase(TestCaseWithData): page_a = self.database.paginate(Person, 'first_name, last_name', -1, page_size) page_b = self.database.paginate(Person, 'first_name, last_name', page_a.pages_total, page_size) self.assertEquals(page_a[1:], page_b[1:]) - self.assertEquals([obj.to_tsv() for obj in page_a.objects], + self.assertEquals([obj.to_tsv() for obj in page_a.objects], [obj.to_tsv() for obj in page_b.objects]) def test_pagination_invalid_page(self): diff --git a/tests/test_engines.py b/tests/test_engines.py index 3639960..ddc3a85 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database, DatabaseException diff --git a/tests/test_enum_fields.py b/tests/test_enum_fields.py index 78df6d3..1364b95 100644 --- a/tests/test_enum_fields.py +++ b/tests/test_enum_fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py index f9490af..05c544d 100644 --- a/tests/test_fixed_string_fields.py +++ b/tests/test_fixed_string_fields.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database @@ -37,7 +37,7 @@ class FixedStringFieldsTest(unittest.TestCase): query = 'SELECT * from $table ORDER BY date_field' results = list(self.database.select(query, FixedStringModel)) self._assert_sample_data(results) - + def test_ad_hoc_model(self): self._insert_sample_data() query = 'SELECT * from $db.fixedstringmodel ORDER BY date_field' diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py index 08bc084..f209995 100644 --- a/tests/test_inheritance.py +++ b/tests/test_inheritance.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest import datetime import pytz @@ -49,4 +50,3 @@ class Model1(ParentModel): class Model2(ParentModel): float_field = Float32Field() - diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py index f877116..fccd722 100644 --- a/tests/test_materialized_fields.py +++ b/tests/test_materialized_fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from datetime import date @@ -65,5 +66,3 @@ class ModelWithMaterializedFields(Model): mat_date = DateField(materialized=u'toDate(date_time_field)') engine = MergeTree('mat_date', ('mat_date',)) - - diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 4541a6b..3478f9f 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.database import Database @@ -54,11 +55,11 @@ class MigrationsTestCase(unittest.TestCase): # Altering enum fields self.database.migrate('tests.sample_migrations', 6) self.assertTrue(self.tableExists(EnumModel1)) - self.assertEquals(self.getTableFields(EnumModel1), + self.assertEquals(self.getTableFields(EnumModel1), [('date', 'Date'), ('f1', "Enum8('dog' = 1, 'cat' = 2, 'cow' = 3)")]) self.database.migrate('tests.sample_migrations', 7) self.assertTrue(self.tableExists(EnumModel1)) - self.assertEquals(self.getTableFields(EnumModel2), + self.assertEquals(self.getTableFields(EnumModel2), [('date', 'Date'), ('f1', "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")]) self.database.migrate('tests.sample_migrations', 8) self.assertTrue(self.tableExists(MaterializedModel)) @@ -157,4 +158,4 @@ class AliasModel(Model): @classmethod def table_name(cls): - return 'alias_date' \ No newline at end of file + return 'alias_date' diff --git a/tests/test_models.py b/tests/test_models.py index b52a2c6..3228a5b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest import datetime import pytz @@ -89,4 +90,3 @@ class SimpleModel(Model): alias_field = Float32Field(alias='float_field') engine = MergeTree('date_field', ('int_field', 'date_field')) - diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py index 7dcab92..c062ca3 100644 --- a/tests/test_nullable_fields.py +++ b/tests/test_nullable_fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest import pytz @@ -75,18 +76,10 @@ class NullableFieldsTest(unittest.TestCase): def _insert_sample_data(self): dt = date(1970, 1, 1) self.database.insert([ - ModelWithNullable(date_field='2016-08-30', - null_str='', null_int=42, null_date=dt, - null_array=None), - ModelWithNullable(date_field='2016-08-30', - null_str='nothing', null_int=None, null_date=None, - null_array=[1, 2, 3]), - ModelWithNullable(date_field='2016-08-31', - null_str=None, null_int=42, null_date=dt, - null_array=[]), - ModelWithNullable(date_field='2016-08-31', - null_str=None, null_int=None, null_date=None, - null_array=[3, 2, 1]) + ModelWithNullable(date_field='2016-08-30', null_str='', null_int=42, null_date=dt), + ModelWithNullable(date_field='2016-08-30', null_str='nothing', null_int=None, null_date=None), + ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=42, null_date=dt), + ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None) ]) def _assert_sample_data(self, results): @@ -105,11 +98,6 @@ class NullableFieldsTest(unittest.TestCase): self.assertIsNone(results[3].null_str) self.assertIsNone(results[3].null_date) - self.assertIsNone(results[0].null_array) - self.assertEquals(results[1].null_array, [1, 2, 3]) - self.assertEquals(results[2].null_array, []) - self.assertEquals(results[3].null_array, [3, 2, 1]) - def test_insert_and_select(self): self._insert_sample_data() query = 'SELECT * from $table ORDER BY date_field' @@ -129,6 +117,5 @@ class ModelWithNullable(Model): null_str = NullableField(StringField(), extra_null_values={''}) null_int = NullableField(Int32Field()) null_date = NullableField(DateField()) - null_array = NullableField(ArrayField(Int32Field())) engine = MergeTree('date_field', ('date_field',)) diff --git a/tests/test_querysets.py b/tests/test_querysets.py index ded1d84..ad834bb 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from __future__ import unicode_literals, print_function import unittest from infi.clickhouse_orm.database import Database @@ -140,7 +140,7 @@ class QuerySetTestCase(TestCaseWithData): def test_slicing(self): db = Database('system') - numbers = range(100) + numbers = list(range(100)) qs = Numbers.objects_in(db) self.assertEquals(qs[0].number, numbers[0]) self.assertEquals(qs[5].number, numbers[5]) @@ -211,7 +211,7 @@ class AggregateTestCase(TestCaseWithData): def test_aggregate_no_grouping(self): qs = Person.objects_in(self.database).aggregate(average_height='avg(height)', count='count()') - print qs.as_sql() + print(qs.as_sql()) self.assertEquals(qs.count(), 1) for row in qs: self.assertAlmostEqual(row.average_height, 1.6923, places=4) @@ -220,14 +220,14 @@ class AggregateTestCase(TestCaseWithData): def test_aggregate_with_filter(self): # When filter comes before aggregate qs = Person.objects_in(self.database).filter(first_name='Warren').aggregate(average_height='avg(height)', count='count()') - print qs.as_sql() + print(qs.as_sql()) self.assertEquals(qs.count(), 1) for row in qs: self.assertAlmostEqual(row.average_height, 1.675, places=4) self.assertEquals(row.count, 2) # When filter comes after aggregate qs = Person.objects_in(self.database).aggregate(average_height='avg(height)', count='count()').filter(first_name='Warren') - print qs.as_sql() + print(qs.as_sql()) self.assertEquals(qs.count(), 1) for row in qs: self.assertAlmostEqual(row.average_height, 1.675, places=4) @@ -235,7 +235,7 @@ class AggregateTestCase(TestCaseWithData): def test_aggregate_with_implicit_grouping(self): qs = Person.objects_in(self.database).aggregate('first_name', average_height='avg(height)', count='count()') - print qs.as_sql() + print(qs.as_sql()) self.assertEquals(qs.count(), 94) total = 0 for row in qs: @@ -246,7 +246,7 @@ class AggregateTestCase(TestCaseWithData): def test_aggregate_with_explicit_grouping(self): qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') - print qs.as_sql() + print(qs.as_sql()) self.assertEquals(qs.count(), 7) total = 0 for row in qs: @@ -256,7 +256,7 @@ class AggregateTestCase(TestCaseWithData): def test_aggregate_with_order_by(self): qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') days = [row.weekday for row in qs.order_by('weekday')] - self.assertEquals(days, range(1, 8)) + self.assertEquals(days, list(range(1, 8))) def test_aggregate_with_indexing(self): qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') @@ -326,4 +326,4 @@ class SampleModel(Model): class Numbers(Model): - number = UInt64Field() \ No newline at end of file + number = UInt64Field() diff --git a/tests/test_readonly.py b/tests/test_readonly.py index a192b62..ae3d54f 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from __future__ import unicode_literals import unittest import six diff --git a/tests/test_simple_fields.py b/tests/test_simple_fields.py index 645d9ed..7720dc1 100644 --- a/tests/test_simple_fields.py +++ b/tests/test_simple_fields.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import unittest from infi.clickhouse_orm.fields import * from datetime import date, datetime @@ -10,7 +11,7 @@ class SimpleFieldsTest(unittest.TestCase): f = DateTimeField() epoch = datetime(1970, 1, 1, tzinfo=pytz.utc) # Valid values - for value in (date(1970, 1, 1), datetime(1970, 1, 1), epoch, + for value in (date(1970, 1, 1), datetime(1970, 1, 1), epoch, epoch.astimezone(pytz.timezone('US/Eastern')), epoch.astimezone(pytz.timezone('Asia/Jerusalem')), '1970-01-01 00:00:00', '1970-01-17 00:00:17', '0000-00-00 00:00:00', 0): dt = f.to_python(value, pytz.utc) @@ -60,4 +61,4 @@ class SimpleFieldsTest(unittest.TestCase): # Range check for value in (-1, 1000): with self.assertRaises(ValueError): - f.validate(value) \ No newline at end of file + f.validate(value) diff --git a/tests/test_system_models.py b/tests/test_system_models.py index 3e48b0c..1a3b49a 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -1,6 +1,6 @@ +from __future__ import unicode_literals import unittest from datetime import date - import os import shutil from infi.clickhouse_orm.database import Database @@ -11,7 +11,8 @@ from infi.clickhouse_orm.system_models import SystemPart class SystemPartTest(unittest.TestCase): - BACKUP_DIR = '/opt/clickhouse/shadow/' + + BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/'] def setUp(self): self.database = Database('test-db') @@ -22,10 +23,11 @@ class SystemPartTest(unittest.TestCase): self.database.drop_database() def _get_backups(self): - if not os.path.exists(self.BACKUP_DIR): - return [] - _, dirnames, _ = next(os.walk(self.BACKUP_DIR)) - return dirnames + for dir in self.BACKUP_DIRS: + if os.path.exists(dir): + _, dirnames, _ = next(os.walk(dir)) + return dirnames + raise unittest.SkipTest('Cannot find backups dir') def test_get_all(self): parts = SystemPart.get(self.database) @@ -63,8 +65,6 @@ class SystemPartTest(unittest.TestCase): parts[0].freeze() backups = set(self._get_backups()) self.assertEqual(len(backups), len(prev_backups) + 1) - # Clean created backup - shutil.rmtree(self.BACKUP_DIR + '{0}'.format(list(backups - prev_backups)[0])) def test_fetch(self): # TODO Not tested, as I have no replication set