Finished Release v0.9.6

CHANGELOG.md

@@ -1,6 +1,12 @@
 Change Log
 ==========
 
+v0.9.6
+------
+- Fix python3 compatibility (TvoroG)
+- Nullable arrays not supported in latest ClickHouse version
+- system.parts table no longer includes "replicated" column in latest ClickHouse version
+
 v0.9.5
 ------
 - Added `QuerySet.paginate()`

buildout.cfg

@@ -29,7 +29,7 @@ homepage = https://github.com/Infinidat/infi.clickhouse_orm
 
 [isolated-python]
 recipe = infi.recipe.python
-version = v2.7.9.4
+version = v2.7.12.4
 
 [setup.py]
 recipe = infi.recipe.template.version

src/infi/clickhouse_orm/database.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import requests
 from collections import namedtuple
 from .models import ModelBase

src/infi/clickhouse_orm/engines.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from .utils import comma_join
 
 

src/infi/clickhouse_orm/fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from six import string_types, text_type, binary_type
 import datetime
 import pytz

src/infi/clickhouse_orm/models.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from logging import getLogger
 
 from six import with_metaclass
@@ -18,7 +19,7 @@ class ModelBase(type):
     ad_hoc_model_cache = {}
 
     def __new__(cls, name, bases, attrs):
-        new_cls = super(ModelBase, cls).__new__(cls, name, bases, attrs)
+        new_cls = super(ModelBase, cls).__new__(cls, str(name), bases, attrs)
         # Collect fields from parent classes
         base_fields = []
         for base in bases:

src/infi/clickhouse_orm/query.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import six
 import pytz
 from copy import copy
@@ -167,6 +168,7 @@ class Q(object):
         return q
 
 
+@six.python_2_unicode_compatible
 class QuerySet(object):
     """
     A queryset is an object that represents a database query using a specific `Model`.
@@ -190,7 +192,6 @@ class QuerySet(object):
         """
         Iterates over the model instances matching this queryset
         """
-        print self.as_sql()
         return self._database.select(self.as_sql(), self._model_cls)
 
     def __bool__(self):
@@ -202,7 +203,7 @@ class QuerySet(object):
     def __nonzero__(self):      # Python 2 compatibility
         return type(self).__bool__(self)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.as_sql()
 
     def __getitem__(self, s):
@@ -211,7 +212,7 @@ class QuerySet(object):
             assert s >= 0, 'negative indexes are not supported'
             qs = copy(self)
             qs._limits = (s, 1)
-            return iter(qs).next()
+            return six.next(iter(qs))
         else:
             # Slice
             assert s.step in (None, 1), 'step is not supported in slices'

src/infi/clickhouse_orm/system_models.py

@@ -2,6 +2,7 @@
 This file contains system readonly models that can be got from database
 https://clickhouse.yandex/reference_en.html#System tables
 """
+from __future__ import unicode_literals
 from six import string_types
 
 from .database import Database
@@ -25,7 +26,6 @@ class SystemPart(Model):
     engine = StringField()  # Name of the table engine, without parameters.
     partition = StringField()  # Name of the partition, in the format YYYYMM.
     name = StringField()  # Name of the part.
-    replicated = UInt8Field()  # Whether the part belongs to replicated data.
 
     # Whether the part is used in a table, or is no longer needed and will be deleted soon.
     # Inactive parts remain after merging.

src/infi/clickhouse_orm/utils.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from six import string_types, binary_type, text_type, PY3
 import codecs
 import re
@@ -43,7 +44,7 @@ def parse_tsv(line):
         line = line.decode()
     if line and line[-1] == '\n':
         line = line[:-1]
-    return [unescape(value) for value in line.split('\t')]
+    return [unescape(value) for value in line.split(str('\t'))]
 
 
 def parse_array(array_string):

tests/base_test_with_data.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.database import Database

tests/test_alias_fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 from datetime import date
 
@@ -65,5 +66,3 @@ class ModelWithAliasFields(Model):
     alias_date = DateField(alias='date_field')
 
     engine = MergeTree('date_field', ('date_field',))
-
-

tests/test_array_fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 from datetime import date
 
@@ -70,4 +71,3 @@ class ModelWithArrays(Model):
     arr_date = ArrayField(DateField())
 
     engine = MergeTree('date_field', ('date_field',))
-

tests/test_buffer.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.models import BufferModel
@@ -25,5 +25,3 @@ class BufferTestCase(TestCaseWithData):
 class PersonBuffer(BufferModel, Person):
 
     engine = Buffer(Person)
-
-

tests/test_database.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.database import Database, DatabaseException

tests/test_engines.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.database import Database, DatabaseException

tests/test_enum_fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.database import Database

tests/test_fixed_string_fields.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.database import Database

tests/test_inheritance.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 import datetime
 import pytz
@@ -49,4 +50,3 @@ class Model1(ParentModel):
 class Model2(ParentModel):
 
     float_field = Float32Field()
-

tests/test_materialized_fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 from datetime import date
 
@@ -65,5 +66,3 @@ class ModelWithMaterializedFields(Model):
     mat_date = DateField(materialized=u'toDate(date_time_field)')
 
     engine = MergeTree('mat_date', ('mat_date',))
-
-

tests/test_migrations.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 
 from infi.clickhouse_orm.database import Database

tests/test_models.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 import datetime
 import pytz
@@ -89,4 +90,3 @@ class SimpleModel(Model):
     alias_field = Float32Field(alias='float_field')
 
     engine = MergeTree('date_field', ('int_field', 'date_field'))
-

tests/test_nullable_fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 import pytz
 
@@ -75,18 +76,10 @@ class NullableFieldsTest(unittest.TestCase):
     def _insert_sample_data(self):
         dt = date(1970, 1, 1)
         self.database.insert([
-            ModelWithNullable(date_field='2016-08-30',
+            ModelWithNullable(date_field='2016-08-30', null_str='', null_int=42, null_date=dt),
-                              null_str='', null_int=42, null_date=dt,
+            ModelWithNullable(date_field='2016-08-30', null_str='nothing', null_int=None, null_date=None),
-                              null_array=None),
+            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=42, null_date=dt),
-            ModelWithNullable(date_field='2016-08-30',
+            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None)
-                              null_str='nothing', null_int=None, null_date=None,
-                              null_array=[1, 2, 3]),
-            ModelWithNullable(date_field='2016-08-31',
-                              null_str=None, null_int=42, null_date=dt,
-                              null_array=[]),
-            ModelWithNullable(date_field='2016-08-31',
-                              null_str=None, null_int=None, null_date=None,
-                              null_array=[3, 2, 1])
         ])
 
     def _assert_sample_data(self, results):
@@ -105,11 +98,6 @@ class NullableFieldsTest(unittest.TestCase):
         self.assertIsNone(results[3].null_str)
         self.assertIsNone(results[3].null_date)
 
-        self.assertIsNone(results[0].null_array)
-        self.assertEquals(results[1].null_array, [1, 2, 3])
-        self.assertEquals(results[2].null_array, [])
-        self.assertEquals(results[3].null_array, [3, 2, 1])
-
     def test_insert_and_select(self):
         self._insert_sample_data()
         query = 'SELECT * from $table ORDER BY date_field'
@@ -129,6 +117,5 @@ class ModelWithNullable(Model):
     null_str = NullableField(StringField(), extra_null_values={''})
     null_int = NullableField(Int32Field())
     null_date = NullableField(DateField())
-    null_array = NullableField(ArrayField(Int32Field()))
 
     engine = MergeTree('date_field', ('date_field',))

tests/test_querysets.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals, print_function
 import unittest
 
 from infi.clickhouse_orm.database import Database
@@ -140,7 +140,7 @@ class QuerySetTestCase(TestCaseWithData):
 
     def test_slicing(self):
         db = Database('system')
-        numbers = range(100)
+        numbers = list(range(100))
         qs = Numbers.objects_in(db)
         self.assertEquals(qs[0].number, numbers[0])
         self.assertEquals(qs[5].number, numbers[5])
@@ -211,7 +211,7 @@ class AggregateTestCase(TestCaseWithData):
 
     def test_aggregate_no_grouping(self):
         qs = Person.objects_in(self.database).aggregate(average_height='avg(height)', count='count()')
-        print qs.as_sql()
+        print(qs.as_sql())
         self.assertEquals(qs.count(), 1)
         for row in qs:
             self.assertAlmostEqual(row.average_height, 1.6923, places=4)
@@ -220,14 +220,14 @@ class AggregateTestCase(TestCaseWithData):
     def test_aggregate_with_filter(self):
         # When filter comes before aggregate
         qs = Person.objects_in(self.database).filter(first_name='Warren').aggregate(average_height='avg(height)', count='count()')
-        print qs.as_sql()
+        print(qs.as_sql())
         self.assertEquals(qs.count(), 1)
         for row in qs:
             self.assertAlmostEqual(row.average_height, 1.675, places=4)
             self.assertEquals(row.count, 2)
         # When filter comes after aggregate
         qs = Person.objects_in(self.database).aggregate(average_height='avg(height)', count='count()').filter(first_name='Warren')
-        print qs.as_sql()
+        print(qs.as_sql())
         self.assertEquals(qs.count(), 1)
         for row in qs:
             self.assertAlmostEqual(row.average_height, 1.675, places=4)
@@ -235,7 +235,7 @@ class AggregateTestCase(TestCaseWithData):
 
     def test_aggregate_with_implicit_grouping(self):
         qs = Person.objects_in(self.database).aggregate('first_name', average_height='avg(height)', count='count()')
-        print qs.as_sql()
+        print(qs.as_sql())
         self.assertEquals(qs.count(), 94)
         total = 0
         for row in qs:
@@ -246,7 +246,7 @@ class AggregateTestCase(TestCaseWithData):
 
     def test_aggregate_with_explicit_grouping(self):
         qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday')
-        print qs.as_sql()
+        print(qs.as_sql())
         self.assertEquals(qs.count(), 7)
         total = 0
         for row in qs:
@@ -256,7 +256,7 @@ class AggregateTestCase(TestCaseWithData):
     def test_aggregate_with_order_by(self):
         qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday')
         days = [row.weekday for row in qs.order_by('weekday')]
-        self.assertEquals(days, range(1, 8))
+        self.assertEquals(days, list(range(1, 8)))
 
     def test_aggregate_with_indexing(self):
         qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday')

tests/test_readonly.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals
 import unittest
 
 import six

tests/test_simple_fields.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import unittest
 from infi.clickhouse_orm.fields import *
 from datetime import date, datetime

tests/test_system_models.py

@@ -1,6 +1,6 @@
+from __future__ import unicode_literals
 import unittest
 from datetime import date
-
 import os
 import shutil
 from infi.clickhouse_orm.database import Database
@@ -11,7 +11,8 @@ from infi.clickhouse_orm.system_models import SystemPart
 
 
 class SystemPartTest(unittest.TestCase):
-    BACKUP_DIR = '/opt/clickhouse/shadow/'
+
+    BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/']
 
     def setUp(self):
         self.database = Database('test-db')
@@ -22,10 +23,11 @@ class SystemPartTest(unittest.TestCase):
         self.database.drop_database()
 
     def _get_backups(self):
-        if not os.path.exists(self.BACKUP_DIR):
+        for dir in self.BACKUP_DIRS:
-            return []
+            if os.path.exists(dir):
-        _, dirnames, _ = next(os.walk(self.BACKUP_DIR))
+                _, dirnames, _ = next(os.walk(dir))
-        return dirnames
+                return dirnames
+        raise unittest.SkipTest('Cannot find backups dir')
 
     def test_get_all(self):
         parts = SystemPart.get(self.database)
@@ -63,8 +65,6 @@ class SystemPartTest(unittest.TestCase):
         parts[0].freeze()
         backups = set(self._get_backups())
         self.assertEqual(len(backups), len(prev_backups) + 1)
-        # Clean created backup
-        shutil.rmtree(self.BACKUP_DIR + '{0}'.format(list(backups - prev_backups)[0]))
 
     def test_fetch(self):
         # TODO Not tested, as I have no replication set