Finished Release v0.9.3

CHANGELOG.md

@@ -1,6 +1,12 @@
 Change Log
 ==========
 
+v0.9.3
+------
+- Changed license from PSF to BSD
+- Nullable fields support (yamiou)
+- Support for queryset slicing
+
 v0.9.2
 ------
 - Added `ne` and `not_in` queryset operators

LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2017 INFINIDAT
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

docs/class_reference.md

@@ -378,6 +378,13 @@ Extends Field
 #### ArrayField(inner_field, default=None, alias=None, materialized=None)
 
 
+### NullableField
+
+Extends Field
+
+#### NullableField(inner_field, default=None, alias=None, materialized=None, extra_null_values=None)
+
+
 ### FixedStringField
 
 Extends StringField
@@ -532,8 +539,8 @@ infi.clickhouse_orm.query
 ### QuerySet
 
 
-A queryset is an object that represents a database query using a specific `Model`. 
+A queryset is an object that represents a database query using a specific `Model`.
-It is lazy, meaning that it does not hit the database until you iterate over its 
+It is lazy, meaning that it does not hit the database until you iterate over its
 matching rows (model instances).
 
 #### QuerySet(model_cls, database)

docs/field_types.md

@@ -17,11 +17,12 @@ Currently the following field types are supported:
 | UInt16Field        | UInt16     | int                 | Range 0 to 65535
 | UInt32Field        | UInt32     | int                 | Range 0 to 4294967295
 | UInt64Field        | UInt64     | int/long            | Range 0 to 18446744073709551615
-| Float32Field       | Float32    | float               | 
+| Float32Field       | Float32    | float               |
-| Float64Field       | Float64    | float               | 
+| Float64Field       | Float64    | float               |
 | Enum8Field         | Enum8      | Enum                | See below
 | Enum16Field        | Enum16     | Enum                | See below
 | ArrayField         | Array      | list                | See below
+| NullableField      | Nullable   | See below           | See below
 
 DateTimeField and Time Zones
 ----------------------------
@@ -103,6 +104,32 @@ Usage:
     # created_date and username will contain a default value
     db.select('SELECT * FROM $db.event', model_class=Event)
 
+Working with nullable fields
+----------------------------
+From [some time](https://github.com/yandex/ClickHouse/pull/70) ClickHouse provides a NULL value support.
+Also see some information [here](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00395_nullable.sql).
+
+Wrapping another field in a `NullableField` makes it possible to assign `None` to that field. For example:
+
+    class EventData(models.Model):
+
+        date = fields.DateField()
+        comment = fields.NullableField(fields.StringField(), extra_null_values={''})
+        score = fields.NullableField(fields.UInt8Field())
+        serie = fields.NullableField(fields.ArrayField(fields.UInt8Field()))
+
+        engine = engines.MergeTree('date', ('date',))
+
+
+    score_event = EventData(date=date.today(), comment=None, score=5, serie=None)
+    comment_event = EventData(date=date.today(), comment='Excellent!', score=None, serie=None)
+    another_event = EventData(date=date.today(), comment='', score=None, serie=None)
+    action_event = EventData(date=date.today(), comment='', score=None, serie=[1, 2, 3])
+
+The `extra_null_values` parameter is an iterable of additional values that should be converted
+to `None`.
+
+NOTE: `ArrayField` of `NullableField` is not supported. Also `EnumField` cannot be nullable.
 
 ---
 

docs/querysets.md

@@ -4,7 +4,7 @@ Querysets
 A queryset is an object that represents a database query using a specific Model. It is lazy, meaning that it does not hit the database until you iterate over its matching rows (model instances). To create a base queryset for a model class, use:
 
     qs = Person.objects_in(database)
-    
+
 This queryset matches all Person instances in the database. You can get these instances using iteration:
 
     for person in qs:
@@ -19,7 +19,7 @@ The `filter` and `exclude` methods are used for filtering the matching instances
     >>> qs = qs.filter(first_name__startswith='V').exclude(birthday__lt='2000-01-01')
     >>> qs.conditions_as_sql()
     u"first_name LIKE 'V%' AND NOT (birthday < '2000-01-01')"
-    
+
 It is possible to specify several fields to filter or exclude by:
 
     >>> qs = Person.objects_in(database).filter(last_name='Smith', height__gt=1.75)
@@ -57,7 +57,7 @@ For example if we want to select only people with Irish last names:
 
     # A list of simple values
     qs = Person.objects_in(database).filter(last_name__in=["Murphy", "O'Sullivan"])
-    
+
     # A string
     subquery = "SELECT name from $db.irishlastname"
     qs = Person.objects_in(database).filter(last_name__in=subquery)
@@ -72,7 +72,7 @@ Counting and Checking Existence
 Use the `count` method to get the number of matches:
 
     Person.objects_in(database).count()
-    
+
 To check if there are any matches at all, you can use any of the following equivalent options:
 
     if qs.count(): ...
@@ -85,7 +85,7 @@ Ordering
 The sorting order of the results can be controlled using the `order_by` method:
 
     qs = Person.objects_in(database).order_by('last_name', 'first_name')
-    
+
 The default order is ascending. To use descending order, add a minus sign before the field name:
 
     qs = Person.objects_in(database).order_by('-height')
@@ -100,6 +100,25 @@ When some of the model fields aren't needed, it is more efficient to omit them f
     qs = Person.objects_in(database).only('first_name', 'birthday')
 
 
+Slicing
+-------
+
+It is possible to get a specific item from the queryset by index.
+
+      qs = Person.objects_in(database).order_by('last_name', 'first_name')
+      first = qs[0]
+
+It is also possible to get a range a instances using a slice. This returns a queryset,
+that you can either iterate over or convert to a list.
+
+      qs = Person.objects_in(database).order_by('last_name', 'first_name')
+      first_ten_people = list(qs[:10])
+      next_ten_people  = list(qs[10:20])
+
+You should use `order_by` to ensure a consistent ordering of the results.
+
+Trying to use negative indexes or a slice with a step (e.g. [0:100:2]) is not supported and will raise an `AssertionError`.
+
 ---
 
 [<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md)

docs/toc.md

@@ -20,12 +20,14 @@
       * [Counting and Checking Existence](querysets.md#counting-and-checking-existence)
       * [Ordering](querysets.md#ordering)
       * [Omitting Fields](querysets.md#omitting-fields)
+      * [Slicing](querysets.md#slicing)
 
    * [Field Types](field_types.md#field-types)
       * [DateTimeField and Time Zones](field_types.md#datetimefield-and-time-zones)
       * [Working with enum fields](field_types.md#working-with-enum-fields)
       * [Working with array fields](field_types.md#working-with-array-fields)
       * [Working with materialized and alias fields](field_types.md#working-with-materialized-and-alias-fields)
+      * [Working with nullable fields](field_types.md#working-with-nullable-fields)
 
    * [Table Engines](table_engines.md#table-engines)
       * [Simple Engines](table_engines.md#simple-engines)
@@ -60,6 +62,7 @@
          * [BaseFloatField](class_reference.md#basefloatfield)
          * [BaseEnumField](class_reference.md#baseenumfield)
          * [ArrayField](class_reference.md#arrayfield)
+         * [NullableField](class_reference.md#nullablefield)
          * [FixedStringField](class_reference.md#fixedstringfield)
          * [UInt8Field](class_reference.md#uint8field)
          * [UInt16Field](class_reference.md#uint16field)

setup.in

@@ -6,14 +6,14 @@ SETUP_INFO = dict(
     author_email = '${infi.recipe.template.version:author_email}',
 
     url = ${infi.recipe.template.version:homepage},
-    license = 'PSF',
+    license = 'BSD',
     description = """${project:description}""",
 
     # http://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers = [
         "Intended Audience :: Developers",
         "Intended Audience :: System Administrators",
-        "License :: OSI Approved :: Python Software Foundation License",
+        "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 2.7",

src/infi/clickhouse_orm/fields.py

@@ -177,7 +177,7 @@ class BaseIntField(Field):
             raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
     def to_db_string(self, value, quote=True):
-        # There's no need to call escape since numbers do not contain 
+        # There's no need to call escape since numbers do not contain
         # special characters, and never need quoting
         return text_type(value)
 
@@ -253,7 +253,7 @@ class BaseFloatField(Field):
             raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
     def to_db_string(self, value, quote=True):
-        # There's no need to call escape since numbers do not contain 
+        # There's no need to call escape since numbers do not contain
         # special characters, and never need quoting
         return text_type(value)
 
@@ -362,3 +362,32 @@ class ArrayField(Field):
         from .utils import escape
         return 'Array(%s)' % self.inner_field.get_sql(with_default=False)
 
+
+class NullableField(Field):
+
+    class_default = None
+
+    def __init__(self, inner_field, default=None, alias=None, materialized=None,
+                 extra_null_values=None):
+        self.inner_field = inner_field
+        self._null_values = [None]
+        if extra_null_values:
+            self._null_values.extend(extra_null_values)
+        super(NullableField, self).__init__(default, alias, materialized)
+
+    def to_python(self, value, timezone_in_use):
+        if value == '\\N' or value is None:
+            return None
+        return self.inner_field.to_python(value, timezone_in_use)
+
+    def validate(self, value):
+        value is None or self.inner_field.validate(value)
+
+    def to_db_string(self, value, quote=True):
+        if value in self._null_values:
+            return '\\N'
+        return self.inner_field.to_db_string(value, quote=quote)
+
+    def get_sql(self, with_default=True):
+        from .utils import escape
+        return 'Nullable(%s)' % self.inner_field.get_sql(with_default=False)

src/infi/clickhouse_orm/models.py

@@ -62,6 +62,10 @@ class ModelBase(type):
         if db_type.startswith('FixedString'):
             length = int(db_type[12 : -1])
             return orm_fields.FixedStringField(length)
+        # Nullable
+        if db_type.startswith('Nullable'):
+            inner_field = cls.create_ad_hoc_field(db_type[9 : -1])
+            return orm_fields.NullableField(inner_field)
         # Simple fields
         name = db_type + 'Field'
         if not hasattr(orm_fields, name):

src/infi/clickhouse_orm/query.py

@@ -59,7 +59,7 @@ class InOperator(Operator):
 class LikeOperator(Operator):
     """
     A LIKE operator that matches the field to a given pattern. Can be
-    case sensitive or insensitive. 
+    case sensitive or insensitive.
     """
 
     def __init__(self, pattern, case_sensitive=True):
@@ -168,8 +168,8 @@ class Q(object):
 
 class QuerySet(object):
     """
-    A queryset is an object that represents a database query using a specific `Model`. 
+    A queryset is an object that represents a database query using a specific `Model`.
-    It is lazy, meaning that it does not hit the database until you iterate over its 
+    It is lazy, meaning that it does not hit the database until you iterate over its
     matching rows (model instances).
     """
 
@@ -183,10 +183,11 @@ class QuerySet(object):
         self._order_by = []
         self._q = []
         self._fields = []
+        self._limits = None
 
     def __iter__(self):
         """
-        Iterates over the model instances matching this queryset 
+        Iterates over the model instances matching this queryset
         """
         return self._database.select(self.as_sql(), self._model_cls)
 
@@ -201,7 +202,25 @@ class QuerySet(object):
 
     def __unicode__(self):
         return self.as_sql()
-        
+
+    def __getitem__(self, s):
+        if isinstance(s, six.integer_types):
+            # Single index
+            assert s >= 0, 'negative indexes are not supported'
+            qs = copy(self)
+            qs._limits = (s, 1)
+            return iter(qs).next()
+        else:
+            # Slice
+            assert s.step in (None, 1), 'step is not supported in slices'
+            start = s.start or 0
+            stop = s.stop or 2**63 - 1
+            assert start >= 0 and stop >= 0, 'negative indexes are not supported'
+            assert start <= stop, 'start of slice cannot be smaller than its end'
+            qs = copy(self)
+            qs._limits = (start, stop - start)
+            return qs
+
     def as_sql(self):
         """
         Returns the whole query as a SQL string.
@@ -210,8 +229,10 @@ class QuerySet(object):
         if self._fields:
             fields = ', '.join('`%s`' % field for field in self._fields)
         ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
-        params = (fields, self._database.db_name, self._model_cls.table_name(), self.conditions_as_sql(), ordering)
+        limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
-        return u'SELECT %s\nFROM `%s`.`%s`\nWHERE %s%s' % params
+        params = (fields, self._model_cls.table_name(),
+                  self.conditions_as_sql(), ordering, limit)
+        return u'SELECT %s\nFROM `%s`\nWHERE %s%s%s' % params
 
     def order_by_as_sql(self):
         """
@@ -236,7 +257,7 @@ class QuerySet(object):
         Returns the number of matching model instances.
         """
         return self._database.count(self._model_cls, self.conditions_as_sql())
-        
+
     def order_by(self, *field_names):
         """
         Returns a new `QuerySet` instance with the ordering changed.

tests/test_nullable_fields.py

@@ -0,0 +1,134 @@
+import unittest
+import pytz
+
+from infi.clickhouse_orm.database import Database
+from infi.clickhouse_orm.models import Model
+from infi.clickhouse_orm.fields import *
+from infi.clickhouse_orm.engines import *
+
+from datetime import date, datetime
+
+
+class NullableFieldsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.database = Database('test-db')
+        self.database.create_table(ModelWithNullable)
+
+    def tearDown(self):
+        self.database.drop_database()
+
+    def test_nullable_datetime_field(self):
+        f = NullableField(DateTimeField())
+        epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
+        # Valid values
+        for value in (date(1970, 1, 1),
+                      datetime(1970, 1, 1),
+                      epoch,
+                      epoch.astimezone(pytz.timezone('US/Eastern')),
+                      epoch.astimezone(pytz.timezone('Asia/Jerusalem')),
+                      '1970-01-01 00:00:00',
+                      '1970-01-17 00:00:17',
+                      '0000-00-00 00:00:00',
+                      0,
+                      '\\N'):
+            dt = f.to_python(value, pytz.utc)
+            if value == '\\N':
+                self.assertIsNone(dt)
+            else:
+                self.assertEquals(dt.tzinfo, pytz.utc)
+            # Verify that conversion to and from db string does not change value
+            dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc)
+            self.assertEquals(dt, dt2)
+        # Invalid values
+        for value in ('nope', '21/7/1999', 0.5):
+            with self.assertRaises(ValueError):
+                f.to_python(value, pytz.utc)
+
+    def test_nullable_uint8_field(self):
+        f = NullableField(UInt8Field())
+        # Valid values
+        for value in (17, '17', 17.0, '\\N'):
+            python_value = f.to_python(value, pytz.utc)
+            if value == '\\N':
+                self.assertIsNone(python_value)
+                self.assertEqual(value, f.to_db_string(python_value))
+            else:
+                self.assertEquals(python_value, 17)
+
+        # Invalid values
+        for value in ('nope', date.today()):
+            with self.assertRaises(ValueError):
+                f.to_python(value, pytz.utc)
+
+    def test_nullable_string_field(self):
+        f = NullableField(StringField())
+        # Valid values
+        for value in ('\\\\N', 'N', 'some text', '\\N'):
+            python_value = f.to_python(value, pytz.utc)
+            if value == '\\N':
+                self.assertIsNone(python_value)
+                self.assertEqual(value, f.to_db_string(python_value))
+            else:
+                self.assertEquals(python_value, value)
+
+    def _insert_sample_data(self):
+        dt = date(1970, 1, 1)
+        self.database.insert([
+            ModelWithNullable(date_field='2016-08-30',
+                              null_str='', null_int=42, null_date=dt,
+                              null_array=None),
+            ModelWithNullable(date_field='2016-08-30',
+                              null_str='nothing', null_int=None, null_date=None,
+                              null_array=[1, 2, 3]),
+            ModelWithNullable(date_field='2016-08-31',
+                              null_str=None, null_int=42, null_date=dt,
+                              null_array=[]),
+            ModelWithNullable(date_field='2016-08-31',
+                              null_str=None, null_int=None, null_date=None,
+                              null_array=[3, 2, 1])
+        ])
+
+    def _assert_sample_data(self, results):
+        dt = date(1970, 1, 1)
+        self.assertEquals(len(results), 4)
+        self.assertIsNone(results[0].null_str)
+        self.assertEquals(results[0].null_int, 42)
+        self.assertEquals(results[0].null_date, dt)
+        self.assertIsNone(results[1].null_date)
+        self.assertEquals(results[1].null_str, 'nothing')
+        self.assertIsNone(results[1].null_date)
+        self.assertIsNone(results[2].null_str)
+        self.assertEquals(results[2].null_date, dt)
+        self.assertEquals(results[2].null_int, 42)
+        self.assertIsNone(results[3].null_int)
+        self.assertIsNone(results[3].null_str)
+        self.assertIsNone(results[3].null_date)
+
+        self.assertIsNone(results[0].null_array)
+        self.assertEquals(results[1].null_array, [1, 2, 3])
+        self.assertEquals(results[2].null_array, [])
+        self.assertEquals(results[3].null_array, [3, 2, 1])
+
+    def test_insert_and_select(self):
+        self._insert_sample_data()
+        query = 'SELECT * from $table ORDER BY date_field'
+        results = list(self.database.select(query, ModelWithNullable))
+        self._assert_sample_data(results)
+
+    def test_ad_hoc_model(self):
+        self._insert_sample_data()
+        query = 'SELECT * from $db.modelwithnullable ORDER BY date_field'
+        results = list(self.database.select(query))
+        self._assert_sample_data(results)
+
+
+class ModelWithNullable(Model):
+
+    date_field = DateField()
+    null_str = NullableField(StringField(), extra_null_values={''})
+    null_int = NullableField(Int32Field())
+    null_date = NullableField(DateField())
+    null_array = NullableField(ArrayField(Int32Field()))
+
+    engine = MergeTree('date_field', ('date_field',))

tests/test_querysets.py

@@ -18,11 +18,11 @@ class QuerySetTestCase(TestCaseWithData):
     def setUp(self):
         super(QuerySetTestCase, self).setUp()
         self.database.insert(self._sample_data())
-        
+
     def _test_qs(self, qs, expected_count):
         logging.info(qs.as_sql())
         for instance in qs:
-            logging.info('\t%s' % instance.to_dict()) 
+            logging.info('\t%s' % instance.to_dict())
         self.assertEquals(qs.count(), expected_count)
 
     def test_no_filtering(self):
@@ -138,6 +138,30 @@ class QuerySetTestCase(TestCaseWithData):
         self._test_qs(qs.filter(num__in=(1, 2, 3)), 3)
         self._test_qs(qs.filter(num__in=range(1, 4)), 3)
 
+    def test_slicing(self):
+        db = Database('system')
+        numbers = range(100)
+        qs = Numbers.objects_in(db)
+        self.assertEquals(qs[0].number, numbers[0])
+        self.assertEquals(qs[5].number, numbers[5])
+        self.assertEquals([row.number for row in qs[:1]], numbers[:1])
+        self.assertEquals([row.number for row in qs[:10]], numbers[:10])
+        self.assertEquals([row.number for row in qs[3:10]], numbers[3:10])
+        self.assertEquals([row.number for row in qs[9:10]], numbers[9:10])
+        self.assertEquals([row.number for row in qs[10:10]], numbers[10:10])
+
+    def test_invalid_slicing(self):
+        db = Database('system')
+        qs = Numbers.objects_in(db)
+        with self.assertRaises(AssertionError):
+            qs[3:10:2]
+        with self.assertRaises(AssertionError):
+            qs[-5]
+        with self.assertRaises(AssertionError):
+            qs[:-5]
+        with self.assertRaises(AssertionError):
+            qs[50:1]
+
 
 Color = Enum('Color', u'red blue green yellow brown white black')
 
@@ -149,4 +173,9 @@ class SampleModel(Model):
     num = Int32Field()
     color = Enum8Field(Color)
 
-    engine = MergeTree('materialized_date', ('materialized_date',))
+    engine = MergeTree('materialized_date', ('materialized_date',))
+
+
+class Numbers(Model):
+
+    number = UInt64Field()