mirror of
https://github.com/Infinidat/infi.clickhouse_orm.git
synced 2024-11-22 09:06:41 +03:00
Finished Release v0.9.3
This commit is contained in:
commit
220616a151
|
@ -1,6 +1,12 @@
|
|||
Change Log
|
||||
==========
|
||||
|
||||
v0.9.3
|
||||
------
|
||||
- Changed license from PSF to BSD
|
||||
- Nullable fields support (yamiou)
|
||||
- Support for queryset slicing
|
||||
|
||||
v0.9.2
|
||||
------
|
||||
- Added `ne` and `not_in` queryset operators
|
||||
|
|
26
LICENSE
Normal file
26
LICENSE
Normal file
|
@ -0,0 +1,26 @@
|
|||
Copyright (c) 2017 INFINIDAT
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@ -378,6 +378,13 @@ Extends Field
|
|||
#### ArrayField(inner_field, default=None, alias=None, materialized=None)
|
||||
|
||||
|
||||
### NullableField
|
||||
|
||||
Extends Field
|
||||
|
||||
#### NullableField(inner_field, default=None, alias=None, materialized=None, extra_null_values=None)
|
||||
|
||||
|
||||
### FixedStringField
|
||||
|
||||
Extends StringField
|
||||
|
|
|
@ -22,6 +22,7 @@ Currently the following field types are supported:
|
|||
| Enum8Field | Enum8 | Enum | See below
|
||||
| Enum16Field | Enum16 | Enum | See below
|
||||
| ArrayField | Array | list | See below
|
||||
| NullableField | Nullable | See below | See below
|
||||
|
||||
DateTimeField and Time Zones
|
||||
----------------------------
|
||||
|
@ -103,6 +104,32 @@ Usage:
|
|||
# created_date and username will contain a default value
|
||||
db.select('SELECT * FROM $db.event', model_class=Event)
|
||||
|
||||
Working with nullable fields
|
||||
----------------------------
|
||||
From [some time](https://github.com/yandex/ClickHouse/pull/70) ClickHouse provides a NULL value support.
|
||||
Also see some information [here](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00395_nullable.sql).
|
||||
|
||||
Wrapping another field in a `NullableField` makes it possible to assign `None` to that field. For example:
|
||||
|
||||
class EventData(models.Model):
|
||||
|
||||
date = fields.DateField()
|
||||
comment = fields.NullableField(fields.StringField(), extra_null_values={''})
|
||||
score = fields.NullableField(fields.UInt8Field())
|
||||
serie = fields.NullableField(fields.ArrayField(fields.UInt8Field()))
|
||||
|
||||
engine = engines.MergeTree('date', ('date',))
|
||||
|
||||
|
||||
score_event = EventData(date=date.today(), comment=None, score=5, serie=None)
|
||||
comment_event = EventData(date=date.today(), comment='Excellent!', score=None, serie=None)
|
||||
another_event = EventData(date=date.today(), comment='', score=None, serie=None)
|
||||
action_event = EventData(date=date.today(), comment='', score=None, serie=[1, 2, 3])
|
||||
|
||||
The `extra_null_values` parameter is an iterable of additional values that should be converted
|
||||
to `None`.
|
||||
|
||||
NOTE: `ArrayField` of `NullableField` is not supported. Also `EnumField` cannot be nullable.
|
||||
|
||||
---
|
||||
|
||||
|
|
|
@ -100,6 +100,25 @@ When some of the model fields aren't needed, it is more efficient to omit them f
|
|||
qs = Person.objects_in(database).only('first_name', 'birthday')
|
||||
|
||||
|
||||
Slicing
|
||||
-------
|
||||
|
||||
It is possible to get a specific item from the queryset by index.
|
||||
|
||||
qs = Person.objects_in(database).order_by('last_name', 'first_name')
|
||||
first = qs[0]
|
||||
|
||||
It is also possible to get a range a instances using a slice. This returns a queryset,
|
||||
that you can either iterate over or convert to a list.
|
||||
|
||||
qs = Person.objects_in(database).order_by('last_name', 'first_name')
|
||||
first_ten_people = list(qs[:10])
|
||||
next_ten_people = list(qs[10:20])
|
||||
|
||||
You should use `order_by` to ensure a consistent ordering of the results.
|
||||
|
||||
Trying to use negative indexes or a slice with a step (e.g. [0:100:2]) is not supported and will raise an `AssertionError`.
|
||||
|
||||
---
|
||||
|
||||
[<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md)
|
|
@ -20,12 +20,14 @@
|
|||
* [Counting and Checking Existence](querysets.md#counting-and-checking-existence)
|
||||
* [Ordering](querysets.md#ordering)
|
||||
* [Omitting Fields](querysets.md#omitting-fields)
|
||||
* [Slicing](querysets.md#slicing)
|
||||
|
||||
* [Field Types](field_types.md#field-types)
|
||||
* [DateTimeField and Time Zones](field_types.md#datetimefield-and-time-zones)
|
||||
* [Working with enum fields](field_types.md#working-with-enum-fields)
|
||||
* [Working with array fields](field_types.md#working-with-array-fields)
|
||||
* [Working with materialized and alias fields](field_types.md#working-with-materialized-and-alias-fields)
|
||||
* [Working with nullable fields](field_types.md#working-with-nullable-fields)
|
||||
|
||||
* [Table Engines](table_engines.md#table-engines)
|
||||
* [Simple Engines](table_engines.md#simple-engines)
|
||||
|
@ -60,6 +62,7 @@
|
|||
* [BaseFloatField](class_reference.md#basefloatfield)
|
||||
* [BaseEnumField](class_reference.md#baseenumfield)
|
||||
* [ArrayField](class_reference.md#arrayfield)
|
||||
* [NullableField](class_reference.md#nullablefield)
|
||||
* [FixedStringField](class_reference.md#fixedstringfield)
|
||||
* [UInt8Field](class_reference.md#uint8field)
|
||||
* [UInt16Field](class_reference.md#uint16field)
|
||||
|
|
4
setup.in
4
setup.in
|
@ -6,14 +6,14 @@ SETUP_INFO = dict(
|
|||
author_email = '${infi.recipe.template.version:author_email}',
|
||||
|
||||
url = ${infi.recipe.template.version:homepage},
|
||||
license = 'PSF',
|
||||
license = 'BSD',
|
||||
description = """${project:description}""",
|
||||
|
||||
# http://pypi.python.org/pypi?%3Aaction=list_classifiers
|
||||
classifiers = [
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: System Administrators",
|
||||
"License :: OSI Approved :: Python Software Foundation License",
|
||||
"License :: OSI Approved :: BSD License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python",
|
||||
"Programming Language :: Python :: 2.7",
|
||||
|
|
|
@ -362,3 +362,32 @@ class ArrayField(Field):
|
|||
from .utils import escape
|
||||
return 'Array(%s)' % self.inner_field.get_sql(with_default=False)
|
||||
|
||||
|
||||
class NullableField(Field):
|
||||
|
||||
class_default = None
|
||||
|
||||
def __init__(self, inner_field, default=None, alias=None, materialized=None,
|
||||
extra_null_values=None):
|
||||
self.inner_field = inner_field
|
||||
self._null_values = [None]
|
||||
if extra_null_values:
|
||||
self._null_values.extend(extra_null_values)
|
||||
super(NullableField, self).__init__(default, alias, materialized)
|
||||
|
||||
def to_python(self, value, timezone_in_use):
|
||||
if value == '\\N' or value is None:
|
||||
return None
|
||||
return self.inner_field.to_python(value, timezone_in_use)
|
||||
|
||||
def validate(self, value):
|
||||
value is None or self.inner_field.validate(value)
|
||||
|
||||
def to_db_string(self, value, quote=True):
|
||||
if value in self._null_values:
|
||||
return '\\N'
|
||||
return self.inner_field.to_db_string(value, quote=quote)
|
||||
|
||||
def get_sql(self, with_default=True):
|
||||
from .utils import escape
|
||||
return 'Nullable(%s)' % self.inner_field.get_sql(with_default=False)
|
||||
|
|
|
@ -62,6 +62,10 @@ class ModelBase(type):
|
|||
if db_type.startswith('FixedString'):
|
||||
length = int(db_type[12 : -1])
|
||||
return orm_fields.FixedStringField(length)
|
||||
# Nullable
|
||||
if db_type.startswith('Nullable'):
|
||||
inner_field = cls.create_ad_hoc_field(db_type[9 : -1])
|
||||
return orm_fields.NullableField(inner_field)
|
||||
# Simple fields
|
||||
name = db_type + 'Field'
|
||||
if not hasattr(orm_fields, name):
|
||||
|
|
|
@ -183,6 +183,7 @@ class QuerySet(object):
|
|||
self._order_by = []
|
||||
self._q = []
|
||||
self._fields = []
|
||||
self._limits = None
|
||||
|
||||
def __iter__(self):
|
||||
"""
|
||||
|
@ -202,6 +203,24 @@ class QuerySet(object):
|
|||
def __unicode__(self):
|
||||
return self.as_sql()
|
||||
|
||||
def __getitem__(self, s):
|
||||
if isinstance(s, six.integer_types):
|
||||
# Single index
|
||||
assert s >= 0, 'negative indexes are not supported'
|
||||
qs = copy(self)
|
||||
qs._limits = (s, 1)
|
||||
return iter(qs).next()
|
||||
else:
|
||||
# Slice
|
||||
assert s.step in (None, 1), 'step is not supported in slices'
|
||||
start = s.start or 0
|
||||
stop = s.stop or 2**63 - 1
|
||||
assert start >= 0 and stop >= 0, 'negative indexes are not supported'
|
||||
assert start <= stop, 'start of slice cannot be smaller than its end'
|
||||
qs = copy(self)
|
||||
qs._limits = (start, stop - start)
|
||||
return qs
|
||||
|
||||
def as_sql(self):
|
||||
"""
|
||||
Returns the whole query as a SQL string.
|
||||
|
@ -210,8 +229,10 @@ class QuerySet(object):
|
|||
if self._fields:
|
||||
fields = ', '.join('`%s`' % field for field in self._fields)
|
||||
ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
|
||||
params = (fields, self._database.db_name, self._model_cls.table_name(), self.conditions_as_sql(), ordering)
|
||||
return u'SELECT %s\nFROM `%s`.`%s`\nWHERE %s%s' % params
|
||||
limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
|
||||
params = (fields, self._model_cls.table_name(),
|
||||
self.conditions_as_sql(), ordering, limit)
|
||||
return u'SELECT %s\nFROM `%s`\nWHERE %s%s%s' % params
|
||||
|
||||
def order_by_as_sql(self):
|
||||
"""
|
||||
|
|
134
tests/test_nullable_fields.py
Normal file
134
tests/test_nullable_fields.py
Normal file
|
@ -0,0 +1,134 @@
|
|||
import unittest
|
||||
import pytz
|
||||
|
||||
from infi.clickhouse_orm.database import Database
|
||||
from infi.clickhouse_orm.models import Model
|
||||
from infi.clickhouse_orm.fields import *
|
||||
from infi.clickhouse_orm.engines import *
|
||||
|
||||
from datetime import date, datetime
|
||||
|
||||
|
||||
class NullableFieldsTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.database = Database('test-db')
|
||||
self.database.create_table(ModelWithNullable)
|
||||
|
||||
def tearDown(self):
|
||||
self.database.drop_database()
|
||||
|
||||
def test_nullable_datetime_field(self):
|
||||
f = NullableField(DateTimeField())
|
||||
epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
|
||||
# Valid values
|
||||
for value in (date(1970, 1, 1),
|
||||
datetime(1970, 1, 1),
|
||||
epoch,
|
||||
epoch.astimezone(pytz.timezone('US/Eastern')),
|
||||
epoch.astimezone(pytz.timezone('Asia/Jerusalem')),
|
||||
'1970-01-01 00:00:00',
|
||||
'1970-01-17 00:00:17',
|
||||
'0000-00-00 00:00:00',
|
||||
0,
|
||||
'\\N'):
|
||||
dt = f.to_python(value, pytz.utc)
|
||||
if value == '\\N':
|
||||
self.assertIsNone(dt)
|
||||
else:
|
||||
self.assertEquals(dt.tzinfo, pytz.utc)
|
||||
# Verify that conversion to and from db string does not change value
|
||||
dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc)
|
||||
self.assertEquals(dt, dt2)
|
||||
# Invalid values
|
||||
for value in ('nope', '21/7/1999', 0.5):
|
||||
with self.assertRaises(ValueError):
|
||||
f.to_python(value, pytz.utc)
|
||||
|
||||
def test_nullable_uint8_field(self):
|
||||
f = NullableField(UInt8Field())
|
||||
# Valid values
|
||||
for value in (17, '17', 17.0, '\\N'):
|
||||
python_value = f.to_python(value, pytz.utc)
|
||||
if value == '\\N':
|
||||
self.assertIsNone(python_value)
|
||||
self.assertEqual(value, f.to_db_string(python_value))
|
||||
else:
|
||||
self.assertEquals(python_value, 17)
|
||||
|
||||
# Invalid values
|
||||
for value in ('nope', date.today()):
|
||||
with self.assertRaises(ValueError):
|
||||
f.to_python(value, pytz.utc)
|
||||
|
||||
def test_nullable_string_field(self):
|
||||
f = NullableField(StringField())
|
||||
# Valid values
|
||||
for value in ('\\\\N', 'N', 'some text', '\\N'):
|
||||
python_value = f.to_python(value, pytz.utc)
|
||||
if value == '\\N':
|
||||
self.assertIsNone(python_value)
|
||||
self.assertEqual(value, f.to_db_string(python_value))
|
||||
else:
|
||||
self.assertEquals(python_value, value)
|
||||
|
||||
def _insert_sample_data(self):
|
||||
dt = date(1970, 1, 1)
|
||||
self.database.insert([
|
||||
ModelWithNullable(date_field='2016-08-30',
|
||||
null_str='', null_int=42, null_date=dt,
|
||||
null_array=None),
|
||||
ModelWithNullable(date_field='2016-08-30',
|
||||
null_str='nothing', null_int=None, null_date=None,
|
||||
null_array=[1, 2, 3]),
|
||||
ModelWithNullable(date_field='2016-08-31',
|
||||
null_str=None, null_int=42, null_date=dt,
|
||||
null_array=[]),
|
||||
ModelWithNullable(date_field='2016-08-31',
|
||||
null_str=None, null_int=None, null_date=None,
|
||||
null_array=[3, 2, 1])
|
||||
])
|
||||
|
||||
def _assert_sample_data(self, results):
|
||||
dt = date(1970, 1, 1)
|
||||
self.assertEquals(len(results), 4)
|
||||
self.assertIsNone(results[0].null_str)
|
||||
self.assertEquals(results[0].null_int, 42)
|
||||
self.assertEquals(results[0].null_date, dt)
|
||||
self.assertIsNone(results[1].null_date)
|
||||
self.assertEquals(results[1].null_str, 'nothing')
|
||||
self.assertIsNone(results[1].null_date)
|
||||
self.assertIsNone(results[2].null_str)
|
||||
self.assertEquals(results[2].null_date, dt)
|
||||
self.assertEquals(results[2].null_int, 42)
|
||||
self.assertIsNone(results[3].null_int)
|
||||
self.assertIsNone(results[3].null_str)
|
||||
self.assertIsNone(results[3].null_date)
|
||||
|
||||
self.assertIsNone(results[0].null_array)
|
||||
self.assertEquals(results[1].null_array, [1, 2, 3])
|
||||
self.assertEquals(results[2].null_array, [])
|
||||
self.assertEquals(results[3].null_array, [3, 2, 1])
|
||||
|
||||
def test_insert_and_select(self):
|
||||
self._insert_sample_data()
|
||||
query = 'SELECT * from $table ORDER BY date_field'
|
||||
results = list(self.database.select(query, ModelWithNullable))
|
||||
self._assert_sample_data(results)
|
||||
|
||||
def test_ad_hoc_model(self):
|
||||
self._insert_sample_data()
|
||||
query = 'SELECT * from $db.modelwithnullable ORDER BY date_field'
|
||||
results = list(self.database.select(query))
|
||||
self._assert_sample_data(results)
|
||||
|
||||
|
||||
class ModelWithNullable(Model):
|
||||
|
||||
date_field = DateField()
|
||||
null_str = NullableField(StringField(), extra_null_values={''})
|
||||
null_int = NullableField(Int32Field())
|
||||
null_date = NullableField(DateField())
|
||||
null_array = NullableField(ArrayField(Int32Field()))
|
||||
|
||||
engine = MergeTree('date_field', ('date_field',))
|
|
@ -138,6 +138,30 @@ class QuerySetTestCase(TestCaseWithData):
|
|||
self._test_qs(qs.filter(num__in=(1, 2, 3)), 3)
|
||||
self._test_qs(qs.filter(num__in=range(1, 4)), 3)
|
||||
|
||||
def test_slicing(self):
|
||||
db = Database('system')
|
||||
numbers = range(100)
|
||||
qs = Numbers.objects_in(db)
|
||||
self.assertEquals(qs[0].number, numbers[0])
|
||||
self.assertEquals(qs[5].number, numbers[5])
|
||||
self.assertEquals([row.number for row in qs[:1]], numbers[:1])
|
||||
self.assertEquals([row.number for row in qs[:10]], numbers[:10])
|
||||
self.assertEquals([row.number for row in qs[3:10]], numbers[3:10])
|
||||
self.assertEquals([row.number for row in qs[9:10]], numbers[9:10])
|
||||
self.assertEquals([row.number for row in qs[10:10]], numbers[10:10])
|
||||
|
||||
def test_invalid_slicing(self):
|
||||
db = Database('system')
|
||||
qs = Numbers.objects_in(db)
|
||||
with self.assertRaises(AssertionError):
|
||||
qs[3:10:2]
|
||||
with self.assertRaises(AssertionError):
|
||||
qs[-5]
|
||||
with self.assertRaises(AssertionError):
|
||||
qs[:-5]
|
||||
with self.assertRaises(AssertionError):
|
||||
qs[50:1]
|
||||
|
||||
|
||||
Color = Enum('Color', u'red blue green yellow brown white black')
|
||||
|
||||
|
@ -150,3 +174,8 @@ class SampleModel(Model):
|
|||
color = Enum8Field(Color)
|
||||
|
||||
engine = MergeTree('materialized_date', ('materialized_date',))
|
||||
|
||||
|
||||
class Numbers(Model):
|
||||
|
||||
number = UInt64Field()
|
Loading…
Reference in New Issue
Block a user