Infinidat/infi.clickhouse_orm
1
1
Fork 0
mirror of https://github.com/Infinidat/infi.clickhouse_orm.git synced 2025-07-25 15:29:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

Finished Release v0.9.8

Browse Source
This commit is contained in:
Itai Shirav 2017-10-31 11:49:04 +02:00
commit ca8586b213
19 changed files with 509 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@ -1,6 +1,16 @@
Change Log
==========
v0.9.8
------
- Bug fix: add field names list explicitly to Database.insert method (anci)
- Added RunPython and RunSQL migrations (M1hacka)
- Allow ISO-formatted datetime values (tsionyx)
- Show field name in error message when invalid value assigned (tsionyx)
- Bug fix: select query fails when query contains '$' symbol (M1hacka)
- Prevent problems with AlterTable migrations related to field order (M1hacka)
- Added documentation about custom fields.
v0.9.7
------
- Add `distinct` method to querysets

1
buildout.cfg
View File

@ -10,6 +10,7 @@ name = infi.clickhouse_orm
company = Infinidat
namespace_packages = ['infi']
install_requires = [
'iso8601 >= 0.1.12',
'pytz',
'requests',
'setuptools',

160
docs/field_types.md
View File

@ -32,7 +32,7 @@ A `DateTimeField` can be assigned values from one of the following types:
- datetime
- date
- integer - number of seconds since the Unix epoch
- string in `YYYY-MM-DD HH:MM:SS` format
- string in `YYYY-MM-DD HH:MM:SS` format or [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601)-compatible format
The assigned value always gets converted to a timezone-aware `datetime` in UTC. If the assigned value is a timezone-aware `datetime` in another timezone, it will be converted to UTC. Otherwise, the assigned value is assumed to already be in UTC.
@ -48,33 +48,37 @@ Python 3.4 and higher supports Enums natively. When using previous Python versio
Example of a model with an enum field:
Gender = Enum('Gender', 'male female unspecified')
```python
Gender = Enum('Gender', 'male female unspecified')
class Person(models.Model):
class Person(models.Model):
first_name = fields.StringField()
last_name = fields.StringField()
birthday = fields.DateField()
gender = fields.Enum32Field(Gender)
first_name = fields.StringField()
last_name = fields.StringField()
birthday = fields.DateField()
gender = fields.Enum32Field(Gender)
engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday'))
engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday'))
suzy = Person(first_name='Suzy', last_name='Jones', gender=Gender.female)
suzy = Person(first_name='Suzy', last_name='Jones', gender=Gender.female)
```
Working with array fields
-------------------------
You can create array fields containing any data type, for example:
class SensorData(models.Model):
```python
class SensorData(models.Model):
date = fields.DateField()
temperatures = fields.ArrayField(fields.Float32Field())
humidity_levels = fields.ArrayField(fields.UInt8Field())
date = fields.DateField()
temperatures = fields.ArrayField(fields.Float32Field())
humidity_levels = fields.ArrayField(fields.UInt8Field())
engine = engines.MergeTree('date', ('date',))
engine = engines.MergeTree('date', ('date',))
data = SensorData(date=date.today(), temperatures=[25.5, 31.2, 28.7], humidity_levels=[41, 39, 66])
data = SensorData(date=date.today(), temperatures=[25.5, 31.2, 28.7], humidity_levels=[41, 39, 66])
```
Working with materialized and alias fields
------------------------------------------
@ -87,22 +91,24 @@ Both field types can't be inserted into the database directly, so they are ignor
Usage:
class Event(models.Model):
```python
class Event(models.Model):
created = fields.DateTimeField()
created_date = fields.DateTimeField(materialized='toDate(created)')
name = fields.StringField()
username = fields.StringField(alias='name')
created = fields.DateTimeField()
created_date = fields.DateTimeField(materialized='toDate(created)')
name = fields.StringField()
username = fields.StringField(alias='name')
engine = engines.MergeTree('created_date', ('created_date', 'created'))
engine = engines.MergeTree('created_date', ('created_date', 'created'))
obj = Event(created=datetime.now(), name='MyEvent')
db = Database('my_test_db')
db.insert([obj])
# All values will be retrieved from database
db.select('SELECT created, created_date, username, name FROM $db.event', model_class=Event)
# created_date and username will contain a default value
db.select('SELECT * FROM $db.event', model_class=Event)
obj = Event(created=datetime.now(), name='MyEvent')
db = Database('my_test_db')
db.insert([obj])
# All values will be retrieved from database
db.select('SELECT created, created_date, username, name FROM $db.event', model_class=Event)
# created_date and username will contain a default value
db.select('SELECT * FROM $db.event', model_class=Event)
```
Working with nullable fields
----------------------------
@ -111,26 +117,104 @@ Also see some information [here](https://github.com/yandex/ClickHouse/blob/maste
Wrapping another field in a `NullableField` makes it possible to assign `None` to that field. For example:
class EventData(models.Model):
```python
class EventData(models.Model):
date = fields.DateField()
comment = fields.NullableField(fields.StringField(), extra_null_values={''})
score = fields.NullableField(fields.UInt8Field())
serie = fields.NullableField(fields.ArrayField(fields.UInt8Field()))
date = fields.DateField()
comment = fields.NullableField(fields.StringField(), extra_null_values={''})
score = fields.NullableField(fields.UInt8Field())
serie = fields.NullableField(fields.ArrayField(fields.UInt8Field()))
engine = engines.MergeTree('date', ('date',))
engine = engines.MergeTree('date', ('date',))
score_event = EventData(date=date.today(), comment=None, score=5, serie=None)
comment_event = EventData(date=date.today(), comment='Excellent!', score=None, serie=None)
another_event = EventData(date=date.today(), comment='', score=None, serie=None)
action_event = EventData(date=date.today(), comment='', score=None, serie=[1, 2, 3])
score_event = EventData(date=date.today(), comment=None, score=5, serie=None)
comment_event = EventData(date=date.today(), comment='Excellent!', score=None, serie=None)
another_event = EventData(date=date.today(), comment='', score=None, serie=None)
action_event = EventData(date=date.today(), comment='', score=None, serie=[1, 2, 3])
```
The `extra_null_values` parameter is an iterable of additional values that should be converted
to `None`.
NOTE: `ArrayField` of `NullableField` is not supported. Also `EnumField` cannot be nullable.
Creating custom field types
---------------------------
Sometimes it is convenient to use data types that are supported in Python, but have no corresponding column type in ClickHouse. In these cases it is possible to define a custom field class that knows how to convert the Pythonic object to a suitable representation in the database, and vice versa.
For example, we can create a BooleanField which will hold `True` and `False` values, but write them to the database as 0 and 1 (in a `UInt8` column). For this purpose we'll subclass the `Field` class, and implement two methods:
- `to_python` which converts any supported value to a `bool`. The method should know how to handle strings (which typically come from the database), booleans, and possibly other valid options. In case the value is not supported, it should raise a `ValueError`.
- `to_db_string` which converts a `bool` into a string for writing to the database.
Here's the full implementation:
```python
from infi.clickhouse_orm.fields import Field
class BooleanField(Field):
# The ClickHouse column type to use
db_type = 'UInt8'
# The default value
class_default = False
def to_python(self, value, timezone_in_use):
# Convert valid values to bool
if value in (1, '1', True):
return True
elif value in (0, '0', False):
return False
else:
raise ValueError('Invalid value for BooleanField: %r' % value)
def to_db_string(self, value, quote=True):
# The value was already converted by to_python, so it's a bool
return '1' if value else '0'
```
Here's another example - a field for storing UUIDs in the database as 16-byte strings. We'll use Python's built-in `UUID` class to handle the conversion from strings, ints and tuples into UUID instances. So in our Python code we'll have the convenience of working with UUID objects, but they will be stored in the database as efficiently as possible:
```python
from infi.clickhouse_orm.fields import Field
from infi.clickhouse_orm.utils import escape
from uuid import UUID
import six
class UUIDField(Field):
# The ClickHouse column type to use
db_type = 'FixedString(16)'
# The default value if empty
class_default = UUID(int=0)
def to_python(self, value, timezone_in_use):
# Convert valid values to UUID instance
if isinstance(value, UUID):
return value
elif isinstance(value, six.string_types):
return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
elif isinstance(value, six.integer_types):
return UUID(int=value)
elif isinstance(value, tuple):
return UUID(fields=value)
else:
raise ValueError('Invalid value for UUIDField: %r' % value)
def to_db_string(self, value, quote=True):
# The value was already converted by to_python, so it's a UUID instance
val = value.bytes
if six.PY3:
val = str(val, 'latin1')
return escape(val, quote)
```
Note that the latin-1 encoding is used as an identity encoding for converting between raw bytes and strings. This is required in Python 3, where `str` and `bytes` are different types.
---
[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md)

33
docs/schema_migrations.md
View File

@ -32,16 +32,19 @@ Each migration file is expected to contain a list of `operations`, for example:
The following operations are supported:
**CreateTable**
A migration operation that creates a table for a given model class. If the table already exists, the operation does nothing.
In case the model class is a `BufferModel`, the operation first creates the underlying on-disk table, and then creates the buffer table.
**DropTable**
A migration operation that drops the table of a given model class. If the table does not exist, the operation does nothing.
**AlterTable**
A migration operation that compares the table of a given model class to the model’s fields, and alters the table to match the model. The operation can:
@ -52,6 +55,7 @@ A migration operation that compares the table of a given model class to the mode
Default values are not altered by this operation.
**AlterTableWithBuffer**
A compound migration operation for altering a buffer table and its underlying on-disk table. The buffer table is dropped, the on-disk table is altered, and then the buffer table is re-created. This is the procedure recommended in the ClickHouse documentation for handling scenarios in which the underlying table needs to be modified.
@ -59,6 +63,35 @@ A compound migration operation for altering a buffer table and its underlying on
Applying this migration operation to a regular table has the same effect as an `AlterTable` operation.
**RunPython**
A migration operation that runs a Python function. The function receives the `Database` instance to operate on.
def forward(database):
database.insert([
TestModel(field=1)
])
operations = [
migrations.RunPython(forward),
]
**RunSQL**
A migration operation that runs raw SQL queries. It expects a string containing an SQL query, or an array of SQL-query strings.
Example:
operations = [
RunSQL('INSERT INTO `test_table` (field) VALUES (1)'),
RunSQL([
'INSERT INTO `test_table` (field) VALUES (2)',
'INSERT INTO `test_table` (field) VALUES (3)'
])
]
Running Migrations
------------------

1
docs/toc.md
View File

@ -31,6 +31,7 @@
* [Working with array fields](field_types.md#working-with-array-fields)
* [Working with materialized and alias fields](field_types.md#working-with-materialized-and-alias-fields)
* [Working with nullable fields](field_types.md#working-with-nullable-fields)
* [Creating custom field types](field_types.md#creating-custom-field-types)
* [Table Engines](table_engines.md#table-engines)
* [Simple Engines](table_engines.md#simple-engines)

8
src/infi/clickhouse_orm/database.py
View File

@ -106,9 +106,13 @@ class Database(object):
if first_instance.readonly or first_instance.system:
raise DatabaseException("You can't insert into read only and system tables")
fields_list = ','.join(
['`%s`' % name for name, _ in first_instance._writable_fields])
def gen():
buf = BytesIO()
buf.write(self._substitute('INSERT INTO $table FORMAT TabSeparated\n', model_class).encode('utf-8'))
query = 'INSERT INTO $table (%s) FORMAT TabSeparated\n' % fields_list
buf.write(self._substitute(query, model_class).encode('utf-8'))
first_instance.set_database(self)
buf.write(first_instance.to_tsv(include_readonly=False).encode('utf-8'))
buf.write('\n'.encode('utf-8'))
@ -270,7 +274,7 @@ class Database(object):
mapping = dict(db="`%s`" % self.db_name)
if model_class:
mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
query = Template(query).substitute(mapping)
query = Template(query).safe_substitute(mapping)
return query
def _get_server_timezone(self):

5
src/infi/clickhouse_orm/engines.py
View File

@ -146,10 +146,11 @@ class Merge(Engine):
self.table_regex = table_regex
# Use current database as default
self.db_name = 'currentDatabase()'
self.db_name = None
def create_table_sql(self):
return "Merge(%s, '%s')" % (self.db_name, self.table_regex)
db_name = ("`%s`" % self.db_name) if self.db_name else 'currentDatabase()'
return "Merge(%s, '%s')" % (db_name, self.table_regex)
def set_db_name(self, db_name):
assert isinstance(db_name, six.string_types), "'db_name' parameter must be string"

44
src/infi/clickhouse_orm/fields.py
View File

@ -1,6 +1,7 @@
from __future__ import unicode_literals
from six import string_types, text_type, binary_type
import datetime
import iso8601
import pytz
import time
from calendar import timegm
@ -61,19 +62,20 @@ class Field(object):
'''
return escape(value, quote)
def get_sql(self, with_default=True):
def get_sql(self, with_default_expression=True):
'''
Returns an SQL expression describing the field (e.g. for CREATE TABLE).
:param with_default: If True, adds default value to sql.
:param with_default_expression: If True, adds default value to sql.
It doesn't affect fields with alias and materialized values.
'''
if self.alias:
return '%s ALIAS %s' % (self.db_type, self.alias)
elif self.materialized:
return '%s MATERIALIZED %s' % (self.db_type, self.materialized)
elif with_default:
default = self.to_db_string(self.default)
return '%s DEFAULT %s' % (self.db_type, default)
if with_default_expression:
if self.alias:
return '%s ALIAS %s' % (self.db_type, self.alias)
elif self.materialized:
return '%s MATERIALIZED %s' % (self.db_type, self.materialized)
else:
default = self.to_db_string(self.default)
return '%s DEFAULT %s' % (self.db_type, default)
else:
return self.db_type
@ -157,8 +159,16 @@ class DateTimeField(Field):
return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
except ValueError:
pass
dt = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
return timezone_in_use.localize(dt).astimezone(pytz.utc)
try:
# left the date naive in case of no tzinfo set
dt = iso8601.parse_date(value, default_timezone=None)
except iso8601.ParseError as e:
raise ValueError(text_type(e))
# convert naive to aware
if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
dt = timezone_in_use.localize(dt)
return dt.astimezone(pytz.utc)
raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
def to_db_string(self, value, quote=True):
@ -295,10 +305,10 @@ class BaseEnumField(Field):
def to_db_string(self, value, quote=True):
return escape(value.name, quote)
def get_sql(self, with_default=True):
def get_sql(self, with_default_expression=True):
values = ['%s = %d' % (escape(item.name), item.value) for item in self.enum_cls]
sql = '%s(%s)' % (self.db_type, ' ,'.join(values))
if with_default:
if with_default_expression:
default = self.to_db_string(self.default)
sql = '%s DEFAULT %s' % (sql, default)
return sql
@ -357,9 +367,9 @@ class ArrayField(Field):
array = [self.inner_field.to_db_string(v, quote=True) for v in value]
return '[' + comma_join(array) + ']'
def get_sql(self, with_default=True):
def get_sql(self, with_default_expression=True):
from .utils import escape
return 'Array(%s)' % self.inner_field.get_sql(with_default=False)
return 'Array(%s)' % self.inner_field.get_sql(with_default_expression=False)
class NullableField(Field):
@ -387,6 +397,6 @@ class NullableField(Field):
return '\\N'
return self.inner_field.to_db_string(value, quote=quote)
def get_sql(self, with_default=True):
def get_sql(self, with_default_expression=True):
from .utils import escape
return 'Nullable(%s)' % self.inner_field.get_sql(with_default=False)
return 'Nullable(%s)' % self.inner_field.get_sql(with_default_expression=False)

63
src/infi/clickhouse_orm/migrations.py
View File

@ -1,3 +1,5 @@
import six
from .models import Model, BufferModel
from .fields import DateField, StringField
from .engines import MergeTree
@ -57,13 +59,18 @@ class AlterTable(Operation):
def apply(self, database):
logger.info(' Alter table %s', self.model_class.table_name())
# Note that MATERIALIZED and ALIAS fields are always at the end of the DESC,
# ADD COLUMN ... AFTER doesn't affect it
table_fields = dict(self._get_table_fields(database))
# Identify fields that were deleted from the model
deleted_fields = set(table_fields.keys()) - set(name for name, field in self.model_class._fields)
for name in deleted_fields:
logger.info(' Drop column %s', name)
self._alter_table(database, 'DROP COLUMN %s' % name)
del table_fields[name]
# Identify fields that were added to the model
prev_name = None
for name, field in self.model_class._fields:
@ -72,14 +79,25 @@ class AlterTable(Operation):
assert prev_name, 'Cannot add a column to the beginning of the table'
cmd = 'ADD COLUMN %s %s AFTER %s' % (name, field.get_sql(), prev_name)
self._alter_table(database, cmd)
prev_name = name
if not field.materialized and not field.alias:
# ALIAS and MATERIALIZED fields are not stored in the database, and raise DatabaseError
# (no AFTER column). So we will skip them
prev_name = name
# Identify fields whose type was changed
model_fields = [(name, field.get_sql(with_default=False)) for name, field in self.model_class._fields]
for model_field, table_field in zip(model_fields, self._get_table_fields(database)):
assert model_field[0] == table_field[0], 'Model fields and table columns in disagreement'
if model_field[1] != table_field[1]:
logger.info(' Change type of column %s from %s to %s', table_field[0], table_field[1], model_field[1])
self._alter_table(database, 'MODIFY COLUMN %s %s' % model_field)
# The order of class attributes can be changed any time, so we can't count on it
# Secondly, MATERIALIZED and ALIAS fields are always at the end of the DESC, so we can't expect them to save
# attribute position. Watch https://github.com/Infinidat/infi.clickhouse_orm/issues/47
model_fields = {name: field.get_sql(with_default_expression=False) for name, field in self.model_class._fields}
for field_name, field_sql in self._get_table_fields(database):
# All fields must have been created and dropped by this moment
assert field_name in model_fields, 'Model fields and table columns in disagreement'
if field_sql != model_fields[field_name]:
logger.info(' Change type of column %s from %s to %s', field_name, field_sql,
model_fields[field_name])
self._alter_table(database, 'MODIFY COLUMN %s %s' % (field_name, model_fields[field_name]))
class AlterTableWithBuffer(Operation):
@ -114,6 +132,37 @@ class DropTable(Operation):
database.drop_table(self.model_class)
class RunPython(Operation):
'''
A migration operation that executes given python function on database
'''
def __init__(self, func):
assert callable(func), "'func' parameter must be function"
self._func = func
def apply(self, database):
logger.info(' Executing python operation %s', self._func.__name__)
self._func(database)
class RunSQL(Operation):
'''
A migration operation that executes given SQL on database
'''
def __init__(self, sql):
if isinstance(sql, six.string_types):
sql = [sql]
assert isinstance(sql, list), "'sql' parameter must be string or list of strings"
self._sql = sql
def apply(self, database):
logger.info(' Executing raw SQL operations')
for item in self._sql:
database.raw(item)
class MigrationHistory(Model):
'''
A model for storing which migrations were already applied to the containing database.

18
src/infi/clickhouse_orm/models.py
View File

@ -1,7 +1,8 @@
from __future__ import unicode_literals
import sys
from logging import getLogger
from six import with_metaclass
from six import with_metaclass, reraise
import pytz
from .fields import Field, StringField
@ -124,8 +125,13 @@ class Model(with_metaclass(ModelBase)):
'''
field = self.get_field(name)
if field:
value = field.to_python(value, pytz.utc)
field.validate(value)
try:
value = field.to_python(value, pytz.utc)
field.validate(value)
except ValueError:
tp, v, tb = sys.exc_info()
new_msg = "{} (field '{}')".format(v, name)
reraise(tp, tp(new_msg), tb)
super(Model, self).__setattr__(name, value)
def set_database(self, db):
@ -274,3 +280,9 @@ class MergeModel(Model):
res = super(MergeModel, self).set_database(db)
self.engine.set_db_name(db.db_name)
return res
@classmethod
def create_table_sql(cls, db_name):
assert isinstance(cls.engine, Merge), "engine must be engines.Merge instance"
cls.engine.set_db_name(db_name)
return super(MergeModel, cls).create_table_sql(db_name)

9
tests/sample_migrations/0012.py Normal file
View File

@ -0,0 +1,9 @@
from infi.clickhouse_orm import migrations
operations = [
migrations.RunSQL("INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-01', 1, 1, 'test') "),
migrations.RunSQL([
"INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-02', 2, 2, 'test2') ",
"INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-03', 3, 3, 'test3') ",
])
]

15
tests/sample_migrations/0013.py Normal file
View File

@ -0,0 +1,15 @@
import datetime
from infi.clickhouse_orm import migrations
from test_migrations import Model3
def forward(database):
database.insert([
Model3(date=datetime.date(2016, 1, 4), f1=4, f3=1, f4='test4')
])
operations = [
migrations.RunPython(forward)
]

7
tests/sample_migrations/0014.py Normal file
View File

@ -0,0 +1,7 @@
from infi.clickhouse_orm import migrations
from ..test_migrations import *
operations = [
migrations.AlterTable(MaterializedModel1),
migrations.AlterTable(AliasModel1)
]

119
tests/test_custom_fields.py Normal file
View File

@ -0,0 +1,119 @@
from __future__ import unicode_literals
import unittest
import six
from uuid import UUID
from infi.clickhouse_orm.database import Database
from infi.clickhouse_orm.fields import Field, Int16Field
from infi.clickhouse_orm.models import Model
from infi.clickhouse_orm.engines import Memory
from infi.clickhouse_orm.utils import escape
class CustomFieldsTest(unittest.TestCase):
def setUp(self):
self.database = Database('test-db')
def tearDown(self):
self.database.drop_database()
def test_boolean_field(self):
# Create a model
class TestModel(Model):
i = Int16Field()
f = BooleanField()
engine = Memory()
self.database.create_table(TestModel)
# Check valid values
for index, value in enumerate([1, '1', True, 0, '0', False]):
rec = TestModel(i=index, f=value)
self.database.insert([rec])
self.assertEquals([rec.f for rec in TestModel.objects_in(self.database).order_by('i')],
[True, True, True, False, False, False])
# Check invalid values
for value in [None, 'zzz', -5, 7]:
with self.assertRaises(ValueError):
TestModel(i=1, f=value)
def test_uuid_field(self):
# Create a model
class TestModel(Model):
i = Int16Field()
f = UUIDField()
engine = Memory()
self.database.create_table(TestModel)
# Check valid values (all values are the same UUID)
values = [
'{12345678-1234-5678-1234-567812345678}',
'12345678123456781234567812345678',
'urn:uuid:12345678-1234-5678-1234-567812345678',
'\x12\x34\x56\x78'*4,
(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
0x12345678123456781234567812345678,
]
for index, value in enumerate(values):
rec = TestModel(i=index, f=value)
self.database.insert([rec])
for rec in TestModel.objects_in(self.database):
self.assertEquals(rec.f, UUID(values[0]))
# Check that ClickHouse encoding functions are supported
for rec in self.database.select("SELECT i, UUIDNumToString(f) AS f FROM testmodel", TestModel):
self.assertEquals(rec.f, UUID(values[0]))
for rec in self.database.select("SELECT 1 as i, UUIDStringToNum('12345678-1234-5678-1234-567812345678') AS f", TestModel):
self.assertEquals(rec.f, UUID(values[0]))
# Check invalid values
for value in [None, 'zzz', -1, '123']:
with self.assertRaises(ValueError):
TestModel(i=1, f=value)
class BooleanField(Field):
# The ClickHouse column type to use
db_type = 'UInt8'
# The default value if empty
class_default = False
def to_python(self, value, timezone_in_use):
# Convert valid values to bool
if value in (1, '1', True):
return True
elif value in (0, '0', False):
return False
else:
raise ValueError('Invalid value for BooleanField: %r' % value)
def to_db_string(self, value, quote=True):
# The value was already converted by to_python, so it's a bool
return '1' if value else '0'
class UUIDField(Field):
# The ClickHouse column type to use
db_type = 'FixedString(16)'
# The default value if empty
class_default = UUID(int=0)
def to_python(self, value, timezone_in_use):
# Convert valid values to UUID instance
if isinstance(value, UUID):
return value
elif isinstance(value, six.string_types):
return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
elif isinstance(value, six.integer_types):
return UUID(int=value)
elif isinstance(value, tuple):
return UUID(fields=value)
else:
raise ValueError('Invalid value for UUIDField: %r' % value)
def to_db_string(self, value, quote=True):
# The value was already converted by to_python, so it's a UUID instance
val = value.bytes
if six.PY3:
val = str(val, 'latin1')
return escape(val, quote)

4
tests/test_database.py
View File

@ -39,6 +39,10 @@ class DatabaseTestCase(TestCaseWithData):
self.assertEqual(results[0].get_database(), self.database)
self.assertEqual(results[1].get_database(), self.database)
def test_dollar_in_select(self):
query = "SELECT * FROM $table WHERE first_name = '$utm_source'"
list(self.database.select(query, Person))
def test_select_partial_fields(self):
self._insert_and_check(self._sample_data(), len(data))
query = "SELECT first_name, last_name FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name"

1
tests/test_join.py
View File

@ -24,7 +24,6 @@ class JoinTest(unittest.TestCase):
self.print_res("SELECT * FROM {}".format(Bar.table_name()))
self.print_res("SELECT b FROM {} ALL LEFT JOIN {} USING id".format(Foo.table_name(), Bar.table_name()))
@unittest.skip('ClickHouse issue - https://github.com/yandex/ClickHouse/issues/635')
def test_with_db_name(self):
self.print_res("SELECT * FROM $db.{}".format(Foo.table_name()))
self.print_res("SELECT * FROM $db.{}".format(Bar.table_name()))

42
tests/test_migrations.py
View File

@ -80,6 +80,24 @@ class MigrationsTestCase(unittest.TestCase):
self.assertEquals(self.getTableFields(Model4), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')])
self.assertEquals(self.getTableFields(Model4Buffer), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')])
self.database.migrate('tests.sample_migrations', 12)
self.assertEqual(self.database.count(Model3), 3)
data = [item.f1 for item in self.database.select('SELECT f1 FROM $table ORDER BY f1', model_class=Model3)]
self.assertListEqual(data, [1, 2, 3])
self.database.migrate('tests.sample_migrations', 13)
self.assertEqual(self.database.count(Model3), 4)
data = [item.f1 for item in self.database.select('SELECT f1 FROM $table ORDER BY f1', model_class=Model3)]
self.assertListEqual(data, [1, 2, 3, 4])
self.database.migrate('tests.sample_migrations', 14)
self.assertTrue(self.tableExists(MaterializedModel1))
self.assertEquals(self.getTableFields(MaterializedModel1),
[('date_time', "DateTime"), ('int_field', 'Int8'), ('date', 'Date')])
self.assertTrue(self.tableExists(AliasModel1))
self.assertEquals(self.getTableFields(AliasModel1),
[('date', 'Date'), ('int_field', 'Int8'), ('date_alias', "Date")])
# Several different models with the same table name, to simulate a table that changes over time
@ -160,6 +178,18 @@ class MaterializedModel(Model):
return 'materalized_date'
class MaterializedModel1(Model):
date_time = DateTimeField()
date = DateField(materialized='toDate(date_time)')
int_field = Int8Field()
engine = MergeTree('date', ('date',))
@classmethod
def table_name(cls):
return 'materalized_date'
class AliasModel(Model):
date = DateField()
date_alias = DateField(alias='date')
@ -171,6 +201,18 @@ class AliasModel(Model):
return 'alias_date'
class AliasModel1(Model):
date = DateField()
date_alias = DateField(alias='date')
int_field = Int8Field()
engine = MergeTree('date', ('date',))
@classmethod
def table_name(cls):
return 'alias_date'
class Model4(Model):
date = DateField()

21
tests/test_models.py
View File

@ -79,6 +79,27 @@ class ModelTestCase(unittest.TestCase):
"datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)
})
def test_field_name_in_error_message_for_invalid_value_in_constructor(self):
bad_value = 1
with self.assertRaises(ValueError) as cm:
SimpleModel(str_field=bad_value)
self.assertEqual(
"Invalid value for StringField: {} (field 'str_field')".format(repr(bad_value)),
text_type(cm.exception)
)
def test_field_name_in_error_message_for_invalid_value_in_assignment(self):
instance = SimpleModel()
bad_value = 'foo'
with self.assertRaises(ValueError) as cm:
instance.float_field = bad_value
self.assertEqual(
"Invalid value for Float32Field - {} (field 'float_field')".format(repr(bad_value)),
text_type(cm.exception)
)
class SimpleModel(Model):

20
tests/test_simple_fields.py
View File

@ -13,14 +13,17 @@ class SimpleFieldsTest(unittest.TestCase):
# Valid values
for value in (date(1970, 1, 1), datetime(1970, 1, 1), epoch,
epoch.astimezone(pytz.timezone('US/Eastern')), epoch.astimezone(pytz.timezone('Asia/Jerusalem')),
'1970-01-01 00:00:00', '1970-01-17 00:00:17', '0000-00-00 00:00:00', 0):
'1970-01-01 00:00:00', '1970-01-17 00:00:17', '0000-00-00 00:00:00', 0,
'2017-07-26T08:31:05', '2017-07-26T08:31:05Z', '2017-07-26 08:31',
'2017-07-26T13:31:05+05', '2017-07-26 13:31:05+0500'):
dt = f.to_python(value, pytz.utc)
self.assertEquals(dt.tzinfo, pytz.utc)
# Verify that conversion to and from db string does not change value
dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc)
self.assertEquals(dt, dt2)
# Invalid values
for value in ('nope', '21/7/1999', 0.5):
for value in ('nope', '21/7/1999', 0.5,
'2017-01 15:06:00', '2017-01-01X15:06:00', '2017-13-01T15:06:00'):
with self.assertRaises(ValueError):
f.to_python(value, pytz.utc)
@ -49,6 +52,19 @@ class SimpleFieldsTest(unittest.TestCase):
dt = datetime(2017, 10, 5, tzinfo=pytz.timezone('Asia/Jerusalem'))
self.assertEquals(f.to_python(dt, pytz.utc), date(2017, 10, 4))
def test_datetime_field_timezone(self):
# Verify that conversion of timezone-aware datetime is correct
f = DateTimeField()
utc_value = datetime(2017, 7, 26, 8, 31, 5, tzinfo=pytz.UTC)
for value in (
'2017-07-26T08:31:05',
'2017-07-26T08:31:05Z',
'2017-07-26T11:31:05+03',
'2017-07-26 11:31:05+0300',
'2017-07-26T03:31:05-0500',
):
self.assertEquals(f.to_python(value, pytz.utc), utc_value)
def test_uint8_field(self):
f = UInt8Field()
# Valid values