Infinidat/infi.clickhouse_orm
1
1
Fork 0
mirror of https://github.com/Infinidat/infi.clickhouse_orm.git synced 2024-11-22 00:56:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/kpotehin/infi.clickhouse_orm into kpotehin-develop

Browse Source
This commit is contained in:
Itai Shirav 2019-06-13 05:08:59 +03:00
commit 4561159218
4 changed files with 67 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
docs/field_types.md
View File

@ -25,6 +25,7 @@ Currently the following field types are supported:
| Decimal32Field | Decimal32 | Decimal | Ditto
| Decimal64Field | Decimal64 | Decimal | Ditto
| Decimal128Field | Decimal128 | Decimal | Ditto
| UUIDField | UUID | Decimal |
| Enum8Field | Enum8 | Enum | See below
| Enum16Field | Enum16 | Enum | See below
| ArrayField | Array | list | See below
@ -183,46 +184,6 @@ class BooleanField(Field):
return '1' if value else '0'
```
Here's another example - a field for storing UUIDs in the database as 16-byte strings. We'll use Python's built-in `UUID` class to handle the conversion from strings, ints and tuples into UUID instances. So in our Python code we'll have the convenience of working with UUID objects, but they will be stored in the database as efficiently as possible:
```python
from infi.clickhouse_orm.fields import Field
from infi.clickhouse_orm.utils import escape
from uuid import UUID
import six
class UUIDField(Field):
# The ClickHouse column type to use
db_type = 'FixedString(16)'
# The default value if empty
class_default = UUID(int=0)
def to_python(self, value, timezone_in_use):
# Convert valid values to UUID instance
if isinstance(value, UUID):
return value
elif isinstance(value, six.string_types):
return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
elif isinstance(value, six.integer_types):
return UUID(int=value)
elif isinstance(value, tuple):
return UUID(fields=value)
else:
raise ValueError('Invalid value for UUIDField: %r' % value)
def to_db_string(self, value, quote=True):
# The value was already converted by to_python, so it's a UUID instance
val = value.bytes
if six.PY3:
val = str(val, 'latin1')
return escape(val, quote)
```
Note that the latin-1 encoding is used as an identity encoding for converting between raw bytes and strings. This is required in Python 3, where `str` and `bytes` are different types.
---
[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md)

23
src/infi/clickhouse_orm/fields.py
View File

@ -1,11 +1,12 @@
from __future__ import unicode_literals
from six import string_types, text_type, binary_type
from six import string_types, text_type, binary_type, integer_types
import datetime
import iso8601
import pytz
import time
from calendar import timegm
from decimal import Decimal, localcontext
from uuid import UUID
from .utils import escape, parse_array, comma_join
@ -452,6 +453,26 @@ class ArrayField(Field):
return 'Array(%s)' % self.inner_field.get_sql(with_default_expression=False)
class UUIDField(Field):
class_default = UUID(int=0)
db_type = 'UUID'
def to_python(self, value, timezone_in_use):
if isinstance(value, UUID):
return value
elif isinstance(value, string_types):
return UUID(bytes=value) if len(value) == 16 else UUID(value)
elif isinstance(value, integer_types):
return UUID(int=value)
elif isinstance(value, tuple):
return UUID(fields=value)
else:
raise ValueError('Invalid value for UUIDField: %r' % value)
def to_db_string(self, value, quote=True):
return escape(str(value), quote)
class NullableField(Field):
class_default = None

63
tests/test_custom_fields.py
View File

@ -1,12 +1,9 @@
from __future__ import unicode_literals
import unittest
import six
from uuid import UUID
from infi.clickhouse_orm.database import Database
from infi.clickhouse_orm.fields import Field, Int16Field
from infi.clickhouse_orm.models import Model
from infi.clickhouse_orm.engines import Memory
from infi.clickhouse_orm.utils import escape
class CustomFieldsTest(unittest.TestCase):
@ -35,37 +32,6 @@ class CustomFieldsTest(unittest.TestCase):
with self.assertRaises(ValueError):
TestModel(i=1, f=value)
def test_uuid_field(self):
# Create a model
class TestModel(Model):
i = Int16Field()
f = UUIDField()
engine = Memory()
self.database.create_table(TestModel)
# Check valid values (all values are the same UUID)
values = [
'{12345678-1234-5678-1234-567812345678}',
'12345678123456781234567812345678',
'urn:uuid:12345678-1234-5678-1234-567812345678',
'\x12\x34\x56\x78'*4,
(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
0x12345678123456781234567812345678,
]
for index, value in enumerate(values):
rec = TestModel(i=index, f=value)
self.database.insert([rec])
for rec in TestModel.objects_in(self.database):
self.assertEqual(rec.f, UUID(values[0]))
# Check that ClickHouse encoding functions are supported
for rec in self.database.select("SELECT i, UUIDNumToString(f) AS f FROM testmodel", TestModel):
self.assertEqual(rec.f, UUID(values[0]))
for rec in self.database.select("SELECT 1 as i, UUIDStringToNum('12345678-1234-5678-1234-567812345678') AS f", TestModel):
self.assertEqual(rec.f, UUID(values[0]))
# Check invalid values
for value in [None, 'zzz', -1, '123']:
with self.assertRaises(ValueError):
TestModel(i=1, f=value)
class BooleanField(Field):
@ -88,32 +54,3 @@ class BooleanField(Field):
# The value was already converted by to_python, so it's a bool
return '1' if value else '0'
class UUIDField(Field):
# The ClickHouse column type to use
db_type = 'FixedString(16)'
# The default value if empty
class_default = UUID(int=0)
def to_python(self, value, timezone_in_use):
# Convert valid values to UUID instance
if isinstance(value, UUID):
return value
elif isinstance(value, six.string_types):
return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
elif isinstance(value, six.integer_types):
return UUID(int=value)
elif isinstance(value, tuple):
return UUID(fields=value)
else:
raise ValueError('Invalid value for UUIDField: %r' % value)
def to_db_string(self, value, quote=True):
# The value was already converted by to_python, so it's a UUID instance
val = value.bytes
if six.PY3:
val = str(val, 'latin1')
return escape(val, quote)

44
tests/test_uuid_fields.py Normal file
View File

@ -0,0 +1,44 @@
from __future__ import unicode_literals
import unittest
from uuid import UUID
from infi.clickhouse_orm.database import Database
from infi.clickhouse_orm.fields import Int16Field, UUIDField
from infi.clickhouse_orm.models import Model
from infi.clickhouse_orm.engines import Memory
class UUIDFieldsTest(unittest.TestCase):
def setUp(self):
self.database = Database('test-db')
def tearDown(self):
self.database.drop_database()
def test_uuid_field(self):
# Create a model
class TestModel(Model):
i = Int16Field()
f = UUIDField()
engine = Memory()
self.database.create_table(TestModel)
# Check valid values (all values are the same UUID)
values = [
'12345678-1234-5678-1234-567812345678',
'{12345678-1234-5678-1234-567812345678}',
'12345678123456781234567812345678',
'urn:uuid:12345678-1234-5678-1234-567812345678',
'\x12\x34\x56\x78'*4,
(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
0x12345678123456781234567812345678,
]
for index, value in enumerate(values):
rec = TestModel(i=index, f=value)
self.database.insert([rec])
for rec in TestModel.objects_in(self.database):
self.assertEqual(rec.f, UUID(values[0]))
# Check invalid values
for value in [None, 'zzz', -1, '123']:
with self.assertRaises(ValueError):
TestModel(i=1, f=value)