Infinidat/infi.clickhouse_orm
1
1
Fork 0
mirror of https://github.com/Infinidat/infi.clickhouse_orm.git synced 2024-11-25 02:03:46 +03:00
Code Issues Packages Projects Releases Wiki Activity

DateTime64 field

Browse Source 
closes #145
This commit is contained in:
Niyaz Batyrshin 2020-06-07 12:50:45 +03:00
parent 4616f8cb0e
commit ed51ad5be6
8 changed files with 77 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/class_reference.md
View File

@ -635,6 +635,13 @@ Extends Field
#### DateTimeField(default=None, alias=None, materialized=None, readonly=None, codec=None)
### DateTime64Field
Extends DateTimeField
#### DateTime64Field(default=None, alias=None, materialized=None, readonly=None, codec=None, precision=6, timezone=None)
### Decimal128Field
Extends DecimalField

1
docs/field_types.md
View File

@ -11,6 +11,7 @@ The following field types are supported:
| FixedStringField | FixedString| str | Encoded as UTF-8 when written to ClickHouse
| DateField | Date | datetime.date | Range 1970-01-01 to 2105-12-31
| DateTimeField | DateTime | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Always in UTC
| DateTime64Field | DateTime64 | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Always in UTC
| Int8Field | Int8 | int | Range -128 to 127
| Int16Field | Int16 | int | Range -32768 to 32767
| Int32Field | Int32 | int | Range -2147483648 to 2147483647

1
docs/toc.md
View File

@ -82,6 +82,7 @@
* [BaseIntField](class_reference.md#baseintfield)
* [DateField](class_reference.md#datefield)
* [DateTimeField](class_reference.md#datetimefield)
* [DateTime64Field](class_reference.md#datetime64field)
* [Decimal128Field](class_reference.md#decimal128field)
* [Decimal32Field](class_reference.md#decimal32field)
* [Decimal64Field](class_reference.md#decimal64field)

42
src/infi/clickhouse_orm/fields.py
View File

@ -1,11 +1,14 @@
from __future__ import unicode_literals
import datetime
from typing import List
import iso8601
import pytz
from calendar import timegm
from decimal import Decimal, localcontext
from uuid import UUID
from logging import getLogger
from pytz import UnknownTimeZoneError
from .utils import escape, parse_array, comma_join, string_or_func, get_subclass_names
from .funcs import F, FunctionOperatorsMixin
from ipaddress import IPv4Address, IPv6Address
@ -86,10 +89,17 @@ class Field(FunctionOperatorsMixin):
- `db`: Database, used for checking supported features.
'''
sql = self.db_type
args = self.get_db_type_args()
if args:
sql += '(%s)' % ', '.join(args)
if with_default_expression:
sql += self._extra_params(db)
return sql
def get_db_type_args(self) -> List[str]:
"""Returns field type arguments"""
return []
def _extra_params(self, db):
sql = ''
if self.alias:
@ -219,6 +229,38 @@ class DateTimeField(Field):
return escape('%010d' % timegm(value.utctimetuple()), quote)
class DateTime64Field(DateTimeField):
db_type = 'DateTime64'
def __init__(self, default=None, alias=None, materialized=None, readonly=None, codec=None,
precision: int = 6, timezone: str = None):
super().__init__(default, alias, materialized, readonly, codec)
assert precision is None or isinstance(precision, int), 'Precision must be int type'
assert timezone is None or isinstance(timezone, str), 'Timezone must be string type'
if timezone:
try:
pytz.timezone(timezone)
except UnknownTimeZoneError:
raise Exception('Timezone must be a valid IANA timezone identifier')
self.precision = precision
self.timezone = timezone
def get_db_type_args(self) -> List[str]:
args = [str(self.precision)]
if self.timezone:
args.append(escape(self.timezone))
return args
def to_db_string(self, value: datetime.datetime, quote=True):
"""
Returns the field's value prepared for writing to the database
Returns string in 0000000000.000000 format, where remainder digits count is equal to precision
"""
width = 11 + self.precision
return escape(f'{value.timestamp():0{width}.{self.precision}f}', quote)
class BaseIntField(Field):
'''
Abstract base class for all integer-type fields.

5
src/infi/clickhouse_orm/funcs.py
View File

@ -767,6 +767,11 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta):
def toDateTime(x):
return F('toDateTime', x)
@staticmethod
@type_conversion
def toDateTime64(x, precision, timezone=NO_VALUE):
return F('toDateTime64', x, precision, timezone)
@staticmethod
def toString(x):
return F('toString', x)

9
src/infi/clickhouse_orm/models.py
View File

@ -6,7 +6,7 @@ from logging import getLogger
import pytz
from .fields import Field, StringField
from .utils import parse_tsv, NO_VALUE, get_subclass_names
from .utils import parse_tsv, NO_VALUE, get_subclass_names, unescape
from .query import QuerySet
from .funcs import F
from .engines import Merge, Distributed
@ -89,6 +89,13 @@ class ModelBase(type):
if db_type.startswith('DateTime('):
# Some functions return DateTimeField with timezone in brackets
return orm_fields.DateTimeField()
# DateTime with timezone
if db_type.startswith('DateTime64('):
precision, *timezone = [s.strip() for s in db_type[11:-1].split(',')]
return orm_fields.DateTime64Field(
precision=int(precision),
timezone=timezone[0][1:-1] if timezone else None
)
# Arrays
if db_type.startswith('Array'):
inner_field = cls.create_ad_hoc_field(db_type[6 : -1])

14
tests/test_datetime_fields.py
View File

@ -20,8 +20,14 @@ class DateFieldsTest(unittest.TestCase):
def test_ad_hoc_model(self):
self.database.insert([
ModelWithDate(date_field='2016-08-30', datetime_field='2016-08-30 03:50:00'),
ModelWithDate(date_field='2016-08-31', datetime_field='2016-08-31 01:30:00')
ModelWithDate(
date_field='2016-08-30',
datetime_field='2016-08-30 03:50:00',
datetime64_field='2016-08-30 03:50:00.001'),
ModelWithDate(
date_field='2016-08-31',
datetime_field='2016-08-31 01:30:00',
datetime64_field='2016-08-31 01:30:00.002')
])
# toStartOfHour returns DateTime('Asia/Yekaterinburg') in my case, so I test it here to
@ -30,15 +36,17 @@ class DateFieldsTest(unittest.TestCase):
self.assertEqual(len(results), 2)
self.assertEqual(results[0].date_field, datetime.date(2016, 8, 30))
self.assertEqual(results[0].datetime_field, datetime.datetime(2016, 8, 30, 3, 50, 0, tzinfo=pytz.UTC))
self.assertEqual(results[0].datetime64_field, datetime.datetime(2016, 8, 30, 3, 50, 0, 1000, tzinfo=pytz.UTC))
self.assertEqual(results[0].hour_start, datetime.datetime(2016, 8, 30, 3, 0, 0, tzinfo=pytz.UTC))
self.assertEqual(results[1].date_field, datetime.date(2016, 8, 31))
self.assertEqual(results[1].datetime_field, datetime.datetime(2016, 8, 31, 1, 30, 0, tzinfo=pytz.UTC))
self.assertEqual(results[1].datetime64_field, datetime.datetime(2016, 8, 31, 1, 30, 0, 2000, tzinfo=pytz.UTC))
self.assertEqual(results[1].hour_start, datetime.datetime(2016, 8, 31, 1, 0, 0, tzinfo=pytz.UTC))
class ModelWithDate(Model):
date_field = DateField()
datetime_field = DateTimeField()
datetime64_field = DateTime64Field()
engine = MergeTree('date_field', ('date_field',))

1
tests/test_funcs.py
View File

@ -351,6 +351,7 @@ class FuncsTestCase(TestCaseWithData):
if self.database.server_timezone != pytz.utc:
raise unittest.SkipTest('This test must run with UTC as the server timezone')
self._test_func(F.toDateTime('2018-12-31 11:22:33'), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toDateTime64('2018-12-31 11:22:33.001', 6), datetime(2018, 12, 31, 11, 22, 33, 1000, tzinfo=pytz.utc))
self._test_func(F.parseDateTimeBestEffort('31/12/2019 10:05AM'), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc))
self._test_func(F.parseDateTimeBestEffortOrNull('31/12/2019 10:05AM'), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc))
self._test_func(F.parseDateTimeBestEffortOrZero('31/12/2019 10:05AM'), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc))