Always keep datetime fields in UTC internally, and convert server timezone to UTC when parsing query results.

This commit is contained in:
Itai Shirav 2017-02-07 15:25:16 +02:00
parent a73a69ef52
commit f29d737f29
4 changed files with 91 additions and 21 deletions

View File

@ -4,9 +4,12 @@ from .models import ModelBase
from .utils import escape, parse_tsv, import_submodules
from math import ceil
import datetime
import logging
from string import Template
from six import PY3, string_types
import pytz
import logging
logger = logging.getLogger('clickhouse_orm')
Page = namedtuple('Page', 'objects number_of_objects pages_total number page_size')
@ -26,6 +29,7 @@ class Database(object):
self.readonly = readonly
if not self.readonly:
self.create_database()
self.server_timezone = self._get_server_timezone()
def create_database(self):
self._send('CREATE DATABASE IF NOT EXISTS `%s`' % self.db_name)
@ -82,7 +86,7 @@ class Database(object):
field_types = parse_tsv(next(lines))
model_class = model_class or ModelBase.create_ad_hoc_model(zip(field_names, field_types))
for line in lines:
yield model_class.from_tsv(line, field_names)
yield model_class.from_tsv(line, field_names, self.server_timezone)
def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None):
count = self.count(model_class, conditions)
@ -154,3 +158,11 @@ class Database(object):
mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
query = Template(query).substitute(mapping)
return query
def _get_server_timezone(self):
try:
r = self._send('SELECT timezone()')
return pytz.timezone(r.text.strip())
except DatabaseException:
logger.exception('Cannot determine server timezone, assuming UTC')
return pytz.utc

View File

@ -2,6 +2,7 @@ from six import string_types, text_type, binary_type
import datetime
import pytz
import time
from calendar import timegm
from .utils import escape, parse_array
@ -24,10 +25,11 @@ class Field(object):
self.alias = alias
self.materialized = materialized
def to_python(self, value):
def to_python(self, value, timezone_in_use):
'''
Converts the input value into the expected Python data type, raising ValueError if the
data can't be converted. Returns the converted value. Subclasses should override this.
The timezone_in_use parameter should be consulted when parsing datetime fields.
'''
return value
@ -77,7 +79,7 @@ class StringField(Field):
class_default = ''
db_type = 'String'
def to_python(self, value):
def to_python(self, value, timezone_in_use):
if isinstance(value, text_type):
return value
if isinstance(value, binary_type):
@ -92,11 +94,11 @@ class DateField(Field):
class_default = min_value
db_type = 'Date'
def to_python(self, value):
if isinstance(value, datetime.date):
return value
def to_python(self, value, timezone_in_use):
if isinstance(value, datetime.datetime):
return value.date()
if isinstance(value, datetime.date):
return value
if isinstance(value, int):
return DateField.class_default + datetime.timedelta(days=value)
if isinstance(value, string_types):
@ -117,26 +119,27 @@ class DateTimeField(Field):
class_default = datetime.datetime.fromtimestamp(0, pytz.utc)
db_type = 'DateTime'
def to_python(self, value):
def to_python(self, value, timezone_in_use):
if isinstance(value, datetime.datetime):
return value
return value.astimezone(pytz.utc) if value.tzinfo else value.replace(tzinfo=pytz.utc)
if isinstance(value, datetime.date):
return datetime.datetime(value.year, value.month, value.day)
return datetime.datetime(value.year, value.month, value.day, tzinfo=pytz.utc)
if isinstance(value, int):
return datetime.datetime.fromtimestamp(value, pytz.utc)
return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
if isinstance(value, string_types):
if value == '0000-00-00 00:00:00':
return self.class_default
return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
dt = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
return timezone_in_use.localize(dt).astimezone(pytz.utc)
raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
def to_db_string(self, value, quote=True):
return escape(int(time.mktime(value.timetuple())), quote)
return escape(timegm(value.utctimetuple()), quote)
class BaseIntField(Field):
def to_python(self, value):
def to_python(self, value, timezone_in_use):
try:
return int(value)
except:
@ -204,7 +207,7 @@ class Int64Field(BaseIntField):
class BaseFloatField(Field):
def to_python(self, value):
def to_python(self, value, timezone_in_use):
try:
return float(value)
except:
@ -229,7 +232,7 @@ class BaseEnumField(Field):
default = list(enum_cls)[0]
super(BaseEnumField, self).__init__(default, alias, materialized)
def to_python(self, value):
def to_python(self, value, timezone_in_use):
if isinstance(value, self.enum_cls):
return value
try:
@ -291,14 +294,14 @@ class ArrayField(Field):
self.inner_field = inner_field
super(ArrayField, self).__init__(default, alias, materialized)
def to_python(self, value):
def to_python(self, value, timezone_in_use):
if isinstance(value, text_type):
value = parse_array(value)
elif isinstance(value, binary_type):
value = parse_array(value.decode('UTF-8'))
elif not isinstance(value, (list, tuple)):
raise ValueError('ArrayField expects list or tuple, not %s' % type(value))
return [self.inner_field.to_python(v) for v in value]
return [self.inner_field.to_python(v, timezone_in_use) for v in value]
def validate(self, value):
for v in value:

View File

@ -3,6 +3,7 @@ from .engines import *
from .fields import Field
from six import with_metaclass
import pytz
from logging import getLogger
logger = getLogger('clickhouse_orm')
@ -96,7 +97,7 @@ class Model(with_metaclass(ModelBase)):
'''
field = self.get_field(name)
if field:
value = field.to_python(value)
value = field.to_python(value, pytz.utc)
field.validate(value)
super(Model, self).__setattr__(name, value)
@ -136,7 +137,7 @@ class Model(with_metaclass(ModelBase)):
return 'DROP TABLE IF EXISTS `%s`.`%s`' % (db_name, cls.table_name())
@classmethod
def from_tsv(cls, line, field_names=None):
def from_tsv(cls, line, field_names=None, timezone_in_use=pytz.utc):
'''
Create a model instance from a tab-separated line. The line may or may not include a newline.
The field_names list must match the fields defined in the model, but does not have to include all of them.
@ -147,7 +148,8 @@ class Model(with_metaclass(ModelBase)):
values = iter(parse_tsv(line))
kwargs = {}
for name in field_names:
kwargs[name] = next(values)
field = getattr(cls, name)
kwargs[name] = field.to_python(next(values), timezone_in_use)
return cls(**kwargs)
def to_tsv(self, insertable_only=False):

View File

@ -0,0 +1,53 @@
import unittest
from infi.clickhouse_orm.fields import *
from datetime import date, datetime
import pytz
class SimpleFieldsTest(unittest.TestCase):
def test_date_field(self):
f = DateField()
# Valid values
for value in (date(1970, 1, 1), datetime(1970, 1, 1), '1970-01-01', '0000-00-00', 0):
self.assertEquals(f.to_python(value, pytz.utc), date(1970, 1, 1))
# Invalid values
for value in ('nope', '21/7/1999', 0.5):
with self.assertRaises(ValueError):
f.to_python(value, pytz.utc)
# Range check
for value in (date(1900, 1, 1), date(2900, 1, 1)):
with self.assertRaises(ValueError):
f.validate(value)
def test_datetime_field(self):
f = DateTimeField()
epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
# Valid values
for value in (date(1970, 1, 1), datetime(1970, 1, 1), epoch,
epoch.astimezone(pytz.timezone('US/Eastern')), epoch.astimezone(pytz.timezone('Asia/Jerusalem')),
'1970-01-01 00:00:00', '0000-00-00 00:00:00', 0):
dt = f.to_python(value, pytz.utc)
self.assertEquals(dt.tzinfo, pytz.utc)
self.assertEquals(dt, epoch)
# Verify that conversion to and from db string does not change value
dt2 = f.to_python(int(f.to_db_string(dt)), pytz.utc)
self.assertEquals(dt, dt2)
# Invalid values
for value in ('nope', '21/7/1999', 0.5):
with self.assertRaises(ValueError):
f.to_python(value, pytz.utc)
def test_uint8_field(self):
f = UInt8Field()
# Valid values
for value in (17, '17', 17.0):
self.assertEquals(f.to_python(value, pytz.utc), 17)
# Invalid values
for value in ('nope', date.today()):
with self.assertRaises(ValueError):
f.to_python(value, pytz.utc)
# Range check
for value in (-1, 1000):
with self.assertRaises(ValueError):
f.validate(value)