Always keep datetime fields in UTC internally, and convert server timezone to UTC when parsing query results.

src/infi/clickhouse_orm/database.py

@@ -4,9 +4,12 @@ from .models import ModelBase
 from .utils import escape, parse_tsv, import_submodules
 from math import ceil
 import datetime
-import logging
 from string import Template
 from six import PY3, string_types
+import pytz
+
+import logging
+logger = logging.getLogger('clickhouse_orm')
 
 
 Page = namedtuple('Page', 'objects number_of_objects pages_total number page_size')
@@ -26,6 +29,7 @@ class Database(object):
         self.readonly = readonly
         if not self.readonly:
             self.create_database()
+        self.server_timezone = self._get_server_timezone()
 
     def create_database(self):
         self._send('CREATE DATABASE IF NOT EXISTS `%s`' % self.db_name)
@@ -82,7 +86,7 @@ class Database(object):
         field_types = parse_tsv(next(lines))
         model_class = model_class or ModelBase.create_ad_hoc_model(zip(field_names, field_types))
         for line in lines:
-            yield model_class.from_tsv(line, field_names)
+            yield model_class.from_tsv(line, field_names, self.server_timezone)
 
     def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None):
         count = self.count(model_class, conditions)
@@ -154,3 +158,11 @@ class Database(object):
                 mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
             query = Template(query).substitute(mapping)
         return query
+
+    def _get_server_timezone(self):
+        try:
+            r = self._send('SELECT timezone()')
+            return pytz.timezone(r.text.strip())
+        except DatabaseException:
+            logger.exception('Cannot determine server timezone, assuming UTC')
+            return pytz.utc

src/infi/clickhouse_orm/fields.py

@@ -2,6 +2,7 @@ from six import string_types, text_type, binary_type
 import datetime
 import pytz
 import time
+from calendar import timegm
 
 from .utils import escape, parse_array
 
@@ -24,10 +25,11 @@ class Field(object):
         self.alias = alias
         self.materialized = materialized
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         '''
         Converts the input value into the expected Python data type, raising ValueError if the
         data can't be converted. Returns the converted value. Subclasses should override this.
+        The timezone_in_use parameter should be consulted when parsing datetime fields.
         '''
         return value
 
@@ -77,7 +79,7 @@ class StringField(Field):
     class_default = ''
     db_type = 'String'
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         if isinstance(value, text_type):
             return value
         if isinstance(value, binary_type):
@@ -92,11 +94,11 @@ class DateField(Field):
     class_default = min_value
     db_type = 'Date'
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
-        if isinstance(value, datetime.date):
-            return value
         if isinstance(value, datetime.datetime):
             return value.date()
+        if isinstance(value, datetime.date):
+            return value
         if isinstance(value, int):
             return DateField.class_default + datetime.timedelta(days=value)
         if isinstance(value, string_types):
@@ -117,26 +119,27 @@ class DateTimeField(Field):
     class_default = datetime.datetime.fromtimestamp(0, pytz.utc)
     db_type = 'DateTime'
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         if isinstance(value, datetime.datetime):
-            return value
+            return value.astimezone(pytz.utc) if value.tzinfo else value.replace(tzinfo=pytz.utc)
         if isinstance(value, datetime.date):
-            return datetime.datetime(value.year, value.month, value.day)
+            return datetime.datetime(value.year, value.month, value.day, tzinfo=pytz.utc)
         if isinstance(value, int):
-            return datetime.datetime.fromtimestamp(value, pytz.utc)
+            return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
         if isinstance(value, string_types):
             if value == '0000-00-00 00:00:00':
                 return self.class_default
-            return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
+            dt = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
+            return timezone_in_use.localize(dt).astimezone(pytz.utc)
         raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
     def to_db_string(self, value, quote=True):
-        return escape(int(time.mktime(value.timetuple())), quote)
+        return escape(timegm(value.utctimetuple()), quote)
 
 
 class BaseIntField(Field):
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         try:
             return int(value)
         except:
@@ -204,7 +207,7 @@ class Int64Field(BaseIntField):
 
 class BaseFloatField(Field):
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         try:
             return float(value)
         except:
@@ -229,7 +232,7 @@ class BaseEnumField(Field):
             default = list(enum_cls)[0]
         super(BaseEnumField, self).__init__(default, alias, materialized)
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         if isinstance(value, self.enum_cls):
             return value
         try:
@@ -291,14 +294,14 @@ class ArrayField(Field):
         self.inner_field = inner_field
         super(ArrayField, self).__init__(default, alias, materialized)
 
-    def to_python(self, value):
+    def to_python(self, value, timezone_in_use):
         if isinstance(value, text_type):
             value = parse_array(value)
         elif isinstance(value, binary_type):
             value = parse_array(value.decode('UTF-8'))
         elif not isinstance(value, (list, tuple)):
             raise ValueError('ArrayField expects list or tuple, not %s' % type(value))
-        return [self.inner_field.to_python(v) for v in value]
+        return [self.inner_field.to_python(v, timezone_in_use) for v in value]
 
     def validate(self, value):
         for v in value:

src/infi/clickhouse_orm/models.py

@@ -3,6 +3,7 @@ from .engines import *
 from .fields import Field
 
 from six import with_metaclass
+import pytz
 
 from logging import getLogger
 logger = getLogger('clickhouse_orm')
@@ -96,7 +97,7 @@ class Model(with_metaclass(ModelBase)):
         '''
         field = self.get_field(name)
         if field:
-            value = field.to_python(value)
+            value = field.to_python(value, pytz.utc)
             field.validate(value)
         super(Model, self).__setattr__(name, value)
 
@@ -136,7 +137,7 @@ class Model(with_metaclass(ModelBase)):
         return 'DROP TABLE IF EXISTS `%s`.`%s`' % (db_name, cls.table_name())
 
     @classmethod
-    def from_tsv(cls, line, field_names=None):
+    def from_tsv(cls, line, field_names=None, timezone_in_use=pytz.utc):
         '''
         Create a model instance from a tab-separated line. The line may or may not include a newline.
         The field_names list must match the fields defined in the model, but does not have to include all of them.
@@ -147,7 +148,8 @@ class Model(with_metaclass(ModelBase)):
         values = iter(parse_tsv(line))
         kwargs = {}
         for name in field_names:
-            kwargs[name] = next(values)
+            field = getattr(cls, name)
+            kwargs[name] = field.to_python(next(values), timezone_in_use)
         return cls(**kwargs)
 
     def to_tsv(self, insertable_only=False):

tests/test_simple_fields.py

@@ -0,0 +1,53 @@
+import unittest
+from infi.clickhouse_orm.fields import *
+from datetime import date, datetime
+import pytz
+
+
+class SimpleFieldsTest(unittest.TestCase):
+
+    def test_date_field(self):
+        f = DateField()
+        # Valid values
+        for value in (date(1970, 1, 1), datetime(1970, 1, 1), '1970-01-01', '0000-00-00', 0):
+            self.assertEquals(f.to_python(value, pytz.utc), date(1970, 1, 1))
+        # Invalid values
+        for value in ('nope', '21/7/1999', 0.5):
+            with self.assertRaises(ValueError):
+                f.to_python(value, pytz.utc)
+        # Range check
+        for value in (date(1900, 1, 1), date(2900, 1, 1)):
+            with self.assertRaises(ValueError):
+                f.validate(value)
+
+    def test_datetime_field(self):
+        f = DateTimeField()
+        epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
+        # Valid values
+        for value in (date(1970, 1, 1), datetime(1970, 1, 1), epoch, 
+                      epoch.astimezone(pytz.timezone('US/Eastern')), epoch.astimezone(pytz.timezone('Asia/Jerusalem')),
+                      '1970-01-01 00:00:00', '0000-00-00 00:00:00', 0):
+            dt = f.to_python(value, pytz.utc)
+            self.assertEquals(dt.tzinfo, pytz.utc)
+            self.assertEquals(dt, epoch)
+            # Verify that conversion to and from db string does not change value
+            dt2 = f.to_python(int(f.to_db_string(dt)), pytz.utc)
+            self.assertEquals(dt, dt2)
+        # Invalid values
+        for value in ('nope', '21/7/1999', 0.5):
+            with self.assertRaises(ValueError):
+                f.to_python(value, pytz.utc)
+
+    def test_uint8_field(self):
+        f = UInt8Field()
+        # Valid values
+        for value in (17, '17', 17.0):
+            self.assertEquals(f.to_python(value, pytz.utc), 17)
+        # Invalid values
+        for value in ('nope', date.today()):
+            with self.assertRaises(ValueError):
+                f.to_python(value, pytz.utc)
+        # Range check
+        for value in (-1, 1000):
+            with self.assertRaises(ValueError):
+                f.validate(value)