Merge branch 'tsionyx-develop' into develop

buildout.cfg

@@ -10,6 +10,7 @@ name = infi.clickhouse_orm
 company = Infinidat
 namespace_packages = ['infi']
 install_requires = [
+	'iso8601 >= 0.1.12',
 	'pytz',
 	'requests',
 	'setuptools',

docs/field_types.md

@@ -32,7 +32,7 @@ A `DateTimeField` can be assigned values from one of the following types:
 -   datetime
 -   date
 -   integer - number of seconds since the Unix epoch
--   string in `YYYY-MM-DD HH:MM:SS` format
+-   string in `YYYY-MM-DD HH:MM:SS` format or [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601)-compatible format
 
 The assigned value always gets converted to a timezone-aware `datetime` in UTC. If the assigned value is a timezone-aware `datetime` in another timezone, it will be converted to UTC. Otherwise, the assigned value is assumed to already be in UTC.
 

src/infi/clickhouse_orm/fields.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 from six import string_types, text_type, binary_type
 import datetime
+import iso8601
 import pytz
 import time
 from calendar import timegm
@@ -157,8 +158,16 @@ class DateTimeField(Field):
                     return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
                 except ValueError:
                     pass
-            dt = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
+            try:
-            return timezone_in_use.localize(dt).astimezone(pytz.utc)
+                # left the date naive in case of no tzinfo set
+                dt = iso8601.parse_date(value, default_timezone=None)
+            except iso8601.ParseError as e:
+                raise ValueError(text_type(e))
+
+            # convert naive to aware
+            if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
+                dt = timezone_in_use.localize(dt)
+            return dt.astimezone(pytz.utc)
         raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
     def to_db_string(self, value, quote=True):

src/infi/clickhouse_orm/models.py

@@ -1,7 +1,8 @@
 from __future__ import unicode_literals
+import sys
 from logging import getLogger
 
-from six import with_metaclass
+from six import with_metaclass, reraise
 import pytz
 
 from .fields import Field, StringField
@@ -124,8 +125,13 @@ class Model(with_metaclass(ModelBase)):
         '''
         field = self.get_field(name)
         if field:
-            value = field.to_python(value, pytz.utc)
+            try:
-            field.validate(value)
+                value = field.to_python(value, pytz.utc)
+                field.validate(value)
+            except ValueError:
+                tp, v, tb = sys.exc_info()
+                new_msg = "{} (field '{}')".format(v, name)
+                reraise(tp, tp(new_msg), tb)
         super(Model, self).__setattr__(name, value)
 
     def set_database(self, db):

tests/test_join.py

@@ -24,7 +24,6 @@ class JoinTest(unittest.TestCase):
         self.print_res("SELECT * FROM {}".format(Bar.table_name()))
         self.print_res("SELECT b FROM {} ALL LEFT JOIN {} USING id".format(Foo.table_name(), Bar.table_name()))
 
-    @unittest.skip('ClickHouse issue - https://github.com/yandex/ClickHouse/issues/635')
     def test_with_db_name(self):
         self.print_res("SELECT * FROM $db.{}".format(Foo.table_name()))
         self.print_res("SELECT * FROM $db.{}".format(Bar.table_name()))

tests/test_models.py

@@ -79,6 +79,27 @@ class ModelTestCase(unittest.TestCase):
                 "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)
             })
 
+    def test_field_name_in_error_message_for_invalid_value_in_constructor(self):
+        bad_value = 1
+        with self.assertRaises(ValueError) as cm:
+            SimpleModel(str_field=bad_value)
+
+        self.assertEqual(
+            "Invalid value for StringField: {} (field 'str_field')".format(repr(bad_value)),
+            text_type(cm.exception)
+        )
+
+    def test_field_name_in_error_message_for_invalid_value_in_assignment(self):
+        instance = SimpleModel()
+        bad_value = 'foo'
+        with self.assertRaises(ValueError) as cm:
+            instance.float_field = bad_value
+
+        self.assertEqual(
+            "Invalid value for Float32Field - {} (field 'float_field')".format(repr(bad_value)),
+            text_type(cm.exception)
+        )
+
 
 class SimpleModel(Model):
 

tests/test_simple_fields.py

@@ -13,14 +13,17 @@ class SimpleFieldsTest(unittest.TestCase):
         # Valid values
         for value in (date(1970, 1, 1), datetime(1970, 1, 1), epoch,
                       epoch.astimezone(pytz.timezone('US/Eastern')), epoch.astimezone(pytz.timezone('Asia/Jerusalem')),
-                      '1970-01-01 00:00:00', '1970-01-17 00:00:17', '0000-00-00 00:00:00', 0):
+                      '1970-01-01 00:00:00', '1970-01-17 00:00:17', '0000-00-00 00:00:00', 0,
+                      '2017-07-26T08:31:05', '2017-07-26T08:31:05Z', '2017-07-26 08:31',
+                      '2017-07-26T13:31:05+05', '2017-07-26 13:31:05+0500'):
             dt = f.to_python(value, pytz.utc)
             self.assertEquals(dt.tzinfo, pytz.utc)
             # Verify that conversion to and from db string does not change value
             dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc)
             self.assertEquals(dt, dt2)
         # Invalid values
-        for value in ('nope', '21/7/1999', 0.5):
+        for value in ('nope', '21/7/1999', 0.5,
+                      '2017-01 15:06:00', '2017-01-01X15:06:00', '2017-13-01T15:06:00'):
             with self.assertRaises(ValueError):
                 f.to_python(value, pytz.utc)
 
@@ -49,6 +52,19 @@ class SimpleFieldsTest(unittest.TestCase):
         dt = datetime(2017, 10, 5, tzinfo=pytz.timezone('Asia/Jerusalem'))
         self.assertEquals(f.to_python(dt, pytz.utc), date(2017, 10, 4))
 
+    def test_datetime_field_timezone(self):
+        # Verify that conversion of timezone-aware datetime is correct
+        f = DateTimeField()
+        utc_value = datetime(2017, 7, 26, 8, 31, 5, tzinfo=pytz.UTC)
+        for value  in (
+                '2017-07-26T08:31:05',
+                '2017-07-26T08:31:05Z',
+                '2017-07-26T11:31:05+03',
+                '2017-07-26 11:31:05+0300',
+                '2017-07-26T03:31:05-0500',
+        ):
+            self.assertEquals(f.to_python(value, pytz.utc), utc_value)
+
     def test_uint8_field(self):
         f = UInt8Field()
         # Valid values