DateTime64 field - additional fixes & docs

CHANGELOG.md

@@ -5,6 +5,12 @@ Unreleased
 ----------
 - Support for model constraints
 - Support for data skipping indexes
+- Added `DateTime64Field` (NiyazNz)
+- Make `DateTimeField` and `DateTime64Field` timezone-aware (NiyazNz)
+
+**Backwards incompatibile changes**
+
+Previously, `DateTimeField` always converted its value from the database timezone to UTC. This is no longer the case: the field's value now preserves the timezone it was defined with, or if not specified - the database's global timezone. This change has no effect if your database timezone is set UTC.
 
 v2.0.1
 ------

docs/field_types.md

@@ -38,16 +38,22 @@ The following field types are supported:
 DateTimeField and Time Zones
 ----------------------------
 
-A `DateTimeField` can be assigned values from one of the following types:
+`DateTimeField` and `DateTime64Field` can accept a `timezone` parameter (either the timezone name or a `pytz` timezone instance). This timezone will be used as the column timezone in ClickHouse. If not provided, the fields will use the timezone defined in the database configuration.
+
+A `DateTimeField` and `DateTime64Field` can be assigned values from one of the following types:
 
 -   datetime
 -   date
 -   integer - number of seconds since the Unix epoch
+-   float (DateTime64Field only) - number of seconds and microseconds since the Unix epoch
 -   string in `YYYY-MM-DD HH:MM:SS` format or [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601)-compatible format
 
-The assigned value always gets converted to a timezone-aware `datetime` in UTC. If the assigned value is a timezone-aware `datetime` in another timezone, it will be converted to UTC. Otherwise, the assigned value is assumed to already be in UTC.
+The assigned value always gets converted to a timezone-aware `datetime` in UTC. The only exception is when the assigned value is a timezone-aware `datetime`, in which case it will not be changed.
+
+DateTime values that are read from the database are kept in the database-defined timezone - either the one defined for the field, or the global timezone defined in the database configuration.
+
+It is strongly recommended to set the server timezone to UTC and to store all datetime values in that timezone, in order to prevent confusion and subtle bugs. Conversion to a different timezone should only be performed when the value needs to be displayed.
 
-DateTime values that are read from the database are also converted to UTC. ClickHouse formats them according to the timezone of the server, and the ORM makes the necessary conversions. This requires a ClickHouse version which is new enough to support the `timezone()` function, otherwise it is assumed to be using UTC. In any case, we recommend settings the server timezone to UTC in order to prevent confusion.
 
 Working with enum fields
 ------------------------

src/infi/clickhouse_orm/fields.py

@@ -267,7 +267,7 @@ class DateTime64Field(DateTimeField):
             '{timestamp:0{width}.{precision}f}'.format(
                 timestamp=value.timestamp(),
                 width=11 + self.precision,
-                precision=6),
+                precision=self.precision),
             quote
         )
 
@@ -278,9 +278,10 @@ class DateTime64Field(DateTimeField):
             if isinstance(value, (int, float)):
                 return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
             if isinstance(value, str):
-                if value.split('.')[0] == '0000-00-00 00:00:00':
+                left_part = value.split('.')[0]
+                if left_part == '0000-00-00 00:00:00':
                     return self.class_default
-                if len(value.split('.')[0]) == 10:
+                if len(left_part) == 10:
                     try:
                         value = float(value)
                         return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)

tests/test_datetime_fields.py

@@ -71,8 +71,11 @@ class ModelWithTz(Model):
 
 
 class DateTimeFieldWithTzTest(unittest.TestCase):
+
     def setUp(self):
         self.database = Database('test-db', log_statements=True)
+        if self.database.server_version < (20, 1, 2, 4):
+            raise unittest.SkipTest('ClickHouse version too old')
         self.database.create_table(ModelWithTz)
 
     def tearDown(self):

tests/test_funcs.py

@@ -34,6 +34,7 @@ class FuncsTestCase(TestCaseWithData):
             result = list(self.database.select(sql))
             logging.info('\t==> %s', result[0].value if result else '<empty>')
             if expected_value != NO_VALUE:
+                print('Comparing %s to %s' % (result[0].value, expected_value))
                 self.assertEqual(result[0].value, expected_value)
             return result[0].value if result else None
         except ServerError as e:
@@ -310,12 +311,13 @@ class FuncsTestCase(TestCaseWithData):
             raise unittest.SkipTest('This test must run with UTC as the server timezone')
         d = date(2018, 12, 31)
         dt = datetime(2018, 12, 31, 11, 22, 33)
+        athens_tz = pytz.timezone('Europe/Athens')
         self._test_func(F.toHour(dt), 11)
         self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc))
         self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc))
-        self._test_func(F.toTime(dt, 'Europe/Athens'), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toTime(dt, 'Europe/Athens'), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33)))
-        self._test_func(F.toTime(dt, pytz.timezone('Europe/Athens')), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toTime(dt, athens_tz), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33)))
-        self._test_func(F.toTimeZone(dt, 'Europe/Athens'), datetime(2018, 12, 31, 13, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toTimeZone(dt, 'Europe/Athens'), athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33)))
         self._test_func(F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0)) # FIXME this may fail if the timing is just right
         self._test_func(F.today(), datetime.utcnow().date())
         self._test_func(F.yesterday(), datetime.utcnow().date() - timedelta(days=1))

tests/test_simple_fields.py

@@ -6,6 +6,7 @@ import pytz
 
 
 class SimpleFieldsTest(unittest.TestCase):
+
     epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
     # Valid values
     dates = [
@@ -32,7 +33,6 @@ class SimpleFieldsTest(unittest.TestCase):
 
     def test_datetime64_field(self):
         f = DateTime64Field()
-        epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
         # Valid values
         for value in self.dates + [
             datetime(1970, 1, 1, microsecond=100000),
@@ -52,6 +52,14 @@ class SimpleFieldsTest(unittest.TestCase):
             with self.assertRaises(ValueError):
                 f.to_python(value, pytz.utc)
 
+    def test_datetime64_field_precision(self):
+        for precision in range(1, 7):
+            f = DateTime64Field(precision=precision, timezone=pytz.utc)
+            dt = f.to_python(datetime(2000, 1, 1, microsecond=123456), pytz.utc)
+            dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc)
+            m = round(123456, precision - 6) # round rightmost microsecond digits according to precision
+            self.assertEqual(dt2, dt.replace(microsecond=m))
+
     def test_date_field(self):
         f = DateField()
         epoch = date(1970, 1, 1)