DateTime64 field - additional fixes & docs

This commit is contained in:
Itai Shirav 2020-06-23 11:04:42 +03:00
parent 0dece65b7b
commit 7fe76c185d
6 changed files with 36 additions and 10 deletions

View File

@ -5,6 +5,12 @@ Unreleased
----------
- Support for model constraints
- Support for data skipping indexes
- Added `DateTime64Field` (NiyazNz)
- Make `DateTimeField` and `DateTime64Field` timezone-aware (NiyazNz)
**Backwards incompatibile changes**
Previously, `DateTimeField` always converted its value from the database timezone to UTC. This is no longer the case: the field's value now preserves the timezone it was defined with, or if not specified - the database's global timezone. This change has no effect if your database timezone is set UTC.
v2.0.1
------

View File

@ -38,16 +38,22 @@ The following field types are supported:
DateTimeField and Time Zones
----------------------------
A `DateTimeField` can be assigned values from one of the following types:
`DateTimeField` and `DateTime64Field` can accept a `timezone` parameter (either the timezone name or a `pytz` timezone instance). This timezone will be used as the column timezone in ClickHouse. If not provided, the fields will use the timezone defined in the database configuration.
A `DateTimeField` and `DateTime64Field` can be assigned values from one of the following types:
- datetime
- date
- integer - number of seconds since the Unix epoch
- float (DateTime64Field only) - number of seconds and microseconds since the Unix epoch
- string in `YYYY-MM-DD HH:MM:SS` format or [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601)-compatible format
The assigned value always gets converted to a timezone-aware `datetime` in UTC. If the assigned value is a timezone-aware `datetime` in another timezone, it will be converted to UTC. Otherwise, the assigned value is assumed to already be in UTC.
The assigned value always gets converted to a timezone-aware `datetime` in UTC. The only exception is when the assigned value is a timezone-aware `datetime`, in which case it will not be changed.
DateTime values that are read from the database are kept in the database-defined timezone - either the one defined for the field, or the global timezone defined in the database configuration.
It is strongly recommended to set the server timezone to UTC and to store all datetime values in that timezone, in order to prevent confusion and subtle bugs. Conversion to a different timezone should only be performed when the value needs to be displayed.
DateTime values that are read from the database are also converted to UTC. ClickHouse formats them according to the timezone of the server, and the ORM makes the necessary conversions. This requires a ClickHouse version which is new enough to support the `timezone()` function, otherwise it is assumed to be using UTC. In any case, we recommend settings the server timezone to UTC in order to prevent confusion.
Working with enum fields
------------------------

View File

@ -267,7 +267,7 @@ class DateTime64Field(DateTimeField):
'{timestamp:0{width}.{precision}f}'.format(
timestamp=value.timestamp(),
width=11 + self.precision,
precision=6),
precision=self.precision),
quote
)
@ -278,9 +278,10 @@ class DateTime64Field(DateTimeField):
if isinstance(value, (int, float)):
return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
if isinstance(value, str):
if value.split('.')[0] == '0000-00-00 00:00:00':
left_part = value.split('.')[0]
if left_part == '0000-00-00 00:00:00':
return self.class_default
if len(value.split('.')[0]) == 10:
if len(left_part) == 10:
try:
value = float(value)
return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)

View File

@ -71,8 +71,11 @@ class ModelWithTz(Model):
class DateTimeFieldWithTzTest(unittest.TestCase):
def setUp(self):
self.database = Database('test-db', log_statements=True)
if self.database.server_version < (20, 1, 2, 4):
raise unittest.SkipTest('ClickHouse version too old')
self.database.create_table(ModelWithTz)
def tearDown(self):

View File

@ -34,6 +34,7 @@ class FuncsTestCase(TestCaseWithData):
result = list(self.database.select(sql))
logging.info('\t==> %s', result[0].value if result else '<empty>')
if expected_value != NO_VALUE:
print('Comparing %s to %s' % (result[0].value, expected_value))
self.assertEqual(result[0].value, expected_value)
return result[0].value if result else None
except ServerError as e:
@ -310,12 +311,13 @@ class FuncsTestCase(TestCaseWithData):
raise unittest.SkipTest('This test must run with UTC as the server timezone')
d = date(2018, 12, 31)
dt = datetime(2018, 12, 31, 11, 22, 33)
athens_tz = pytz.timezone('Europe/Athens')
self._test_func(F.toHour(dt), 11)
self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, 'Europe/Athens'), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, pytz.timezone('Europe/Athens')), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTimeZone(dt, 'Europe/Athens'), datetime(2018, 12, 31, 13, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, 'Europe/Athens'), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33)))
self._test_func(F.toTime(dt, athens_tz), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33)))
self._test_func(F.toTimeZone(dt, 'Europe/Athens'), athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33)))
self._test_func(F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0)) # FIXME this may fail if the timing is just right
self._test_func(F.today(), datetime.utcnow().date())
self._test_func(F.yesterday(), datetime.utcnow().date() - timedelta(days=1))

View File

@ -6,6 +6,7 @@ import pytz
class SimpleFieldsTest(unittest.TestCase):
epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
# Valid values
dates = [
@ -32,7 +33,6 @@ class SimpleFieldsTest(unittest.TestCase):
def test_datetime64_field(self):
f = DateTime64Field()
epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)
# Valid values
for value in self.dates + [
datetime(1970, 1, 1, microsecond=100000),
@ -52,6 +52,14 @@ class SimpleFieldsTest(unittest.TestCase):
with self.assertRaises(ValueError):
f.to_python(value, pytz.utc)
def test_datetime64_field_precision(self):
for precision in range(1, 7):
f = DateTime64Field(precision=precision, timezone=pytz.utc)
dt = f.to_python(datetime(2000, 1, 1, microsecond=123456), pytz.utc)
dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc)
m = round(123456, precision - 6) # round rightmost microsecond digits according to precision
self.assertEqual(dt2, dt.replace(microsecond=m))
def test_date_field(self):
f = DateField()
epoch = date(1970, 1, 1)