diff --git a/src/django_clickhouse/db_clients.py b/src/django_clickhouse/db_clients.py index db3d73f..523e2d5 100644 --- a/src/django_clickhouse/db_clients.py +++ b/src/django_clickhouse/db_clients.py @@ -4,10 +4,16 @@ from .exceptions import DBAliasError from .configuration import config +DEFAULT_DB_ALIAS = 'default' + + class ConnectionProxy: _connections = {} def get_connection(self, alias): + if alias is None: + alias = DEFAULT_DB_ALIAS + if alias not in self._connections: if alias not in config.DATABASES: raise DBAliasError(alias) @@ -16,7 +22,7 @@ class ConnectionProxy: return self._connections[alias] - def __getattr__(self, item): + def __getitem__(self, item): return self.get_connection(item) diff --git a/src/django_clickhouse/serializers.py b/src/django_clickhouse/serializers.py index d8dedff..a94ed06 100644 --- a/src/django_clickhouse/serializers.py +++ b/src/django_clickhouse/serializers.py @@ -5,11 +5,12 @@ from django.forms import model_to_dict class Django2ClickHouseModelSerializer: - serialize_fields = None - exclude_serialize_fields = None + def __init__(self, fields=None, exclude_fields=None): + self.serialize_fields = fields + self.exclude_serialize_fields = exclude_fields - def serialize(self, obj, model_cls, **kwargs): - # type: (DjangoModel, Type['ClickHouseModel'], **dict) -> 'ClickHouseModel' + def serialize(self, obj, model_cls): + # type: (DjangoModel, Type['ClickHouseModel']) -> 'ClickHouseModel' data = model_to_dict(obj, self.serialize_fields, self.exclude_serialize_fields) # Remove None values, they should be initialized as defaults diff --git a/src/django_clickhouse/utils.py b/src/django_clickhouse/utils.py index f924b81..a6d45f9 100644 --- a/src/django_clickhouse/utils.py +++ b/src/django_clickhouse/utils.py @@ -1,41 +1,53 @@ -from typing import Union, Any +import datetime +from typing import Union, Any, Optional +import pytz import six from importlib import import_module +from infi.clickhouse_orm.database import Database -def get_clickhouse_tz_offset(): +from .db_clients import connections + + +def get_tz_offset(db_alias=None): # type: (Optional[str]) -> int """ - Получает смещение временной зоны сервера ClickHouse в минутах + Returns ClickHouse server timezone offset in minutes + :param db_alias: The database alias used :return: Integer """ - # Если даты форматируются вручную, то сервер воспринимает их как локаль сервера. Надо ее вычесть. - return int(settings.CLICKHOUSE_DB.server_timezone.utcoffset(datetime.datetime.utcnow()).total_seconds() / 60) + db = connections[db_alias] + return int(db.server_timezone.utcoffset(datetime.datetime.utcnow()).total_seconds() / 60) -def format_datetime(dt, timezone_offset=0, day_end=False): +def format_datetime(dt, timezone_offset=0, day_end=False, db_alias=None): + # type: (Union[datetime.date, datetime.datetime], int, bool, Optional[str]) -> str """ - Форматирует datetime.datetime в строковое представление, которое можно использовать в запросах к ClickHouse - :param dt: Объект datetime.datetime или datetime.date - :param timezone_offset: Смещение временной зоны в минутах - :param day_end: Если флаг установлен, то будет взято время окончания дня, а не начала - :return: Строковое представление даты-времени + Formats datetime and date objects to format that can be used in WHERE conditions of query + :param dt: datetime.datetime or datetime.date object + :param timezone_offset: timezone offset (minutes) + :param day_end: If datetime.date is given and flag is set, returns day end time, not day start. + :param db_alias: The database alias used + :return: A string representing datetime """ assert isinstance(dt, (datetime.datetime, datetime.date)), "dt must be datetime.datetime instance" assert type(timezone_offset) is int, "timezone_offset must be integer" - # datetime.datetime наследует datetime.date. Поэтому нельзя делать условие без отрицания + # datetime.datetime inherits datetime.date. So I can't just make isinstance(dt, datetime.date) if not isinstance(dt, datetime.datetime): t = datetime.time.max if day_end else datetime.time.min dt = datetime.datetime.combine(dt, t) + # Convert datetime to UTC, if it has timezone if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None: dt = pytz.utc.localize(dt) else: dt = dt.astimezone(pytz.utc) - # Если даты форматируются вручную, то сервер воспринимает их как локаль сервера. - return (dt - datetime.timedelta(minutes=timezone_offset - get_clickhouse_tz_offset())).strftime("%Y-%m-%d %H:%M:%S") + # Dates in ClickHouse are parsed in server local timezone. So I need to add server timezone + server_dt = dt - datetime.timedelta(minutes=timezone_offset - get_tz_offset(db_alias)) + + return server_dt.strftime("%Y-%m-%d %H:%M:%S") def lazy_class_import(obj): # type: (Union[str, Any]) -> Any diff --git a/tests/clickhouse_models.py b/tests/clickhouse_models.py index 0598764..fb92d08 100644 --- a/tests/clickhouse_models.py +++ b/tests/clickhouse_models.py @@ -9,7 +9,8 @@ class TestClickHouseModel(ClickHouseModel): django_model = TestModel sync_delay = 5 + id = fields.Int32Field() created_date = fields.DateField() - value = fields.UInt32Field() + value = fields.Int32Field() engine = MergeTree('created_Date') diff --git a/tests/settings.py b/tests/settings.py index 6af53e0..562278f 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -34,7 +34,16 @@ INSTALLED_APPS = [ "tests" ] +CLICKHOUSE_DATABASES = { + 'default': { + 'db_name': 'test', + 'username': 'default', + 'password': '' + } +} + CLICKHOUSE_SYNC_BATCH_SIZE = 5000 + CLICKHOUSE_REDIS_CONFIG = { 'host': '127.0.0.1', 'port': 6379, diff --git a/tests/test_serializers.py b/tests/test_serializers.py new file mode 100644 index 0000000..4ad66bd --- /dev/null +++ b/tests/test_serializers.py @@ -0,0 +1,38 @@ +import datetime + +from django.test import TestCase + +from django_clickhouse.serializers import Django2ClickHouseModelSerializer +from tests.clickhouse_models import TestClickHouseModel +from tests.models import TestModel + + +class Django2ClickHouseModelSerializerTest(TestCase): + fixtures = ['test_model'] + + def setUp(self): + self.obj = TestModel.objects.get(pk=1) + + def test_all(self): + serializer = Django2ClickHouseModelSerializer() + res = serializer.serialize(self.obj, TestClickHouseModel) + self.assertIsInstance(res, TestClickHouseModel) + self.assertEqual(self.obj.id, res.id) + self.assertEqual(self.obj.value, res.value) + self.assertEqual(self.obj.created_date, res.created_date) + + def test_fields(self): + serializer = Django2ClickHouseModelSerializer(fields=('value')) + res = serializer.serialize(self.obj, TestClickHouseModel) + self.assertIsInstance(res, TestClickHouseModel) + self.assertEqual(0, res.id) + self.assertEqual(datetime.date(1970, 1, 1), res.created_date) + self.assertEqual(self.obj.value, res.value) + + def test_exclude_fields(self): + serializer = Django2ClickHouseModelSerializer(exclude_fields=('created_date',)) + res = serializer.serialize(self.obj, TestClickHouseModel) + self.assertIsInstance(res, TestClickHouseModel) + self.assertEqual(datetime.date(1970, 1, 1), res.created_date) + self.assertEqual(self.obj.id, res.id) + self.assertEqual(self.obj.value, res.value) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..63fba9e --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,47 @@ +import datetime +from unittest import TestCase + +import pytz + +from django_clickhouse.utils import get_tz_offset, format_datetime + + +class GetTZOffsetTest(TestCase): + def test_func(self): + self.assertEqual(300, get_tz_offset()) + + +class FormatDateTimeTest(TestCase): + @staticmethod + def _get_zone_time(dt): + """ + На момент написания тестов в РФ было какое-то странное смещение (для Москвы, например +2:30, для Перми +4:03) + :param dt: Объект datetime.datetime + :return: Строковый ожидаемый результат + """ + moscow_minute_offset = dt.utcoffset().total_seconds() / 60 + zone_h, zone_m = abs(int(moscow_minute_offset / 60)), int(moscow_minute_offset % 60) + + # +5 за счет времени тестового сервера ClickHouse + return (dt - datetime.timedelta(hours=zone_h - 5, minutes=zone_m)).strftime("%Y-%m-%d %H:%M:%S") + + def test_conversion(self): + dt = datetime.datetime(2017, 1, 2, 3, 4, 5) + self.assertEqual(format_datetime(dt), '2017-01-02 08:04:05') + dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.utc) + self.assertEqual(format_datetime(dt), '2017-01-02 08:04:05') + dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow')) + self.assertEqual(format_datetime(dt), self._get_zone_time(dt)) + dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow')) + offset = int(pytz.timezone('Europe/Moscow').utcoffset(dt).total_seconds() / 60) + self.assertEqual(format_datetime(dt, timezone_offset=offset), '2017-01-02 03:04:05') + + def test_date_conversion(self): + dt = datetime.date(2017, 1, 2) + self.assertEqual(format_datetime(dt), '2017-01-02 05:00:00') + dt = datetime.date(2017, 1, 2) + self.assertEqual(format_datetime(dt, day_end=True), '2017-01-03 04:59:59') + dt = datetime.date(2017, 1, 2) + self.assertEqual(format_datetime(dt, day_end=True, timezone_offset=60), '2017-01-03 03:59:59') + dt = datetime.date(2017, 1, 2) + self.assertEqual(format_datetime(dt, timezone_offset=60), '2017-01-02 04:00:00')