Added tests for utils and serializers

src/django_clickhouse/db_clients.py

@@ -4,10 +4,16 @@ from .exceptions import DBAliasError
 from .configuration import config
 
 
+DEFAULT_DB_ALIAS = 'default'
+
+
 class ConnectionProxy:
     _connections = {}
 
     def get_connection(self, alias):
+        if alias is None:
+            alias = DEFAULT_DB_ALIAS
+
         if alias not in self._connections:
             if alias not in config.DATABASES:
                 raise DBAliasError(alias)
@@ -16,7 +22,7 @@ class ConnectionProxy:
 
         return self._connections[alias]
 
-    def __getattr__(self, item):
+    def __getitem__(self, item):
         return self.get_connection(item)
 
 

src/django_clickhouse/serializers.py

@@ -5,11 +5,12 @@ from django.forms import model_to_dict
 
 
 class Django2ClickHouseModelSerializer:
-    serialize_fields = None
-    exclude_serialize_fields = None
+    def __init__(self, fields=None, exclude_fields=None):
+        self.serialize_fields = fields
+        self.exclude_serialize_fields = exclude_fields
 
-    def serialize(self, obj, model_cls, **kwargs):
-        # type: (DjangoModel, Type['ClickHouseModel'], **dict) -> 'ClickHouseModel'
+    def serialize(self, obj, model_cls):
+        # type: (DjangoModel, Type['ClickHouseModel']) -> 'ClickHouseModel'
         data = model_to_dict(obj, self.serialize_fields, self.exclude_serialize_fields)
 
         # Remove None values, they should be initialized as defaults

src/django_clickhouse/utils.py

@@ -1,41 +1,53 @@
-from typing import Union, Any
+import datetime
+from typing import Union, Any, Optional
 
+import pytz
 import six
 from importlib import import_module
 
+from infi.clickhouse_orm.database import Database
 
-def get_clickhouse_tz_offset():
+from .db_clients import connections
+
+
+def get_tz_offset(db_alias=None):  # type: (Optional[str]) -> int
     """
-    Получает смещение временной зоны сервера ClickHouse в минутах
+    Returns ClickHouse server timezone offset in minutes
+    :param db_alias: The database alias used
     :return: Integer
     """
-    # Если даты форматируются вручную, то сервер воспринимает их как локаль сервера. Надо ее вычесть.
-    return int(settings.CLICKHOUSE_DB.server_timezone.utcoffset(datetime.datetime.utcnow()).total_seconds() / 60)
+    db = connections[db_alias]
+    return int(db.server_timezone.utcoffset(datetime.datetime.utcnow()).total_seconds() / 60)
 
 
-def format_datetime(dt, timezone_offset=0, day_end=False):
+def format_datetime(dt, timezone_offset=0, day_end=False, db_alias=None):
+    # type: (Union[datetime.date, datetime.datetime], int, bool, Optional[str]) -> str
     """
-    Форматирует datetime.datetime в строковое представление, которое можно использовать в запросах к ClickHouse
-    :param dt: Объект datetime.datetime или datetime.date
-    :param timezone_offset: Смещение временной зоны в минутах
-    :param day_end: Если флаг установлен, то будет взято время окончания дня, а не начала
-    :return: Строковое представление даты-времени
+    Formats datetime and date objects to format that can be used in WHERE conditions of query
+    :param dt: datetime.datetime or datetime.date object
+    :param timezone_offset: timezone offset (minutes)
+    :param day_end: If datetime.date is given and flag is set, returns day end time, not day start.
+    :param db_alias: The database alias used
+    :return: A string representing datetime
     """
     assert isinstance(dt, (datetime.datetime, datetime.date)), "dt must be datetime.datetime instance"
     assert type(timezone_offset) is int, "timezone_offset must be integer"
 
-    # datetime.datetime наследует datetime.date. Поэтому нельзя делать условие без отрицания
+    # datetime.datetime inherits datetime.date. So I can't just make isinstance(dt, datetime.date)
     if not isinstance(dt, datetime.datetime):
         t = datetime.time.max if day_end else datetime.time.min
         dt = datetime.datetime.combine(dt, t)
 
+    # Convert datetime to UTC, if it has timezone
     if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
         dt = pytz.utc.localize(dt)
     else:
         dt = dt.astimezone(pytz.utc)
 
-    # Если даты форматируются вручную, то сервер воспринимает их как локаль сервера.
-    return (dt - datetime.timedelta(minutes=timezone_offset - get_clickhouse_tz_offset())).strftime("%Y-%m-%d %H:%M:%S")
+    # Dates in ClickHouse are parsed in server local timezone. So I need to add server timezone
+    server_dt = dt - datetime.timedelta(minutes=timezone_offset - get_tz_offset(db_alias))
+
+    return server_dt.strftime("%Y-%m-%d %H:%M:%S")
 
 
 def lazy_class_import(obj):  # type: (Union[str, Any]) -> Any

tests/clickhouse_models.py

@@ -9,7 +9,8 @@ class TestClickHouseModel(ClickHouseModel):
     django_model = TestModel
     sync_delay = 5
 
+    id = fields.Int32Field()
     created_date = fields.DateField()
-    value = fields.UInt32Field()
+    value = fields.Int32Field()
 
     engine = MergeTree('created_Date')

tests/settings.py

@@ -34,7 +34,16 @@ INSTALLED_APPS = [
     "tests"
 ]
 
+CLICKHOUSE_DATABASES = {
+    'default': {
+        'db_name': 'test',
+        'username': 'default',
+        'password': ''
+    }
+}
+
 CLICKHOUSE_SYNC_BATCH_SIZE = 5000
+
 CLICKHOUSE_REDIS_CONFIG = {
     'host': '127.0.0.1',
     'port': 6379,

tests/test_serializers.py

@@ -0,0 +1,38 @@
+import datetime
+
+from django.test import TestCase
+
+from django_clickhouse.serializers import Django2ClickHouseModelSerializer
+from tests.clickhouse_models import TestClickHouseModel
+from tests.models import TestModel
+
+
+class Django2ClickHouseModelSerializerTest(TestCase):
+    fixtures = ['test_model']
+
+    def setUp(self):
+        self.obj = TestModel.objects.get(pk=1)
+
+    def test_all(self):
+        serializer = Django2ClickHouseModelSerializer()
+        res = serializer.serialize(self.obj, TestClickHouseModel)
+        self.assertIsInstance(res, TestClickHouseModel)
+        self.assertEqual(self.obj.id, res.id)
+        self.assertEqual(self.obj.value, res.value)
+        self.assertEqual(self.obj.created_date, res.created_date)
+
+    def test_fields(self):
+        serializer = Django2ClickHouseModelSerializer(fields=('value'))
+        res = serializer.serialize(self.obj, TestClickHouseModel)
+        self.assertIsInstance(res, TestClickHouseModel)
+        self.assertEqual(0, res.id)
+        self.assertEqual(datetime.date(1970, 1, 1), res.created_date)
+        self.assertEqual(self.obj.value, res.value)
+
+    def test_exclude_fields(self):
+        serializer = Django2ClickHouseModelSerializer(exclude_fields=('created_date',))
+        res = serializer.serialize(self.obj, TestClickHouseModel)
+        self.assertIsInstance(res, TestClickHouseModel)
+        self.assertEqual(datetime.date(1970, 1, 1), res.created_date)
+        self.assertEqual(self.obj.id, res.id)
+        self.assertEqual(self.obj.value, res.value)

tests/test_utils.py

@@ -0,0 +1,47 @@
+import datetime
+from unittest import TestCase
+
+import pytz
+
+from django_clickhouse.utils import get_tz_offset, format_datetime
+
+
+class GetTZOffsetTest(TestCase):
+    def test_func(self):
+        self.assertEqual(300, get_tz_offset())
+
+
+class FormatDateTimeTest(TestCase):
+    @staticmethod
+    def _get_zone_time(dt):
+        """
+        На момент написания тестов в РФ было какое-то странное смещение (для Москвы, например +2:30, для Перми +4:03)
+        :param dt: Объект datetime.datetime
+        :return: Строковый ожидаемый результат
+        """
+        moscow_minute_offset = dt.utcoffset().total_seconds() / 60
+        zone_h, zone_m = abs(int(moscow_minute_offset / 60)), int(moscow_minute_offset % 60)
+
+        # +5 за счет времени тестового сервера ClickHouse
+        return (dt - datetime.timedelta(hours=zone_h - 5, minutes=zone_m)).strftime("%Y-%m-%d %H:%M:%S")
+
+    def test_conversion(self):
+        dt = datetime.datetime(2017, 1, 2, 3, 4, 5)
+        self.assertEqual(format_datetime(dt), '2017-01-02 08:04:05')
+        dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.utc)
+        self.assertEqual(format_datetime(dt), '2017-01-02 08:04:05')
+        dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow'))
+        self.assertEqual(format_datetime(dt), self._get_zone_time(dt))
+        dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow'))
+        offset = int(pytz.timezone('Europe/Moscow').utcoffset(dt).total_seconds() / 60)
+        self.assertEqual(format_datetime(dt, timezone_offset=offset), '2017-01-02 03:04:05')
+
+    def test_date_conversion(self):
+        dt = datetime.date(2017, 1, 2)
+        self.assertEqual(format_datetime(dt), '2017-01-02 05:00:00')
+        dt = datetime.date(2017, 1, 2)
+        self.assertEqual(format_datetime(dt, day_end=True), '2017-01-03 04:59:59')
+        dt = datetime.date(2017, 1, 2)
+        self.assertEqual(format_datetime(dt, day_end=True, timezone_offset=60), '2017-01-03 03:59:59')
+        dt = datetime.date(2017, 1, 2)
+        self.assertEqual(format_datetime(dt, timezone_offset=60), '2017-01-02 04:00:00')