From ae253e0587fb3fdd1ac83904b7d53cc8797bb06c Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Tue, 25 Jun 2019 18:10:04 +0500
Subject: [PATCH 01/27] =?UTF-8?q?=D0=91=D0=B0=D0=B3=D1=84=D0=B8=D0=BA?=
 =?UTF-8?q?=D1=81=D1=8B=20=D1=81=D0=BE=D0=B2=D0=BC=D0=B5=D1=81=D1=82=D0=B8?=
 =?UTF-8?q?=D0=BC=D0=BE=D1=81=D1=82=D0=B8=20=D1=81=20python=203.7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/django_clickhouse/clickhouse_models.py |  9 +++++++--
 src/django_clickhouse/compatibility.py     | 10 ++--------
 src/django_clickhouse/utils.py             |  2 +-
 tests/test_compatibility.py                | 12 ++++++------
 4 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py
index 2d11d37..80b0446 100644
--- a/src/django_clickhouse/clickhouse_models.py
+++ b/src/django_clickhouse/clickhouse_models.py
@@ -67,7 +67,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
         field_names = field_names or cls.fields(writable=False).keys()
 
         # Strange, but sometimes the columns are in different order...
-        field_names = tuple(sorted(field_names))
+        field_names = sorted(field_names)
 
         if defaults:
             defaults_new = deepcopy(cls._defaults)
@@ -75,7 +75,12 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
         else:
             defaults_new = cls._defaults
 
-        return namedtuple("%sTuple" % cls.__name__, field_names, defaults=defaults_new)
+        # defaults should be rightmost arguments
+        required_field_names = tuple(name for name in field_names if name not in defaults_new)
+
+        default_field_names, default_values = zip(*defaults_new.items())
+
+        return namedtuple("%sTuple" % cls.__name__, required_field_names + default_field_names, defaults=default_values)
 
     @classmethod
     def objects_in(cls, database):  # type: (Database) -> QuerySet
diff --git a/src/django_clickhouse/compatibility.py b/src/django_clickhouse/compatibility.py
index 8213afe..b35ce14 100644
--- a/src/django_clickhouse/compatibility.py
+++ b/src/django_clickhouse/compatibility.py
@@ -1,8 +1,5 @@
 import sys
-from collections import namedtuple as basenamedtuple, Mapping
-from functools import lru_cache
-
-from copy import deepcopy
+from collections import namedtuple as basenamedtuple
 
 
 def namedtuple(*args, **kwargs):
@@ -16,10 +13,7 @@ def namedtuple(*args, **kwargs):
         defaults = kwargs.pop('defaults', {})
         TupleClass = basenamedtuple(*args, **kwargs)
         TupleClass.__new__.__defaults__ = (None,) * len(TupleClass._fields)
-        if isinstance(defaults, Mapping):
-            prototype = TupleClass(**defaults)
-        else:
-            prototype = TupleClass(*defaults)
+        prototype = TupleClass(*defaults)
         TupleClass.__new__.__defaults__ = tuple(prototype)
         return TupleClass
     else:
diff --git a/src/django_clickhouse/utils.py b/src/django_clickhouse/utils.py
index 0ee57ec..a279d3a 100644
--- a/src/django_clickhouse/utils.py
+++ b/src/django_clickhouse/utils.py
@@ -159,7 +159,7 @@ def int_ranges(items: Iterable[int]) -> Iterator[Tuple[int, int]]:
             yield interval
 
     if interval_start is None:
-        raise StopIteration()
+        return
     else:
         yield interval_start, prev_item
 
diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py
index e652ae0..0c0cd30 100644
--- a/tests/test_compatibility.py
+++ b/tests/test_compatibility.py
@@ -1,17 +1,17 @@
-from unittest import TestCase
+from django.test import TestCase
 
 from django_clickhouse.compatibility import namedtuple
 
 
 class NamedTupleTest(TestCase):
     def test_defaults(self):
-        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults={'c': 3})
+        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults=[3])
         self.assertTupleEqual((1, 2, 3), tuple(TestTuple(1, b=2)))
         self.assertTupleEqual((1, 2, 4), tuple(TestTuple(1, 2, 4)))
         self.assertTupleEqual((1, 2, 4), tuple(TestTuple(a=1, b=2, c=4)))
 
     def test_exceptions(self):
-        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults={'c': 3})
+        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults=[3])
 
         # BUG On python < 3.7 this error is not raised, as not given defaults are filled by None
         # with self.assertRaises(TypeError):
@@ -22,8 +22,8 @@ class NamedTupleTest(TestCase):
 
     def test_different_defaults(self):
         # Test that 2 tuple type defaults don't affect each other
-        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults={'c': 3})
-        OtherTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults={'c': 4})
+        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults=[3])
+        OtherTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults=[4])
         t1 = TestTuple(a=1, b=2)
         t2 = OtherTuple(a=3, b=4)
         self.assertTupleEqual((1, 2, 3), tuple(t1))
@@ -31,7 +31,7 @@ class NamedTupleTest(TestCase):
 
     def test_defaults_cache(self):
         # Test that 2 tuple instances don't affect each other's defaults
-        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults={'c': 3})
+        TestTuple = namedtuple('TestTuple', ('a', 'b', 'c'), defaults=[3])
         self.assertTupleEqual((1, 2, 4), tuple(TestTuple(a=1, b=2, c=4)))
         self.assertTupleEqual((1, 2, 3), tuple(TestTuple(a=1, b=2)))
 

From fafcbe4c9336463ae86eb501292ad17c0868b49f Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 26 Jun 2019 12:24:46 +0500
Subject: [PATCH 02/27] Fixed python 3.7 compatibility issues

---
 src/django_clickhouse/clickhouse_models.py | 10 ++++----
 src/django_clickhouse/database.py          |  6 +++--
 tests/clickhouse_models.py                 |  2 +-
 tests/fixtures/test_model.json             | 27 ++++++++++++++++++++++
 tests/test_database.py                     | 16 +++++++++++++
 tests/test_models.py                       | 14 +++++------
 6 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py
index 80b0446..967b59f 100644
--- a/src/django_clickhouse/clickhouse_models.py
+++ b/src/django_clickhouse/clickhouse_models.py
@@ -66,9 +66,6 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
     def get_tuple_class(cls, field_names=None, defaults=None):
         field_names = field_names or cls.fields(writable=False).keys()
 
-        # Strange, but sometimes the columns are in different order...
-        field_names = sorted(field_names)
-
         if defaults:
             defaults_new = deepcopy(cls._defaults)
             defaults_new.update(defaults)
@@ -78,9 +75,12 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
         # defaults should be rightmost arguments
         required_field_names = tuple(name for name in field_names if name not in defaults_new)
 
-        default_field_names, default_values = zip(*defaults_new.items())
+        default_field_names, default_values = zip(*sorted(defaults_new.items(), key=lambda t: t[0]))
 
-        return namedtuple("%sTuple" % cls.__name__, required_field_names + default_field_names, defaults=default_values)
+        # Strange, but sometimes the columns are in different order...
+        field_names = tuple(sorted(required_field_names)) + default_field_names
+
+        return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values)
 
     @classmethod
     def objects_in(cls, database):  # type: (Database) -> QuerySet
diff --git a/src/django_clickhouse/database.py b/src/django_clickhouse/database.py
index ee96995..8c6c930 100644
--- a/src/django_clickhouse/database.py
+++ b/src/django_clickhouse/database.py
@@ -88,11 +88,13 @@ class Database(InfiDatabase):
 
         fields_list = ','.join('`%s`' % name for name in first_tuple._fields)
         fields_dict = model_class.fields(writable=True)
-        fields = [fields_dict[name] for name in first_tuple._fields]
         statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__)
 
         def tuple_to_csv(tup):
-            return '\t'.join(field.to_db_string(val, quote=False) for field, val in zip(fields, tup)) + '\n'
+            return '\t'.join(
+                fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
+                for field_name in first_tuple._fields
+            ) + '\n'
 
         def gen():
             buf = BytesIO()
diff --git a/tests/clickhouse_models.py b/tests/clickhouse_models.py
index 541f33d..77893a3 100644
--- a/tests/clickhouse_models.py
+++ b/tests/clickhouse_models.py
@@ -12,7 +12,7 @@ class ClickHouseTestModel(ClickHouseModel):
 
     id = fields.Int32Field()
     created_date = fields.DateField()
-    value = fields.Int32Field()
+    value = fields.Int32Field(default=100500)
     str_field = fields.StringField()
 
     engine = ReplacingMergeTree('created_date', ('id',))
diff --git a/tests/fixtures/test_model.json b/tests/fixtures/test_model.json
index 35a1145..aaf7400 100644
--- a/tests/fixtures/test_model.json
+++ b/tests/fixtures/test_model.json
@@ -16,5 +16,32 @@
       "created_date": "2018-02-01",
       "created": "2018-02-01 00:00:00"
     }
+  },
+  {
+    "model": "tests.TestModel",
+    "pk": 3,
+    "fields": {
+      "value": 300,
+      "created_date": "2018-03-01",
+      "created": "2018-03-01 00:00:00"
+    }
+  },
+  {
+    "model": "tests.TestModel",
+    "pk": 4,
+    "fields": {
+      "value": 400,
+      "created_date": "2018-04-01",
+      "created": "2018-04-01 00:00:00"
+    }
+  },
+  {
+    "model": "tests.TestModel",
+    "pk": 5,
+    "fields": {
+      "value": 500,
+      "created_date": "2018-05-01",
+      "created": "2018-05-01 00:00:00"
+    }
   }
 ]
\ No newline at end of file
diff --git a/tests/test_database.py b/tests/test_database.py
index 4549131..120a3b8 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -31,6 +31,22 @@ class CollapsingMergeTreeTest(TestCase):
             'str_field': str(i)
         } for i in range(10)], [item.to_dict() for item in qs])
 
+    def test_insert_tuples_defaults(self):
+        tuple_class = ClickHouseTestModel.get_tuple_class(defaults={'created_date': date.today()})
+        data = [
+            tuple_class(id=i, str_field=str(i))
+            for i in range(10)
+        ]
+        self.db.insert_tuples(ClickHouseTestModel, data)
+
+        qs = ClickHouseTestModel.objects.order_by('id').all()
+        self.assertListEqual([{
+            'id': i,
+            'created_date': date.today(),
+            'value': 100500,
+            'str_field': str(i)
+        } for i in range(10)], [item.to_dict() for item in qs])
+
     def test_insert_tuples_batch_size(self):
         tuple_class = ClickHouseTestModel.get_tuple_class()
         data = [
diff --git a/tests/test_models.py b/tests/test_models.py
index 4202a9d..0de5c92 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -28,6 +28,7 @@ class TestOperations(TransactionTestCase):
     def setUp(self):
         self.storage = self.django_model.get_clickhouse_storage()
         self.storage.flush()
+        self.before_op_items = list(self.django_model.objects.all())
 
     def tearDown(self):
         self.storage.flush()
@@ -56,8 +57,8 @@ class TestOperations(TransactionTestCase):
                  for i in range(5)]
         items = self.django_model.objects.bulk_create(items)
         self.assertEqual(5, len(items))
-        self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items],
-                             self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
+        self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items},
+                            set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)))
 
     def test_get_or_create(self):
         instance, created = self.django_model.objects. \
@@ -96,10 +97,9 @@ class TestOperations(TransactionTestCase):
         self.assertListEqual([('update', "%s.1" % self.db_alias)],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
-        # Update, after which updated element will not suit update conditions
         self.django_model.objects.filter(created_date__lt=datetime.date.today()). \
             update(created_date=datetime.date.today())
-        self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)],
+        self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_qs_update_returning(self):
@@ -110,7 +110,7 @@ class TestOperations(TransactionTestCase):
         # Update, after which updated element will not suit update conditions
         self.django_model.objects.filter(created_date__lt=datetime.date.today()). \
             update_returning(created_date=datetime.date.today())
-        self.assertListEqual([('update', "%s.1" % self.db_alias), ('update', "%s.2" % self.db_alias)],
+        self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_qs_delete_returning(self):
@@ -118,9 +118,9 @@ class TestOperations(TransactionTestCase):
         self.assertListEqual([('delete', "%s.1" % self.db_alias)],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
-        # Update, после которого исходный фильтр уже не сработает
+        # Delete, after which updated element will not suit update conditions
         self.django_model.objects.filter(created_date__lt=datetime.date.today()).delete_returning()
-        self.assertListEqual([('delete', "%s.1" % self.db_alias), ('delete', "%s.2" % self.db_alias)],
+        self.assertListEqual([('delete', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
     def test_delete(self):

From 73bcaf29f09e4f6173b231b073a097ea31db1030 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 26 Jun 2019 18:27:57 +0500
Subject: [PATCH 03/27] Travis testing

---
 .travis.yml | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..dc70795
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,76 @@
+dist: xenial
+sudo: required
+language: python
+cache:
+  pip: true
+  apt: true
+
+services:
+  - postgresql
+  - rabbitmq
+  - redis-server
+addons:
+  postgresql: "11"
+  apt:
+    sources:
+      - sourceline: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
+    packages:
+      - dirmngr
+      - apt-transport-https
+      - postgresql-contrib-9.6
+      - postgresql-10
+      - postgresql-contrib-10
+      - postgresql-client-10
+      - postgresql-11
+      - postgresql-contrib-11
+      - postgresql-client-11
+      - unzip
+
+python:
+  - 3.6
+  - 3.7
+
+env:
+  - PG=9.6 CLICKHOUSE=19.4
+  - PG=10 CLICKHOUSE=19.4
+  - PG=11 CLICKHOUSE=19.4
+
+before_install:
+   # Use default PostgreSQL 11 port
+   - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf
+   - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf
+
+  # Start PostgreSQL version we need
+  - sudo systemctl stop postgresql && sudo systemctl start postgresql@$PG-main
+
+  # ClickHouse sources
+  - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
+  - sudo apt-get update
+
+
+install:
+  # Install ClickHouse
+  - sudo apt-get install clickhouse-client=$CLICKHOUSE.* clickhouse-server=$CLICKHOUSE.* clickhouse-common-static=$CLICKHOUSE.*
+  - sudo service clickhouse-server restart
+
+  - pip install -r requirements.conf
+  - python setup.py -q install
+
+before_script:
+  # Output software versions
+  - erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().'  -noshell
+  - rabbitmqctl status | grep "RabbitMQ"
+  - clickhouse-client --query "SELECT version();"
+  - psql -tc 'SHOW server_version' -U postgres
+
+  - psql -tc 'SHOW server_version' -U postgres
+  - psql -c 'CREATE ROLE test;' -U postgres
+  - psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres
+  - psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres
+  - psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres
+  - psql -c 'CREATE DATABASE test OWNER test;' -U postgres
+  - psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres
+  - psql -c 'CREATE DATABASE test_test OWNER test;' -U postgres
+
+script:
+  python runtests.py

From 04c5ba292a148088cf4c599bdc7f5a098c230d4c Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 16:41:25 +0500
Subject: [PATCH 04/27] Bugfix in installing migrations

---
 src/django_clickhouse/migrations.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/django_clickhouse/migrations.py b/src/django_clickhouse/migrations.py
index 7d2f839..5227baa 100644
--- a/src/django_clickhouse/migrations.py
+++ b/src/django_clickhouse/migrations.py
@@ -119,12 +119,11 @@ class MigrationHistory(ClickHouseModel):
         :return: None
         """
         # Ensure that table for migration storing is created
-        for db_alias in cls.migrate_non_replicated_db_aliases:
-            connections[db_alias].create_table(cls)
+        for db_name in cls.migrate_non_replicated_db_aliases:
+            connections[db_name].create_table(cls)
 
-        cls.objects.bulk_create([
-            cls(db_alias=db_alias, package_name=migrations_package, module_name=name, applied=datetime.date.today())
-        ])
+        cls.objects.create(db_alias=db_alias, package_name=migrations_package, module_name=name,
+                           applied=datetime.date.today())
 
     @classmethod
     def get_applied_migrations(cls, db_alias, migrations_package):  # type: (str, str) -> Set[str]

From b5c0567c78cfd85412b707b0f3403d4f3af66eef Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 16:51:26 +0500
Subject: [PATCH 05/27] Bugfix in installing migrations

---
 .travis.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dc70795..0499430 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,9 +31,9 @@ python:
   - 3.7
 
 env:
-  - PG=9.6 CLICKHOUSE=19.4
-  - PG=10 CLICKHOUSE=19.4
-  - PG=11 CLICKHOUSE=19.4
+  - PG=9.6 CLICKHOUSE=19.4 DJANGO=2.1
+  - PG=10 CLICKHOUSE=19.4 DJANGO=2.1
+  - PG=11 CLICKHOUSE=19.4 DJANGO=2.1
 
 before_install:
    # Use default PostgreSQL 11 port
@@ -53,7 +53,8 @@ install:
   - sudo apt-get install clickhouse-client=$CLICKHOUSE.* clickhouse-server=$CLICKHOUSE.* clickhouse-common-static=$CLICKHOUSE.*
   - sudo service clickhouse-server restart
 
-  - pip install -r requirements.conf
+  - pip install -r requirements.txt
+  - pip install -q Django==$DJANGO.*
   - python setup.py -q install
 
 before_script:

From 29988649f54923b6704cd0077a86adb2149a20b2 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 16:58:22 +0500
Subject: [PATCH 06/27] Bugfix in config

---
 .travis.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0499430..514f2d4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -36,9 +36,9 @@ env:
   - PG=11 CLICKHOUSE=19.4 DJANGO=2.1
 
 before_install:
-   # Use default PostgreSQL 11 port
-   - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf
-   - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf
+  # Use default PostgreSQL 11 port
+  - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf
+  - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf
 
   # Start PostgreSQL version we need
   - sudo systemctl stop postgresql && sudo systemctl start postgresql@$PG-main

From 49a3018d03da167b808ce7edf628650a9e6428ad Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 17:04:36 +0500
Subject: [PATCH 07/27] Bugfix in config

---
 .travis.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 514f2d4..684c900 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-dist: xenial
+dist: trusty
 sudo: required
 language: python
 cache:
@@ -34,6 +34,9 @@ env:
   - PG=9.6 CLICKHOUSE=19.4 DJANGO=2.1
   - PG=10 CLICKHOUSE=19.4 DJANGO=2.1
   - PG=11 CLICKHOUSE=19.4 DJANGO=2.1
+  - PG=9.6 CLICKHOUSE=19.4 DJANGO=2.2
+  - PG=10 CLICKHOUSE=19.4 DJANGO=2.2
+  - PG=11 CLICKHOUSE=19.4 DJANGO=2.2
 
 before_install:
   # Use default PostgreSQL 11 port

From 1093dfdc047857449f72c3fb5513d1dd3409be36 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 17:08:19 +0500
Subject: [PATCH 08/27] Bugfix in config

---
 .travis.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 684c900..68a2e47 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,12 +17,9 @@ addons:
     packages:
       - dirmngr
       - apt-transport-https
-      - postgresql-contrib-9.6
       - postgresql-10
-      - postgresql-contrib-10
       - postgresql-client-10
       - postgresql-11
-      - postgresql-contrib-11
       - postgresql-client-11
       - unzip
 

From 5a01aab3346300e2e0b569fd0ce80b364f09b136 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 17:12:39 +0500
Subject: [PATCH 09/27] Bugfix in config

---
 .travis.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 68a2e47..aedc261 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-dist: trusty
+dist: xenial
 sudo: required
 language: python
 cache:
@@ -7,21 +7,28 @@ cache:
 
 services:
   - postgresql
-  - rabbitmq
   - redis-server
 addons:
   postgresql: "11"
   apt:
     sources:
       - sourceline: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
+      - sourceline: "deb https://packages.erlang-solutions.com/ubuntu xenial contrib"
+        key_url: "https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc"
+      - sourceline: "deb https://dl.bintray.com/rabbitmq/debian xenial main"
+        key_url: "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc"
     packages:
       - dirmngr
       - apt-transport-https
+      - postgresql-contrib-9.6
       - postgresql-10
+      - postgresql-contrib-10
       - postgresql-client-10
       - postgresql-11
+      - postgresql-contrib-11
       - postgresql-client-11
       - unzip
+      - rabbitmq-server
 
 python:
   - 3.6

From 3c5f705ef5a4263321532b408b40b73b6cf7becc Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 17:14:32 +0500
Subject: [PATCH 10/27] Test on latest clickhouse version

---
 .travis.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index aedc261..74bad29 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -35,12 +35,12 @@ python:
   - 3.7
 
 env:
-  - PG=9.6 CLICKHOUSE=19.4 DJANGO=2.1
-  - PG=10 CLICKHOUSE=19.4 DJANGO=2.1
-  - PG=11 CLICKHOUSE=19.4 DJANGO=2.1
-  - PG=9.6 CLICKHOUSE=19.4 DJANGO=2.2
-  - PG=10 CLICKHOUSE=19.4 DJANGO=2.2
-  - PG=11 CLICKHOUSE=19.4 DJANGO=2.2
+  - PG=9.6 DJANGO=2.1
+  - PG=10 DJANGO=2.1
+  - PG=11 DJANGO=2.1
+  - PG=9.6 DJANGO=2.2
+  - PG=10 DJANGO=2.2
+  - PG=11 DJANGO=2.2
 
 before_install:
   # Use default PostgreSQL 11 port
@@ -57,7 +57,7 @@ before_install:
 
 install:
   # Install ClickHouse
-  - sudo apt-get install clickhouse-client=$CLICKHOUSE.* clickhouse-server=$CLICKHOUSE.* clickhouse-common-static=$CLICKHOUSE.*
+  - sudo apt-get install clickhouse-client clickhouse-server clickhouse-common-static
   - sudo service clickhouse-server restart
 
   - pip install -r requirements.txt

From 2f4dbdc151e352697dc7eae695d26d146345e8b5 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 17:18:42 +0500
Subject: [PATCH 11/27] Test on latest clickhouse version

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 74bad29..0a434e9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -78,7 +78,6 @@ before_script:
   - psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres
   - psql -c 'CREATE DATABASE test OWNER test;' -U postgres
   - psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres
-  - psql -c 'CREATE DATABASE test_test OWNER test;' -U postgres
 
 script:
   python runtests.py

From 8a581f27dbeeea45dd9ae21d9669f868a96cf1a4 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Thu, 27 Jun 2019 17:22:40 +0500
Subject: [PATCH 12/27] Install not required redis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 0a434e9..dade7ce 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -62,6 +62,7 @@ install:
 
   - pip install -r requirements.txt
   - pip install -q Django==$DJANGO.*
+  - pip install redis
   - python setup.py -q install
 
 before_script:

From 355b8c73212ea2e7fbd8e287470f2a94c6618284 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 28 Jun 2019 11:09:17 +0500
Subject: [PATCH 13/27] Fix python 3.6 namedtuple compatibility

---
 src/django_clickhouse/compatibility.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/django_clickhouse/compatibility.py b/src/django_clickhouse/compatibility.py
index b35ce14..940d398 100644
--- a/src/django_clickhouse/compatibility.py
+++ b/src/django_clickhouse/compatibility.py
@@ -10,11 +10,9 @@ def namedtuple(*args, **kwargs):
     :return: namedtuple class
     """
     if sys.version_info < (3, 7):
-        defaults = kwargs.pop('defaults', {})
+        defaults = kwargs.pop('defaults', ())
         TupleClass = basenamedtuple(*args, **kwargs)
-        TupleClass.__new__.__defaults__ = (None,) * len(TupleClass._fields)
-        prototype = TupleClass(*defaults)
-        TupleClass.__new__.__defaults__ = tuple(prototype)
+        TupleClass.__new__.__defaults__ = (None,) * (len(TupleClass._fields) - len(defaults)) + tuple(defaults)
         return TupleClass
     else:
         return basenamedtuple(*args, **kwargs)

From a3e8d71bd86374f90ad063d14a9e56152c4e8afd Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 28 Jun 2019 11:29:15 +0500
Subject: [PATCH 14/27] Fix issue
 https://github.com/carrotquest/django-clickhouse/issues/1

---
 tests/test_utils.py | 38 ++++++++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/tests/test_utils.py b/tests/test_utils.py
index 5afb27b..d001306 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,5 +1,5 @@
+import calendar
 import datetime
-import time
 from queue import Queue
 from time import gmtime, localtime
 
@@ -12,9 +12,27 @@ from django_clickhouse.utils import get_tz_offset, format_datetime, lazy_class_i
     SingletonMeta
 
 
+def system_tz_offset():  # type: () -> int
+    """
+    ClickHouse timezone is equal to system zone offset in seconds.
+    THis function gets system timezone
+    :return: Time zone offset in minutes
+    """
+    return int((calendar.timegm(gmtime()) - calendar.timegm(localtime())) / 60)
+
+
+def local_dt_str(dt) -> str:
+    """
+    Returns string representation of an aware datetime object, localized by adding system_tz_offset()
+    :param dt: Datetime to change
+    :return: Formatted string
+    """
+    return (dt + datetime.timedelta(minutes=system_tz_offset())).strftime('%Y-%m-%d %H:%M:%S')
+
+
 class GetTZOffsetTest(TestCase):
     def test_func(self):
-        self.assertEqual(300, get_tz_offset())
+        self.assertEqual(system_tz_offset(), get_tz_offset())
 
 
 class FormatDateTimeTest(TestCase):
@@ -28,14 +46,13 @@ class FormatDateTimeTest(TestCase):
         moscow_minute_offset = dt.utcoffset().total_seconds() / 60
         zone_h, zone_m = abs(int(moscow_minute_offset / 60)), int(moscow_minute_offset % 60)
 
-        # +5 за счет времени тестового сервера ClickHouse
-        return (dt - datetime.timedelta(hours=zone_h - 5, minutes=zone_m)).strftime("%Y-%m-%d %H:%M:%S")
+        return local_dt_str(dt - datetime.timedelta(hours=zone_h, minutes=zone_m))
 
     def test_conversion(self):
         dt = datetime.datetime(2017, 1, 2, 3, 4, 5)
-        self.assertEqual(format_datetime(dt), '2017-01-02 08:04:05')
+        self.assertEqual(format_datetime(dt), local_dt_str(dt))
         dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.utc)
-        self.assertEqual(format_datetime(dt), '2017-01-02 08:04:05')
+        self.assertEqual(format_datetime(dt), local_dt_str(dt))
         dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow'))
         self.assertEqual(format_datetime(dt), self._get_zone_time(dt))
         dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow'))
@@ -44,13 +61,14 @@ class FormatDateTimeTest(TestCase):
 
     def test_date_conversion(self):
         dt = datetime.date(2017, 1, 2)
-        self.assertEqual(format_datetime(dt), '2017-01-02 05:00:00')
+        self.assertEqual(format_datetime(dt), local_dt_str(datetime.datetime(2017, 1, 2, 0, 0, 0)))
         dt = datetime.date(2017, 1, 2)
-        self.assertEqual(format_datetime(dt, day_end=True), '2017-01-03 04:59:59')
+        self.assertEqual(format_datetime(dt, day_end=True), local_dt_str(datetime.datetime(2017, 1, 2, 23, 59, 59)))
         dt = datetime.date(2017, 1, 2)
-        self.assertEqual(format_datetime(dt, day_end=True, timezone_offset=60), '2017-01-03 03:59:59')
+        self.assertEqual(format_datetime(dt, day_end=True, timezone_offset=60),
+                         local_dt_str(datetime.datetime(2017, 1, 2, 22, 59, 59)))
         dt = datetime.date(2017, 1, 2)
-        self.assertEqual(format_datetime(dt, timezone_offset=60), '2017-01-02 04:00:00')
+        self.assertEqual(format_datetime(dt, timezone_offset=60), local_dt_str(datetime.datetime(2017, 1, 1, 23, 0, 0)))
 
 
 class TestLazyClassImport(TestCase):

From e99c6a1db3811b760eb1bdcd984fac12d6712298 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 28 Jun 2019 12:21:08 +0500
Subject: [PATCH 15/27] Removed test on clickhouse timezone offset as it
 doesn't work in different time zones

---
 tests/test_utils.py | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/tests/test_utils.py b/tests/test_utils.py
index d001306..e66c1a6 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,7 +1,5 @@
-import calendar
 import datetime
 from queue import Queue
-from time import gmtime, localtime
 
 import pytz
 from django.test import TestCase
@@ -12,27 +10,13 @@ from django_clickhouse.utils import get_tz_offset, format_datetime, lazy_class_i
     SingletonMeta
 
 
-def system_tz_offset():  # type: () -> int
-    """
-    ClickHouse timezone is equal to system zone offset in seconds.
-    THis function gets system timezone
-    :return: Time zone offset in minutes
-    """
-    return int((calendar.timegm(gmtime()) - calendar.timegm(localtime())) / 60)
-
-
 def local_dt_str(dt) -> str:
     """
     Returns string representation of an aware datetime object, localized by adding system_tz_offset()
     :param dt: Datetime to change
     :return: Formatted string
     """
-    return (dt + datetime.timedelta(minutes=system_tz_offset())).strftime('%Y-%m-%d %H:%M:%S')
-
-
-class GetTZOffsetTest(TestCase):
-    def test_func(self):
-        self.assertEqual(system_tz_offset(), get_tz_offset())
+    return (dt + datetime.timedelta(minutes=get_tz_offset())).strftime('%Y-%m-%d %H:%M:%S')
 
 
 class FormatDateTimeTest(TestCase):

From 8b9e2332dba3a4dcf3b6d9fef1c2dc68f6f82e88 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 28 Jun 2019 13:10:29 +0500
Subject: [PATCH 16/27] Searching bug in datetime

---
 src/django_clickhouse/utils.py | 1 +
 tests/test_utils.py            | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/django_clickhouse/utils.py b/src/django_clickhouse/utils.py
index a279d3a..fb4a044 100644
--- a/src/django_clickhouse/utils.py
+++ b/src/django_clickhouse/utils.py
@@ -54,6 +54,7 @@ def format_datetime(dt, timezone_offset=0, day_end=False, db_alias=None):
 
     # Dates in ClickHouse are parsed in server local timezone. So I need to add server timezone
     server_dt = dt - datetime.timedelta(minutes=timezone_offset - get_tz_offset(db_alias))
+    print(dt, timezone_offset, get_tz_offset(db_alias))
 
     return server_dt.strftime("%Y-%m-%d %H:%M:%S")
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e66c1a6..a830469 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -27,8 +27,8 @@ class FormatDateTimeTest(TestCase):
         :param dt: Объект datetime.datetime
         :return: Строковый ожидаемый результат
         """
-        moscow_minute_offset = dt.utcoffset().total_seconds() / 60
-        zone_h, zone_m = abs(int(moscow_minute_offset / 60)), int(moscow_minute_offset % 60)
+        minute_offset = dt.utcoffset().total_seconds() / 60
+        zone_h, zone_m = abs(int(minute_offset / 60)), int(minute_offset % 60)
 
         return local_dt_str(dt - datetime.timedelta(hours=zone_h, minutes=zone_m))
 

From 37712b4156ecfa40f38e51acd34110ac5ed5e6a0 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 28 Jun 2019 13:33:22 +0500
Subject: [PATCH 17/27] Fixing test bug

---
 src/django_clickhouse/utils.py | 1 -
 tests/test_utils.py            | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/django_clickhouse/utils.py b/src/django_clickhouse/utils.py
index fb4a044..a279d3a 100644
--- a/src/django_clickhouse/utils.py
+++ b/src/django_clickhouse/utils.py
@@ -54,7 +54,6 @@ def format_datetime(dt, timezone_offset=0, day_end=False, db_alias=None):
 
     # Dates in ClickHouse are parsed in server local timezone. So I need to add server timezone
     server_dt = dt - datetime.timedelta(minutes=timezone_offset - get_tz_offset(db_alias))
-    print(dt, timezone_offset, get_tz_offset(db_alias))
 
     return server_dt.strftime("%Y-%m-%d %H:%M:%S")
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index a830469..bcbe3a4 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -41,7 +41,8 @@ class FormatDateTimeTest(TestCase):
         self.assertEqual(format_datetime(dt), self._get_zone_time(dt))
         dt = datetime.datetime(2017, 1, 2, 3, 4, 5, tzinfo=pytz.timezone('Europe/Moscow'))
         offset = int(pytz.timezone('Europe/Moscow').utcoffset(dt).total_seconds() / 60)
-        self.assertEqual(format_datetime(dt, timezone_offset=offset), '2017-01-02 03:04:05')
+        self.assertEqual(format_datetime(dt, timezone_offset=offset),
+                         local_dt_str(datetime.datetime(2017, 1, 2, 3, 4, 5) - datetime.timedelta(minutes=offset*2)))
 
     def test_date_conversion(self):
         dt = datetime.date(2017, 1, 2)

From 85a785eb595644b5925aafaee9d132274bd4c38d Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Mon, 15 Jul 2019 16:56:16 +0500
Subject: [PATCH 18/27] Opportunity to pass database data as strings without
 formatting

---
 src/django_clickhouse/clickhouse_models.py |  5 +++--
 src/django_clickhouse/database.py          | 23 +++++++++++++---------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py
index 967b59f..1bb28e7 100644
--- a/src/django_clickhouse/clickhouse_models.py
+++ b/src/django_clickhouse/clickhouse_models.py
@@ -58,6 +58,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
     sync_storage = None
     sync_delay = None
     sync_lock_timeout = None
+    sync_formatted_tuples = False
 
     # This attribute is initialized in metaclass, as it must get model class as a parameter
     objects = None  # type: QuerySet
@@ -204,11 +205,11 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
     def insert_batch(cls, batch):
         """
         Inserts batch into database
-        :param batch:
+        :param batch: Batch of tuples to insert
         :return:
         """
         if batch:
-            cls.get_database(for_write=True).insert_tuples(cls, batch)
+            cls.get_database(for_write=True).insert_tuples(cls, batch, formatted=cls.sync_formatted_tuples)
 
     @classmethod
     def sync_batch_from_storage(cls):
diff --git a/src/django_clickhouse/database.py b/src/django_clickhouse/database.py
index 8c6c930..6abc516 100644
--- a/src/django_clickhouse/database.py
+++ b/src/django_clickhouse/database.py
@@ -67,13 +67,14 @@ class Database(InfiDatabase):
 
                 yield item
 
-    def insert_tuples(self, model_class, model_tuples, batch_size=None):
-        # type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int]) -> None
+    def insert_tuples(self, model_class, model_tuples, batch_size=None, formatted=False):
+        # type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int], bool) -> None
         """
         Inserts model_class namedtuples
         :param model_class: Clickhouse model, namedtuples are made from
         :param model_tuples: An iterable of tuples to insert
         :param batch_size: Size of batch
+        :param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string
         :return: None
         """
         tuples_iterator = iter(model_tuples)
@@ -90,17 +91,21 @@ class Database(InfiDatabase):
         fields_dict = model_class.fields(writable=True)
         statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__)
 
+        query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
+                % (self.db_name, model_class.table_name(), fields_list)
+        query_enc = query.encode('utf-8')
+
         def tuple_to_csv(tup):
-            return '\t'.join(
-                fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
-                for field_name in first_tuple._fields
-            ) + '\n'
+            if formatted:
+                str_gen = (getattr(tup, field_name) for field_name in first_tuple._fields)
+            else:
+                str_gen = (fields_dict[field_name].to_db_string(getattr(tup, field_name), quote=False)
+                           for field_name in first_tuple._fields)
+
+            return '%s\n' % '\t'.join(str_gen)
 
         def gen():
             buf = BytesIO()
-            query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
-                    % (self.db_name, model_class.table_name(), fields_list)
-            query_enc = query.encode('utf-8')
             buf.write(query_enc)
             buf.write(tuple_to_csv(first_tuple).encode('utf-8'))
 

From 842f43b24255675521b34a157313dd824165798c Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Mon, 15 Jul 2019 17:43:36 +0500
Subject: [PATCH 19/27] Added method to flush single import_key in storage

---
 src/django_clickhouse/clickhouse_models.py |  2 ++
 src/django_clickhouse/storages.py          | 10 ++++++++++
 2 files changed, 12 insertions(+)

diff --git a/src/django_clickhouse/clickhouse_models.py b/src/django_clickhouse/clickhouse_models.py
index 1bb28e7..1b34c8d 100644
--- a/src/django_clickhouse/clickhouse_models.py
+++ b/src/django_clickhouse/clickhouse_models.py
@@ -58,6 +58,8 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
     sync_storage = None
     sync_delay = None
     sync_lock_timeout = None
+
+    # This flag gives ability to disable to_db_string while inserting data, if it is already formatted
     sync_formatted_tuples = False
 
     # This attribute is initialized in metaclass, as it must get model class as a parameter
diff --git a/src/django_clickhouse/storages.py b/src/django_clickhouse/storages.py
index e16a5ac..84720a4 100644
--- a/src/django_clickhouse/storages.py
+++ b/src/django_clickhouse/storages.py
@@ -280,6 +280,16 @@ class RedisStorage(with_metaclass(SingletonMeta, Storage)):
                 key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, model.get_import_key())
                 statsd.gauge(key, 0)
 
+    def flush_import_key(self, import_key):
+        keys = [
+            self.REDIS_KEY_RANK_TEMPLATE.format(import_key=import_key),
+            self.REDIS_KEY_OPS_TEMPLATE.format(import_key=import_key),
+            self.REDIS_KEY_LOCK.format(import_key=import_key),
+            self.REDIS_KEY_LAST_SYNC_TS.format(import_key=import_key)
+        ]
+        self._redis.delete(*keys)
+        statsd.gauge("%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key), 0)
+
     def get_last_sync_time(self, import_key):
         sync_ts_key = self.REDIS_KEY_LAST_SYNC_TS.format(import_key=import_key)
         res = self._redis.get(sync_ts_key)

From 635f3d9a42f0a0c46a3c8f7308a2243eb8e15362 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Tue, 23 Jul 2019 15:27:49 +0500
Subject: [PATCH 20/27] Fixed setup.py requirements

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 9c600bc..25b64f4 100644
--- a/setup.py
+++ b/setup.py
@@ -23,5 +23,5 @@ setup(
     description='Django extension to integrate with ClickHouse database',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    # requires=requires
+    requires=requires
 )

From a51cc81db216419451accbbd39a32edc47825aba Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 24 Jul 2019 09:52:54 +0500
Subject: [PATCH 21/27] Added method to flush single import_key in storage

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 1283c62..d48e96f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ pytz
 six
 typing
 psycopg2
-infi.clickhouse-orm
+infi.clickhouse-orm (>= 1.0)
 celery
 statsd
 django-pg-returning
\ No newline at end of file

From 446494af925a8297d93705f083cdd1f665d43e02 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 24 Jul 2019 09:57:01 +0500
Subject: [PATCH 22/27] Added method to flush single import_key in storage

---
 requirements.txt | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index d48e96f..1283c62 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ pytz
 six
 typing
 psycopg2
-infi.clickhouse-orm (>= 1.0)
+infi.clickhouse-orm
 celery
 statsd
 django-pg-returning
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 25b64f4..c20a379 100644
--- a/setup.py
+++ b/setup.py
@@ -23,5 +23,5 @@ setup(
     description='Django extension to integrate with ClickHouse database',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    requires=requires
+    install_requires=requires
 )

From 5f218d0da0761d90574f1049cf9bedc0b0a6557d Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 6 Sep 2019 11:49:00 +0500
Subject: [PATCH 23/27] =?UTF-8?q?=D0=A4=D0=B8=D0=BA=D1=81=20=D0=BC=D0=B5?=
 =?UTF-8?q?=D0=BD=D0=B5=D0=B4=D0=B6=D0=B5=D1=80=D0=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/django_clickhouse/models.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/django_clickhouse/models.py b/src/django_clickhouse/models.py
index 0f89239..1a00d99 100644
--- a/src/django_clickhouse/models.py
+++ b/src/django_clickhouse/models.py
@@ -7,9 +7,10 @@ from typing import Optional, Any, Type, Set
 
 import six
 from django.db import transaction
+from django.db.models.manager import BaseManager
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
-from django.db.models import QuerySet as DjangoQuerySet, Manager as DjangoManager, Model as DjangoModel
+from django.db.models import QuerySet as DjangoQuerySet, Model as DjangoModel
 from statsd.defaults.django import statsd
 
 from .configuration import config
@@ -116,12 +117,7 @@ if not getattr(BulkUpdateManagerMixin, 'fake', False):
 ClickHouseSyncQuerySet = type('ClickHouseSyncModelQuerySet', tuple(qs_bases), {})
 
 
-class ClickHouseSyncManagerMixin:
-    def get_queryset(self):
-        return ClickHouseSyncQuerySet(model=self.model, using=self._db)
-
-
-class ClickHouseSyncManager(ClickHouseSyncManagerMixin, DjangoManager):
+class ClickHouseSyncManager(BaseManager.from_queryset(ClickHouseSyncQuerySet)):
     pass
 
 

From 0f1364b2b893698448e5fdf4c7963b816627de22 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 11 Sep 2019 15:34:37 +0500
Subject: [PATCH 24/27] Bugfix in QuerySetMixin inheritance

---
 src/django_clickhouse/models.py | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/src/django_clickhouse/models.py b/src/django_clickhouse/models.py
index 1a00d99..57b32f8 100644
--- a/src/django_clickhouse/models.py
+++ b/src/django_clickhouse/models.py
@@ -32,16 +32,18 @@ except ImportError:
         fake = True
 
 
-class ClickHouseSyncUpdateReturningQuerySetMixin(UpdateReturningMixin):
+class ClickHouseSyncRegisterMixin:
+    def _register_ops(self, operation, result):
+        pk_name = self.model._meta.pk.name
+        pk_list = [getattr(item, pk_name) for item in result]
+        self.model.register_clickhouse_operations(operation, *pk_list, using=self.db)
+
+
+class ClickHouseSyncUpdateReturningQuerySetMixin(ClickHouseSyncRegisterMixin, UpdateReturningMixin):
     """
     This mixin adopts methods of django-pg-returning library
     """
 
-    def _register_ops(self, operation, result):
-        pk_name = self.model._meta.pk.name
-        pk_list = result.values_list(pk_name, flat=True)
-        self.model.register_clickhouse_operations(operation, *pk_list, using=self.db)
-
     def update_returning(self, **updates):
         result = super().update_returning(**updates)
         self._register_ops('update', result)
@@ -53,7 +55,7 @@ class ClickHouseSyncUpdateReturningQuerySetMixin(UpdateReturningMixin):
         return result
 
 
-class ClickHouseSyncBulkUpdateQuerySetMixin(BulkUpdateManagerMixin):
+class ClickHouseSyncBulkUpdateQuerySetMixin(ClickHouseSyncRegisterMixin, BulkUpdateManagerMixin):
     """
     This mixin adopts methods of django-pg-bulk-update library
     """
@@ -69,11 +71,6 @@ class ClickHouseSyncBulkUpdateQuerySetMixin(BulkUpdateManagerMixin):
 
         return returning
 
-    def _register_ops(self, result):
-        pk_name = self.model._meta.pk.name
-        pk_list = [getattr(item, pk_name) for item in result]
-        self.model.register_clickhouse_operations('update', *pk_list, using=self.db)
-
     def bulk_update(self, *args, **kwargs):
         original_returning = kwargs.pop('returning', None)
         kwargs['returning'] = self._update_returning_param(original_returning)
@@ -89,19 +86,18 @@ class ClickHouseSyncBulkUpdateQuerySetMixin(BulkUpdateManagerMixin):
         return result.count() if original_returning is None else result
 
 
-class ClickHouseSyncQuerySetMixin:
+class ClickHouseSyncQuerySetMixin(ClickHouseSyncRegisterMixin):
     def update(self, **kwargs):
         # BUG I use update_returning method here. But it is not suitable for databases other then PostgreSQL
         # and requires django-pg-update-returning installed
         pk_name = self.model._meta.pk.name
-        res = self.only(pk_name).update_returning(**kwargs).values_list(pk_name, flat=True)
-        self.model.register_clickhouse_operations('update', *res, using=self.db)
+        res = self.only(pk_name).update_returning(**kwargs)
+        self._register_ops('update', res)
         return len(res)
 
     def bulk_create(self, objs, batch_size=None):
         objs = super().bulk_create(objs, batch_size=batch_size)
-        self.model.register_clickhouse_operations('insert', *[obj.pk for obj in objs], using=self.db)
-
+        self._register_ops('insert', objs)
         return objs
 
 

From 6493373c8f4ed8b02bbcaab68def0f88e035e0c9 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 11 Sep 2019 15:35:43 +0500
Subject: [PATCH 25/27] Bugfix in QuerySetMixin inheritance

---
 src/django_clickhouse/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/django_clickhouse/models.py b/src/django_clickhouse/models.py
index 57b32f8..52c979e 100644
--- a/src/django_clickhouse/models.py
+++ b/src/django_clickhouse/models.py
@@ -75,14 +75,14 @@ class ClickHouseSyncBulkUpdateQuerySetMixin(ClickHouseSyncRegisterMixin, BulkUpd
         original_returning = kwargs.pop('returning', None)
         kwargs['returning'] = self._update_returning_param(original_returning)
         result = super().bulk_update(*args, **kwargs)
-        self._register_ops(result)
+        self._register_ops('update', result)
         return result.count() if original_returning is None else result
 
     def bulk_update_or_create(self, *args, **kwargs):
         original_returning = kwargs.pop('returning', None)
         kwargs['returning'] = self._update_returning_param(original_returning)
         result = super().bulk_update_or_create(*args, **kwargs)
-        self._register_ops(result)
+        self._register_ops('update', result)
         return result.count() if original_returning is None else result
 
 

From a39a04fa522d0d68efafcc07513351c1045fe58b Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 4 Oct 2019 17:54:47 +0500
Subject: [PATCH 26/27] Added tests and fixes for bulk_create_returning and
 save_returning

---
 src/django_clickhouse/models.py |  5 ++++-
 tests/models.py                 | 20 +++++++++++++++++---
 tests/test_models.py            | 23 +++++++++++++++++++++++
 3 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/src/django_clickhouse/models.py b/src/django_clickhouse/models.py
index 52c979e..97815f0 100644
--- a/src/django_clickhouse/models.py
+++ b/src/django_clickhouse/models.py
@@ -102,7 +102,7 @@ class ClickHouseSyncQuerySetMixin(ClickHouseSyncRegisterMixin):
 
 
 # I add library dependant mixins to base classes only if libraries are installed
-qs_bases = [ClickHouseSyncQuerySetMixin, DjangoQuerySet]
+qs_bases = [ClickHouseSyncQuerySetMixin]
 
 if not getattr(UpdateReturningMixin, 'fake', False):
     qs_bases.append(ClickHouseSyncUpdateReturningQuerySetMixin)
@@ -110,6 +110,9 @@ if not getattr(UpdateReturningMixin, 'fake', False):
 if not getattr(BulkUpdateManagerMixin, 'fake', False):
     qs_bases.append(ClickHouseSyncBulkUpdateQuerySetMixin)
 
+
+# QuerySet must be the last one, so it can be redeclared in mixins
+qs_bases.append(DjangoQuerySet)
 ClickHouseSyncQuerySet = type('ClickHouseSyncModelQuerySet', tuple(qs_bases), {})
 
 
diff --git a/tests/models.py b/tests/models.py
index b27b271..78c8e21 100644
--- a/tests/models.py
+++ b/tests/models.py
@@ -2,17 +2,31 @@
 This file contains sample models to use in tests
 """
 from django.db import models
+from django.db.models.manager import BaseManager
+from django_pg_returning.models import UpdateReturningModel
 
-from django_clickhouse.models import ClickHouseSyncModel
+from django_clickhouse.models import ClickHouseSyncModel, ClickHouseSyncQuerySet
 
 
-class TestModel(ClickHouseSyncModel):
+class TestQuerySet(ClickHouseSyncQuerySet):
+    pass
+
+
+class TestManager(BaseManager.from_queryset(TestQuerySet)):
+    pass
+
+
+class TestModel(UpdateReturningModel, ClickHouseSyncModel):
+    objects = TestManager()
+
     value = models.IntegerField()
     created_date = models.DateField()
     created = models.DateTimeField()
 
 
-class SecondaryTestModel(ClickHouseSyncModel):
+class SecondaryTestModel(UpdateReturningModel, ClickHouseSyncModel):
+    objects = TestManager()
+
     value = models.IntegerField()
     created_date = models.DateField()
     created = models.DateTimeField()
diff --git a/tests/test_models.py b/tests/test_models.py
index 0de5c92..978cdc2 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -102,6 +102,16 @@ class TestOperations(TransactionTestCase):
         self.assertListEqual([('update', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
+    def test_bulk_create_returning(self):
+        items = [
+            self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=i)
+            for i in range(5)
+        ]
+        items = self.django_model.objects.bulk_create_returning(items)
+        self.assertEqual(5, len(items))
+        self.assertSetEqual({('insert', "%s.%d" % (self.db_alias, instance.pk)) for instance in items},
+                            set(self.storage.get_operations(self.clickhouse_model.get_import_key(), 10)))
+
     def test_qs_update_returning(self):
         self.django_model.objects.filter(pk=1).update_returning(created_date=datetime.date.today())
         self.assertListEqual([('update', "%s.1" % self.db_alias)],
@@ -123,6 +133,19 @@ class TestOperations(TransactionTestCase):
         self.assertListEqual([('delete', "%s.%d" % (self.db_alias, item.id)) for item in self.before_op_items],
                              self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
 
+    def test_save_returning(self):
+        # INSERT operation
+        instance = self.django_model(created_date=datetime.date.today(), created=datetime.datetime.now(), value=2)
+        instance.save_returning()
+        self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk))],
+                             self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
+
+        # UPDATE operation
+        instance.save_returning()
+        self.assertListEqual([('insert', "%s.%d" % (self.db_alias, instance.pk)),
+                              ('update', "%s.%d" % (self.db_alias, instance.pk))],
+                             self.storage.get_operations(self.clickhouse_model.get_import_key(), 10))
+
     def test_delete(self):
         instance = self.django_model.objects.get(pk=1)
         instance.delete()

From 0751c67ce778f3f2999fcd3fedb775e915e7d625 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Fri, 4 Oct 2019 17:58:24 +0500
Subject: [PATCH 27/27] Added tests and fixes for bulk_create_returning and
 save_returning

---
 tests/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models.py b/tests/models.py
index 78c8e21..a0de1ec 100644
--- a/tests/models.py
+++ b/tests/models.py
@@ -3,7 +3,7 @@ This file contains sample models to use in tests
 """
 from django.db import models
 from django.db.models.manager import BaseManager
-from django_pg_returning.models import UpdateReturningModel
+from django_pg_returning import UpdateReturningModel
 
 from django_clickhouse.models import ClickHouseSyncModel, ClickHouseSyncQuerySet