From 3bc5f27cdacf9eda9d36b1ab1110b8bcf1f563ae Mon Sep 17 00:00:00 2001
From: M1ha <m1hacka@mail.ru>
Date: Sat, 8 Dec 2018 11:40:05 +0500
Subject: [PATCH 01/30] 1) Unified QuerySet filter and exclude methods,  so
 both can load *q and **kwargs at the same time  and accept prewhere flag 2)
 Added ability to add prewhere clause  in QuerySet.filter() and
 QuerySet.exclude() methods 3) Added ability to check, if Q() object is empty
 (including bool check) 4) Refactored QuerySet.as_sql() method:  + don't add
 GROUP BY and WHERE if it's not needed  + ability to add PREWHERE condition  +
 Common style of adding optional query parts

---
 src/infi/clickhouse_orm/query.py | 81 +++++++++++++++++++++++---------
 1 file changed, 59 insertions(+), 22 deletions(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 1035dd4..4a77226 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -183,6 +183,14 @@ class Q(object):
         self._negate = False
         self._mode = self.AND_MODE
 
+    @property
+    def is_empty(self):
+        """
+        Checks if there are any conditions in Q object
+        :return: Boolean
+        """
+        return bool(self._fovs or self._l_child or self._r_child)
+
     @classmethod
     def _construct_from(cls, l_child, r_child, mode):
         q = Q()
@@ -203,7 +211,7 @@ class Q(object):
             sql = ' {} '.format(self._mode).join(fov.to_sql(model_cls) for fov in self._fovs)
         else:
             if self._l_child and self._r_child:
-                sql = '({} {} {})'.format(
+                sql = '({}) {} ({})'.format(
                         self._l_child.to_sql(model_cls), self._mode, self._r_child.to_sql(model_cls))
             else:
                 return '1'
@@ -222,6 +230,9 @@ class Q(object):
         q._negate = True
         return q
 
+    def __bool__(self):
+        return not self.is_empty
+
 
 @six.python_2_unicode_compatible
 class QuerySet(object):
@@ -239,7 +250,8 @@ class QuerySet(object):
         self._model_cls = model_cls
         self._database = database
         self._order_by = []
-        self._q = []
+        self._where_q = Q()
+        self._prewhere_q = Q()
         self._fields = model_cls.fields().keys()
         self._limits = None
         self._distinct = False
@@ -303,14 +315,14 @@ class QuerySet(object):
             for field in self._order_by
         ])
 
-    def conditions_as_sql(self):
+    def conditions_as_sql(self, q_object):
         """
-        Returns the contents of the query's `WHERE` clause as a string.
+        Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
         """
-        if self._q:
-            return u' AND '.join([q.to_sql(self._model_cls) for q in self._q])
+        if q_object:
+            return u' AND '.join([q.to_sql(self._model_cls) for q in q_object])
         else:
-            return u'1'
+            return u''
 
     def count(self):
         """
@@ -342,25 +354,40 @@ class QuerySet(object):
         qs._fields = field_names
         return qs
 
-    def filter(self, *q, **filter_fields):
+    def _filter_or_exclude(self, *q, **kwargs):
+        reverse = kwargs.pop('reverse', False)
+        prewhere = kwargs.pop('prewhere', False)
+        condition = copy(self._where_q)
+        qs = copy(self)
+
+        if q:
+            condition &= q
+
+        if kwargs:
+            condition &= Q(**kwargs)
+
+        if reverse:
+            condition = ~condition
+
+        if prewhere:
+            qs._prewhere_q = condition
+        else:
+            qs._where_q = condition
+
+        return qs
+
+    def filter(self, *q, **kwargs):
         """
         Returns a copy of this queryset that includes only rows matching the conditions.
         Add q object to query if it specified.
         """
-        qs = copy(self)
-        if q:
-            qs._q = list(self._q) + list(q)
-        else:
-            qs._q = list(self._q) + [Q(**filter_fields)]
-        return qs
+        return self._filter_or_exclude(*q, **kwargs)
 
-    def exclude(self, **filter_fields):
+    def exclude(self, *q, **kwargs):
         """
         Returns a copy of this queryset that excludes all rows matching the conditions.
         """
-        qs = copy(self)
-        qs._q = list(self._q) + [~Q(**filter_fields)]
-        return qs
+        return self._filter_or_exclude(*q, reverse=True, **kwargs)
 
     def paginate(self, page_num=1, page_size=100):
         """
@@ -476,20 +503,30 @@ class AggregateQuerySet(QuerySet):
         Returns the whole query as a SQL string.
         """
         distinct = 'DISTINCT ' if self._distinct else ''
-        grouping = comma_join('`%s`' % field for field in self._grouping_fields)
         fields = comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
+
         params = dict(
             distinct=distinct,
-            grouping=grouping or "''",
             fields=fields,
             table=self._model_cls.table_name(),
-            conds=self.conditions_as_sql()
         )
-        sql = u'SELECT %(distinct)s%(fields)s\nFROM `%(table)s`\nWHERE %(conds)s\nGROUP BY %(grouping)s' % params
+        sql = u'SELECT %(distinct)s%(fields)s\nFROM `%(table)s`' % params
+
+        if self._prewhere_q:
+            sql += '\nPREWHERE ' + self.conditions_as_sql(self._prewhere_q)
+
+        if self._where_q:
+            sql += '\nWHERE ' + self.conditions_as_sql(self._where_q)
+
+        if self._grouping_fields:
+            sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields)
+
         if self._order_by:
             sql += '\nORDER BY ' + self.order_by_as_sql()
+
         if self._limits:
             sql += '\nLIMIT %d, %d' % self._limits
+
         return sql
 
     def __iter__(self):

From 95055996adbfc8a2b3ba2b1c419d9c7f24d96bdc Mon Sep 17 00:00:00 2001
From: M1ha <m1hacka@mail.ru>
Date: Sat, 8 Dec 2018 11:57:08 +0500
Subject: [PATCH 02/30] 1) Further refactoring of as_sql():   merged very
 similar QuerySet and AggregateQuerySet methods 2) Fixed some bugs, caused by
 conditions_as_sql() parameters change

---
 src/infi/clickhouse_orm/query.py | 68 ++++++++++++++------------------
 tests/test_querysets.py          |  8 ++--
 2 files changed, 33 insertions(+), 43 deletions(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 4a77226..b6a8baf 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -252,6 +252,7 @@ class QuerySet(object):
         self._order_by = []
         self._where_q = Q()
         self._prewhere_q = Q()
+        self._grouping_fields = []
         self._fields = model_cls.fields().keys()
         self._limits = None
         self._distinct = False
@@ -292,19 +293,34 @@ class QuerySet(object):
             qs._limits = (start, stop - start)
             return qs
 
+    def select_fields_as_sql(self):
+        return comma_join('`%s`' % field for field in self._fields) if self._fields else '*'
+
     def as_sql(self):
         """
         Returns the whole query as a SQL string.
         """
         distinct = 'DISTINCT ' if self._distinct else ''
-        fields = '*'
-        if self._fields:
-            fields = comma_join('`%s`' % field for field in self._fields)
-        ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
-        limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
-        params = (distinct, fields, self._model_cls.table_name(),
-                  self.conditions_as_sql(), ordering, limit)
-        return u'SELECT %s%s\nFROM `%s`\nWHERE %s%s%s' % params
+
+        params = (distinct, self.select_fields_as_sql(), self._model_cls.table_name())
+        sql = u'SELECT %s%s\nFROM `%s`\n' % params
+
+        if self._prewhere_q:
+            sql += '\nPREWHERE ' + self.conditions_as_sql(self._prewhere_q)
+
+        if self._where_q:
+            sql += '\nWHERE ' + self.conditions_as_sql(self._where_q)
+
+        if self._grouping_fields:
+            sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields)
+
+        if self._order_by:
+            sql += '\nORDER BY ' + self.order_by_as_sql()
+
+        if self._limits:
+            sql += '\nLIMIT %d, %d' % self._limits
+
+        return
 
     def order_by_as_sql(self):
         """
@@ -333,8 +349,10 @@ class QuerySet(object):
             sql = u'SELECT count() FROM (%s)' % self.as_sql()
             raw = self._database.raw(sql)
             return int(raw) if raw else 0
+
         # Simple case
-        return self._database.count(self._model_cls, self.conditions_as_sql())
+        conditions = self.conditions_as_sql(self._where_q & self._prewhere_q)
+        return self._database.count(self._model_cls, conditions)
 
     def order_by(self, *field_names):
         """
@@ -498,36 +516,8 @@ class AggregateQuerySet(QuerySet):
         """
         raise NotImplementedError('Cannot re-aggregate an AggregateQuerySet')
 
-    def as_sql(self):
-        """
-        Returns the whole query as a SQL string.
-        """
-        distinct = 'DISTINCT ' if self._distinct else ''
-        fields = comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
-
-        params = dict(
-            distinct=distinct,
-            fields=fields,
-            table=self._model_cls.table_name(),
-        )
-        sql = u'SELECT %(distinct)s%(fields)s\nFROM `%(table)s`' % params
-
-        if self._prewhere_q:
-            sql += '\nPREWHERE ' + self.conditions_as_sql(self._prewhere_q)
-
-        if self._where_q:
-            sql += '\nWHERE ' + self.conditions_as_sql(self._where_q)
-
-        if self._grouping_fields:
-            sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields)
-
-        if self._order_by:
-            sql += '\nORDER BY ' + self.order_by_as_sql()
-
-        if self._limits:
-            sql += '\nLIMIT %d, %d' % self._limits
-
-        return sql
+    def select_fields_as_sql(self):
+        return comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
 
     def __iter__(self):
         return self._database.select(self.as_sql()) # using an ad-hoc model
diff --git a/tests/test_querysets.py b/tests/test_querysets.py
index a4fef14..9dd1ca4 100644
--- a/tests/test_querysets.py
+++ b/tests/test_querysets.py
@@ -369,13 +369,13 @@ class AggregateTestCase(TestCaseWithData):
             the__next__number = Int32Field()
             engine = Memory()
         qs = Mdl.objects_in(self.database).filter(the__number=1)
-        self.assertEqual(qs.conditions_as_sql(), 'the__number = 1')
+        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__number = 1')
         qs = Mdl.objects_in(self.database).filter(the__number__gt=1)
-        self.assertEqual(qs.conditions_as_sql(), 'the__number > 1')
+        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__number > 1')
         qs = Mdl.objects_in(self.database).filter(the__next__number=1)
-        self.assertEqual(qs.conditions_as_sql(), 'the__next__number = 1')
+        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__next__number = 1')
         qs = Mdl.objects_in(self.database).filter(the__next__number__gt=1)
-        self.assertEqual(qs.conditions_as_sql(), 'the__next__number > 1')
+        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__next__number > 1')
 
 
 Color = Enum('Color', u'red blue green yellow brown white black')

From 12463c61b8369d484ef08e09b18803168eedf002 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Tue, 11 Dec 2018 17:25:54 +0500
Subject: [PATCH 03/30] 1) Fixed bugs 2) Edited the docs 3) Added test for
 prewhere

---
 docs/querysets.md                | 14 +++++++++++---
 src/infi/clickhouse_orm/query.py | 33 ++++++++++++++++----------------
 tests/test_querysets.py          |  9 ++++++++-
 3 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/docs/querysets.md b/docs/querysets.md
index 260e66c..c4e84ea 100644
--- a/docs/querysets.md
+++ b/docs/querysets.md
@@ -17,21 +17,29 @@ The `filter` and `exclude` methods are used for filtering the matching instances
 
     >>> qs = Person.objects_in(database)
     >>> qs = qs.filter(first_name__startswith='V').exclude(birthday__lt='2000-01-01')
-    >>> qs.conditions_as_sql()
+    >>> qs.conditions_as_sql(qs._where)
     u"first_name LIKE 'V%' AND NOT (birthday < '2000-01-01')"
 
 It is possible to specify several fields to filter or exclude by:
 
     >>> qs = Person.objects_in(database).filter(last_name='Smith', height__gt=1.75)
-    >>> qs.conditions_as_sql()
+    >>> qs.conditions_as_sql(qs._where)
     u"last_name = 'Smith' AND height > 1.75"
 
 For filters with compound conditions you can use `Q` objects inside `filter` with overloaded operators `&` (AND), `|` (OR) and `~` (NOT):
 
     >>> qs = Person.objects_in(database).filter((Q(first_name='Ciaran', last_name='Carver') | Q(height_lte=1.8)) & ~Q(first_name='David'))
-    >>> qs.conditions_as_sql()
+    >>> qs.conditions_as_sql(qs._where)
     u"((first_name = 'Ciaran' AND last_name = 'Carver') OR height <= 1.8) AND (NOT (first_name = 'David'))"
 
+By default conditions from `filter` and `exclude` methods are add to `WHERE` clause. 
+For better aggregation performance you can add them to `PREWHERE` section using `prewhere=True` parameter
+
+    >>> qs = Person.objects_in(database)
+    >>> qs = qs.filter(first_name__startswith='V', prewhere=True)
+    >>> qs.conditions_as_sql(qs._prewhere)
+    u"first_name LIKE 'V%'"
+    
 There are different operators that can be used, by passing `<fieldname>__<operator>=<value>` (two underscores separate the field name from the operator). In case no operator is given, `eq` is used by default. Below are all the supported operators.
 
 | Operator       | Equivalent SQL                               | Comments                           |
diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index b6a8baf..c4839aa 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -189,14 +189,14 @@ class Q(object):
         Checks if there are any conditions in Q object
         :return: Boolean
         """
-        return bool(self._fovs or self._l_child or self._r_child)
+        return not bool(self._fovs or self._l_child or self._r_child)
 
     @classmethod
     def _construct_from(cls, l_child, r_child, mode):
         q = Q()
         q._l_child = l_child
         q._r_child = r_child
-        q._mode = mode # AND/OR
+        q._mode = mode  # AND/OR
         return q
 
     def _build_fov(self, key, value):
@@ -209,14 +209,17 @@ class Q(object):
     def to_sql(self, model_cls):
         if self._fovs:
             sql = ' {} '.format(self._mode).join(fov.to_sql(model_cls) for fov in self._fovs)
+        elif self._l_child and self._r_child:
+            sql = '({}) {} ({})'.format(self._l_child.to_sql(model_cls), self._mode, self._r_child.to_sql(model_cls))
+        elif self._l_child or self._r_child:
+            # Return existing condition
+            sql = (self._l_child or self._r_child).to_sql(model_cls)
         else:
-            if self._l_child and self._r_child:
-                sql = '({}) {} ({})'.format(
-                        self._l_child.to_sql(model_cls), self._mode, self._r_child.to_sql(model_cls))
-            else:
-                return '1'
+            sql = '1'
+
         if self._negate:
             sql = 'NOT (%s)' % sql
+
         return sql
 
     def __or__(self, other):
@@ -303,7 +306,7 @@ class QuerySet(object):
         distinct = 'DISTINCT ' if self._distinct else ''
 
         params = (distinct, self.select_fields_as_sql(), self._model_cls.table_name())
-        sql = u'SELECT %s%s\nFROM `%s`\n' % params
+        sql = u'SELECT %s%s\nFROM `%s`' % params
 
         if self._prewhere_q:
             sql += '\nPREWHERE ' + self.conditions_as_sql(self._prewhere_q)
@@ -320,7 +323,7 @@ class QuerySet(object):
         if self._limits:
             sql += '\nLIMIT %d, %d' % self._limits
 
-        return
+        return sql
 
     def order_by_as_sql(self):
         """
@@ -335,10 +338,7 @@ class QuerySet(object):
         """
         Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
         """
-        if q_object:
-            return u' AND '.join([q.to_sql(self._model_cls) for q in q_object])
-        else:
-            return u''
+        return q_object.to_sql(self._model_cls)
 
     def count(self):
         """
@@ -378,8 +378,8 @@ class QuerySet(object):
         condition = copy(self._where_q)
         qs = copy(self)
 
-        if q:
-            condition &= q
+        for q_obj in q:
+            condition &= q_obj
 
         if kwargs:
             condition &= Q(**kwargs)
@@ -487,7 +487,8 @@ class AggregateQuerySet(QuerySet):
         self._grouping_fields = grouping_fields
         self._calculated_fields = calculated_fields
         self._order_by = list(base_qs._order_by)
-        self._q = list(base_qs._q)
+        self._where_q = base_qs._where_q
+        self._prewhere_q = base_qs._prewhere_q
         self._limits = base_qs._limits
         self._distinct = base_qs._distinct
 
diff --git a/tests/test_querysets.py b/tests/test_querysets.py
index 9dd1ca4..14db99c 100644
--- a/tests/test_querysets.py
+++ b/tests/test_querysets.py
@@ -11,7 +11,7 @@ from datetime import date, datetime
 try:
     Enum # exists in Python 3.4+
 except NameError:
-    from enum import Enum # use the enum34 library instead
+    from enum import Enum  # use the enum34 library instead
 
 
 class QuerySetTestCase(TestCaseWithData):
@@ -29,6 +29,13 @@ class QuerySetTestCase(TestCaseWithData):
         self.assertEqual(count, expected_count)
         self.assertEqual(qs.count(), expected_count)
 
+    def test_prewhere(self):
+        # We can't distinguish prewhere and where results, it affects performance only.
+        # So let's control prewhere acts like where does
+        qs = Person.objects_in(self.database)
+        self.assertTrue(qs.filter(first_name='Connor', prewhere=True))
+        self.assertFalse(qs.filter(first_name='Willy', prewhere=True))
+
     def test_no_filtering(self):
         qs = Person.objects_in(self.database)
         self._test_qs(qs, len(data))

From 0c92e2ac741608278d550af307903c0828c8cb8f Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Tue, 11 Dec 2018 18:04:08 +0500
Subject: [PATCH 04/30] Simplified conditions, built by Q objects, if many
 conditions are joined in same mode

---
 src/infi/clickhouse_orm/query.py | 61 ++++++++++++++++++++++++--------
 1 file changed, 46 insertions(+), 15 deletions(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index c4839aa..0052a0c 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 import six
 import pytz
-from copy import copy
+from copy import copy, deepcopy
 from math import ceil
 from .utils import comma_join
 
@@ -170,6 +170,11 @@ class FOV(object):
     def to_sql(self, model_cls):
         return self._operator.to_sql(model_cls, self._field_name, self._value)
 
+    def __deepcopy__(self, memodict={}):
+        res = copy(self)
+        res._value = deepcopy(self._value)
+        return res
+
 
 class Q(object):
 
@@ -178,8 +183,7 @@ class Q(object):
 
     def __init__(self, **filter_fields):
         self._fovs = [self._build_fov(k, v) for k, v in six.iteritems(filter_fields)]
-        self._l_child = None
-        self._r_child = None
+        self._children = []
         self._negate = False
         self._mode = self.AND_MODE
 
@@ -189,14 +193,22 @@ class Q(object):
         Checks if there are any conditions in Q object
         :return: Boolean
         """
-        return not bool(self._fovs or self._l_child or self._r_child)
+        return not bool(self._fovs or self._children)
 
     @classmethod
     def _construct_from(cls, l_child, r_child, mode):
-        q = Q()
-        q._l_child = l_child
-        q._r_child = r_child
-        q._mode = mode  # AND/OR
+        if mode == l_child._mode:
+            q = deepcopy(l_child)
+            q._children.append(deepcopy(r_child))
+        elif mode == r_child._mode:
+            q = deepcopy(r_child)
+            q._children.append(deepcopy(l_child))
+        else:
+            # Different modes
+            q = Q()
+            q._children = [l_child, r_child]
+            q._mode = mode  # AND/OR
+
         return q
 
     def _build_fov(self, key, value):
@@ -207,15 +219,23 @@ class Q(object):
         return FOV(field_name, operator, value)
 
     def to_sql(self, model_cls):
+        condition_sql = []
+
         if self._fovs:
-            sql = ' {} '.format(self._mode).join(fov.to_sql(model_cls) for fov in self._fovs)
-        elif self._l_child and self._r_child:
-            sql = '({}) {} ({})'.format(self._l_child.to_sql(model_cls), self._mode, self._r_child.to_sql(model_cls))
-        elif self._l_child or self._r_child:
-            # Return existing condition
-            sql = (self._l_child or self._r_child).to_sql(model_cls)
-        else:
+            condition_sql.extend([fov.to_sql(model_cls) for fov in self._fovs])
+
+        if self._children:
+            condition_sql.extend([child.to_sql(model_cls) for child in self._children if child])
+
+        if not condition_sql:
+            # Empty Q() object returns everything
             sql = '1'
+        elif len(condition_sql) == 1:
+            # Skip not needed brackets over single condition
+            sql = condition_sql[0]
+        else:
+            # Each condition must be enclosed in brackets, or order of operations may be wrong
+            sql = '(%s)' % ') {} ('.format(self._mode).join(condition_sql)
 
         if self._negate:
             sql = 'NOT (%s)' % sql
@@ -236,6 +256,17 @@ class Q(object):
     def __bool__(self):
         return not self.is_empty
 
+    def __deepcopy__(self, memodict={}):
+        q = Q()
+        q._fovs = [deepcopy(fov) for fov in self._fovs]
+        q._negate = self._negate
+        q._mode = self._mode
+
+        if self._children:
+            q._children = [deepcopy(child) for child in self._children]
+
+        return q
+
 
 @six.python_2_unicode_compatible
 class QuerySet(object):

From 5f4023f120c2cb7a908b4aa0fae477b54bc867c0 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 12 Dec 2018 15:33:35 +0500
Subject: [PATCH 05/30] Added with_totals method

---
 docs/querysets.md                | 13 +++++++++++++
 src/infi/clickhouse_orm/query.py | 17 +++++++++++++++++
 tests/test_querysets.py          | 11 +++++++++++
 3 files changed, 41 insertions(+)

diff --git a/docs/querysets.md b/docs/querysets.md
index c4e84ea..b42fe1b 100644
--- a/docs/querysets.md
+++ b/docs/querysets.md
@@ -199,6 +199,19 @@ This queryset is translated to:
 
 After calling `aggregate` you can still use most of the regular queryset methods, such as `count`, `order_by` and `paginate`. It is not possible, however, to call `only` or `aggregate`. It is also not possible to filter the queryset on calculated fields, only on fields that exist in the model.
 
+If you limit aggregation results, it might be useful to get total aggregation values for all rows.
+To achieve this, you can use `with_totals` method. It will return extra row (last) with
+values aggregated for all rows suitable for filters.
+
+    qs = Person.objects_in(database).aggregate('first_name' num='count()').with_totals().order_by('-count')[:3]
+    >>> print qs.count()
+    4
+    >>> for row in qs: 
+    >>>     print(row.first_name, row.count)
+    'Cassandra' 2
+    'Alexandra' 2
+    '' 100
+
 ---
 
 [<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md)
\ No newline at end of file
diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 0052a0c..50ffd68 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -287,6 +287,7 @@ class QuerySet(object):
         self._where_q = Q()
         self._prewhere_q = Q()
         self._grouping_fields = []
+        self._grouping_with_totals = False
         self._fields = model_cls.fields().keys()
         self._limits = None
         self._distinct = False
@@ -348,6 +349,9 @@ class QuerySet(object):
         if self._grouping_fields:
             sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields)
 
+            if self._grouping_with_totals:
+                sql += ' WITH TOTALS'
+
         if self._order_by:
             sql += '\nORDER BY ' + self.order_by_as_sql()
 
@@ -551,6 +555,9 @@ class AggregateQuerySet(QuerySet):
     def select_fields_as_sql(self):
         return comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
 
+    def group_by_as_sql(self):
+        return 'GROUP BY'
+
     def __iter__(self):
         return self._database.select(self.as_sql()) # using an ad-hoc model
 
@@ -561,3 +568,13 @@ class AggregateQuerySet(QuerySet):
         sql = u'SELECT count() FROM (%s)' % self.as_sql()
         raw = self._database.raw(sql)
         return int(raw) if raw else 0
+
+    def with_totals(self):
+        """
+        Adds WITH TOTALS modifier ot GROUP BY, making query return extra row
+        with aggregate function calculated across all the rows. More information:
+        https://clickhouse.yandex/docs/en/query_language/select/#with-totals-modifier
+        """
+        qs = copy(self)
+        qs._grouping_with_totals = True
+        return qs
diff --git a/tests/test_querysets.py b/tests/test_querysets.py
index 14db99c..d0baaad 100644
--- a/tests/test_querysets.py
+++ b/tests/test_querysets.py
@@ -370,6 +370,17 @@ class AggregateTestCase(TestCaseWithData):
         print(qs.as_sql())
         self.assertEqual(qs.count(), 1)
 
+    def test_aggregate_with_totals(self):
+        qs = Person.objects_in(self.database).aggregate('first_name', count='count()').\
+            with_totals().order_by('-count')[:5]
+        print(qs.as_sql())
+        result = list(qs)
+        self.assertEqual(len(result), 6)
+        for row in result[:-1]:
+            self.assertEqual(2, row.count)
+
+        self.assertEqual(100, result[-1].count)
+
     def test_double_underscore_field(self):
         class Mdl(Model):
             the__number = Int32Field()

From d1e61dc420af7ce7dd71e832693ba9453b68e624 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Mon, 17 Dec 2018 10:26:35 +0500
Subject: [PATCH 06/30] Fixed invalid condtion joins in _filter_or_exclude
 (multiple_exclude_test)

---
 src/infi/clickhouse_orm/query.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 22c1484..b705320 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -411,9 +411,10 @@ class QuerySet(object):
     def _filter_or_exclude(self, *q, **kwargs):
         reverse = kwargs.pop('reverse', False)
         prewhere = kwargs.pop('prewhere', False)
-        condition = copy(self._where_q)
+
         qs = copy(self)
 
+        condition = Q()
         for q_obj in q:
             condition &= q_obj
 
@@ -423,6 +424,7 @@ class QuerySet(object):
         if reverse:
             condition = ~condition
 
+        condition = copy(self._prewhere_q if prewhere else self._where_q) & condition
         if prewhere:
             qs._prewhere_q = condition
         else:

From 6902de3b94825569a6d2f11a79bbe7af10bce103 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 19 Dec 2018 10:06:57 +0500
Subject: [PATCH 07/30] Fix fo conditions_as_sql method by review

---
 src/infi/clickhouse_orm/query.py | 9 +++++----
 tests/test_querysets.py          | 8 ++++----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index b705320..9eee645 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -345,10 +345,10 @@ class QuerySet(object):
         sql = u'SELECT %s%s\nFROM `%s`%s' % params
 
         if self._prewhere_q:
-            sql += '\nPREWHERE ' + self.conditions_as_sql(self._prewhere_q)
+            sql += '\nPREWHERE ' + self.conditions_as_sql(prewhere=True)
 
         if self._where_q:
-            sql += '\nWHERE ' + self.conditions_as_sql(self._where_q)
+            sql += '\nWHERE ' + self.conditions_as_sql(prewhere=False)
 
         if self._grouping_fields:
             sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields)
@@ -370,10 +370,11 @@ class QuerySet(object):
             for field in self._order_by
         ])
 
-    def conditions_as_sql(self, q_object):
+    def conditions_as_sql(self, prewhere=False):
         """
         Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
         """
+        q_object = self._prewhere_q if prewhere else self._where_q
         return q_object.to_sql(self._model_cls)
 
     def count(self):
@@ -387,7 +388,7 @@ class QuerySet(object):
             return int(raw) if raw else 0
 
         # Simple case
-        conditions = self.conditions_as_sql(self._where_q & self._prewhere_q)
+        conditions = (self._where_q & self._prewhere_q).to_sql(self._model_cls)
         return self._database.count(self._model_cls, conditions)
 
     def order_by(self, *field_names):
diff --git a/tests/test_querysets.py b/tests/test_querysets.py
index f5c20c4..298d859 100644
--- a/tests/test_querysets.py
+++ b/tests/test_querysets.py
@@ -417,13 +417,13 @@ class AggregateTestCase(TestCaseWithData):
             the__next__number = Int32Field()
             engine = Memory()
         qs = Mdl.objects_in(self.database).filter(the__number=1)
-        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__number = 1')
+        self.assertEqual(qs.conditions_as_sql(), 'the__number = 1')
         qs = Mdl.objects_in(self.database).filter(the__number__gt=1)
-        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__number > 1')
+        self.assertEqual(qs.conditions_as_sql(), 'the__number > 1')
         qs = Mdl.objects_in(self.database).filter(the__next__number=1)
-        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__next__number = 1')
+        self.assertEqual(qs.conditions_as_sql(), 'the__next__number = 1')
         qs = Mdl.objects_in(self.database).filter(the__next__number__gt=1)
-        self.assertEqual(qs.conditions_as_sql(qs._where_q), 'the__next__number > 1')
+        self.assertEqual(qs.conditions_as_sql(), 'the__next__number > 1')
 
 
 Color = Enum('Color', u'red blue green yellow brown white black')

From d11f8b3b76398cc1d844e282a463c5c821fc0df8 Mon Sep 17 00:00:00 2001
From: M1ha <work_shvein_mihail@mail.ru>
Date: Wed, 19 Dec 2018 10:10:03 +0500
Subject: [PATCH 08/30] Docs updated

---
 docs/class_reference.md | 8 ++++----
 docs/querysets.md       | 8 ++++----
 docs/ref.md             | 4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/class_reference.md b/docs/class_reference.md
index c992d8e..6eb2353 100644
--- a/docs/class_reference.md
+++ b/docs/class_reference.md
@@ -835,10 +835,10 @@ is equivalent to:
 Returns the whole query as a SQL string.
 
 
-#### conditions_as_sql()
+#### conditions_as_sql(prewhere=True)
 
 
-Returns the contents of the query's `WHERE` clause as a string.
+Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
 
 
 #### count()
@@ -943,10 +943,10 @@ This method is not supported on `AggregateQuerySet`.
 Returns the whole query as a SQL string.
 
 
-#### conditions_as_sql()
+#### conditions_as_sql(prewhere=True)
 
 
-Returns the contents of the query's `WHERE` clause as a string.
+Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
 
 
 #### count()
diff --git a/docs/querysets.md b/docs/querysets.md
index 965950e..2329000 100644
--- a/docs/querysets.md
+++ b/docs/querysets.md
@@ -17,19 +17,19 @@ The `filter` and `exclude` methods are used for filtering the matching instances
 
     >>> qs = Person.objects_in(database)
     >>> qs = qs.filter(first_name__startswith='V').exclude(birthday__lt='2000-01-01')
-    >>> qs.conditions_as_sql(qs._where)
+    >>> qs.conditions_as_sql()
     u"first_name LIKE 'V%' AND NOT (birthday < '2000-01-01')"
 
 It is possible to specify several fields to filter or exclude by:
 
     >>> qs = Person.objects_in(database).filter(last_name='Smith', height__gt=1.75)
-    >>> qs.conditions_as_sql(qs._where)
+    >>> qs.conditions_as_sql()
     u"last_name = 'Smith' AND height > 1.75"
 
 For filters with compound conditions you can use `Q` objects inside `filter` with overloaded operators `&` (AND), `|` (OR) and `~` (NOT):
 
     >>> qs = Person.objects_in(database).filter((Q(first_name='Ciaran', last_name='Carver') | Q(height_lte=1.8)) & ~Q(first_name='David'))
-    >>> qs.conditions_as_sql(qs._where)
+    >>> qs.conditions_as_sql()
     u"((first_name = 'Ciaran' AND last_name = 'Carver') OR height <= 1.8) AND (NOT (first_name = 'David'))"
 
 By default conditions from `filter` and `exclude` methods are add to `WHERE` clause. 
@@ -37,7 +37,7 @@ For better aggregation performance you can add them to `PREWHERE` section using
 
     >>> qs = Person.objects_in(database)
     >>> qs = qs.filter(first_name__startswith='V', prewhere=True)
-    >>> qs.conditions_as_sql(qs._prewhere)
+    >>> qs.conditions_as_sql(prewhere=True)
     u"first_name LIKE 'V%'"
     
 There are different operators that can be used, by passing `<fieldname>__<operator>=<value>` (two underscores separate the field name from the operator). In case no operator is given, `eq` is used by default. Below are all the supported operators.
diff --git a/docs/ref.md b/docs/ref.md
index a298d04..e750d18 100644
--- a/docs/ref.md
+++ b/docs/ref.md
@@ -482,9 +482,9 @@ infi.clickhouse_orm.query
 #### QuerySet(model_cls, database)
 
 
-#### conditions_as_sql()
+#### conditions_as_sql(prewhere=True)
 
-Return the contents of the queryset's WHERE clause.
+Return the contents of the queryset's WHERE or `PREWHERE` clause.
 
 
 #### count()

From 1388dd9e27bda0fc2fb8614e4adf320c86914719 Mon Sep 17 00:00:00 2001
From: Kirill Potekhin <potehik@gmail.com>
Date: Thu, 20 Dec 2018 01:35:13 +0300
Subject: [PATCH 09/30] UUIDField added for ClickHouse UUID type

---
 src/infi/clickhouse_orm/fields.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py
index 8c11805..71d9543 100644
--- a/src/infi/clickhouse_orm/fields.py
+++ b/src/infi/clickhouse_orm/fields.py
@@ -1,11 +1,12 @@
 from __future__ import unicode_literals
-from six import string_types, text_type, binary_type
+from six import string_types, text_type, binary_type, integer_types
 import datetime
 import iso8601
 import pytz
 import time
 from calendar import timegm
 from decimal import Decimal, localcontext
+from uuid import UUID
 
 from .utils import escape, parse_array, comma_join
 
@@ -452,6 +453,26 @@ class ArrayField(Field):
         return 'Array(%s)' % self.inner_field.get_sql(with_default_expression=False)
 
 
+class UUIDField(Field):
+    class_default = UUID(int=0)
+    db_type = 'UUID'
+
+    def to_python(self, value, timezone_in_use):
+        if isinstance(value, UUID):
+            return value
+        elif isinstance(value, string_types):
+            return UUID(bytes=value) if len(value) == 16 else UUID(value)
+        elif isinstance(value, integer_types):
+            return UUID(int=value)
+        elif isinstance(value, tuple):
+            return UUID(fields=value)
+        else:
+            raise ValueError('Invalid value for UUIDField: %r' % value)
+
+    def to_db_string(self, value, quote=True):
+        return escape(str(value), quote)
+
+
 class NullableField(Field):
 
     class_default = None

From ed62cf5da7eb71f6d01b1b09cf2ccedf8ebb4467 Mon Sep 17 00:00:00 2001
From: Kirill Potekhin <potehik@gmail.com>
Date: Thu, 20 Dec 2018 01:36:46 +0300
Subject: [PATCH 10/30] tests for UUIDField

---
 tests/test_uuid_fields.py | 44 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 tests/test_uuid_fields.py

diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py
new file mode 100644
index 0000000..6fc2278
--- /dev/null
+++ b/tests/test_uuid_fields.py
@@ -0,0 +1,44 @@
+from __future__ import unicode_literals
+import unittest
+from uuid import UUID
+from infi.clickhouse_orm.database import Database
+from infi.clickhouse_orm.fields import Int16Field, UUIDField
+from infi.clickhouse_orm.models import Model
+from infi.clickhouse_orm.engines import Memory
+
+
+class UUIDFieldsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.database = Database('test-db')
+
+    def tearDown(self):
+        self.database.drop_database()
+
+    def test_uuid_field(self):
+        # Create a model
+        class TestModel(Model):
+            i = Int16Field()
+            f = UUIDField()
+            engine = Memory()
+        self.database.create_table(TestModel)
+        # Check valid values (all values are the same UUID)
+        values = [
+            '12345678-1234-5678-1234-567812345678',
+            '{12345678-1234-5678-1234-567812345678}',
+            '12345678123456781234567812345678',
+            'urn:uuid:12345678-1234-5678-1234-567812345678',
+            '\x12\x34\x56\x78'*4,
+            (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
+            0x12345678123456781234567812345678,
+        ]
+        for index, value in enumerate(values):
+            rec = TestModel(i=index, f=value)
+            self.database.insert([rec])
+        for rec in TestModel.objects_in(self.database):
+            self.assertEqual(rec.f, UUID(values[0]))
+        # Check invalid values
+        for value in [None, 'zzz', -1, '123']:
+            with self.assertRaises(ValueError):
+                TestModel(i=1, f=value)
+

From 434747894c888f33a1442c42d219a0cd44f35d82 Mon Sep 17 00:00:00 2001
From: Kirill Potekhin <potehik@gmail.com>
Date: Thu, 20 Dec 2018 01:37:15 +0300
Subject: [PATCH 11/30] UUIDField added to docs

---
 docs/field_types.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/field_types.md b/docs/field_types.md
index d94782e..b23bde3 100644
--- a/docs/field_types.md
+++ b/docs/field_types.md
@@ -25,6 +25,7 @@ Currently the following field types are supported:
 | Decimal32Field     | Decimal32  | Decimal             | Ditto
 | Decimal64Field     | Decimal64  | Decimal             | Ditto
 | Decimal128Field    | Decimal128 | Decimal             | Ditto
+| UUIDField          | UUID       | Decimal             | 
 | Enum8Field         | Enum8      | Enum                | See below
 | Enum16Field        | Enum16     | Enum                | See below
 | ArrayField         | Array      | list                | See below

From e7542dadbd03ae63669943ac2f6855bac50ad1ad Mon Sep 17 00:00:00 2001
From: Kirill Potekhin <potehik@gmail.com>
Date: Thu, 20 Dec 2018 01:41:42 +0300
Subject: [PATCH 12/30] custom UUIDField deleted from tests and docs

---
 docs/field_types.md         | 40 -----------------------
 tests/test_custom_fields.py | 63 -------------------------------------
 2 files changed, 103 deletions(-)

diff --git a/docs/field_types.md b/docs/field_types.md
index b23bde3..50234e8 100644
--- a/docs/field_types.md
+++ b/docs/field_types.md
@@ -184,46 +184,6 @@ class BooleanField(Field):
         return '1' if value else '0'
 ```
 
-Here's another example - a field for storing UUIDs in the database as 16-byte strings. We'll use Python's built-in `UUID` class to handle the conversion from strings, ints and tuples into UUID instances. So in our Python code we'll have the convenience of working with UUID objects, but they will be stored in the database as efficiently as possible:
-
-```python
-from infi.clickhouse_orm.fields import Field
-from infi.clickhouse_orm.utils import escape
-from uuid import UUID
-import six
-
-class UUIDField(Field):
-
-    # The ClickHouse column type to use
-    db_type = 'FixedString(16)'
-
-    # The default value if empty
-    class_default = UUID(int=0)
-
-    def to_python(self, value, timezone_in_use):
-        # Convert valid values to UUID instance
-        if isinstance(value, UUID):
-            return value
-        elif isinstance(value, six.string_types):
-            return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
-        elif isinstance(value, six.integer_types):
-            return UUID(int=value)
-        elif isinstance(value, tuple):
-            return UUID(fields=value)
-        else:
-            raise ValueError('Invalid value for UUIDField: %r' % value)
-
-    def to_db_string(self, value, quote=True):
-        # The value was already converted by to_python, so it's a UUID instance
-        val = value.bytes
-        if six.PY3:
-            val = str(val, 'latin1')
-        return escape(val, quote)
-
-```
-
-Note that the latin-1 encoding is used as an identity encoding for converting between raw bytes and strings. This is required in Python 3, where `str` and `bytes` are different types.
-
 ---
 
 [<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md)
\ No newline at end of file
diff --git a/tests/test_custom_fields.py b/tests/test_custom_fields.py
index fc52e51..76dbd15 100644
--- a/tests/test_custom_fields.py
+++ b/tests/test_custom_fields.py
@@ -1,12 +1,9 @@
 from __future__ import unicode_literals
 import unittest
-import six
-from uuid import UUID
 from infi.clickhouse_orm.database import Database
 from infi.clickhouse_orm.fields import Field, Int16Field
 from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.engines import Memory
-from infi.clickhouse_orm.utils import escape
 
 
 class CustomFieldsTest(unittest.TestCase):
@@ -35,37 +32,6 @@ class CustomFieldsTest(unittest.TestCase):
             with self.assertRaises(ValueError):
                 TestModel(i=1, f=value)
 
-    def test_uuid_field(self):
-        # Create a model
-        class TestModel(Model):
-            i = Int16Field()
-            f = UUIDField()
-            engine = Memory()
-        self.database.create_table(TestModel)
-        # Check valid values (all values are the same UUID)
-        values = [
-            '{12345678-1234-5678-1234-567812345678}',
-            '12345678123456781234567812345678',
-            'urn:uuid:12345678-1234-5678-1234-567812345678',
-            '\x12\x34\x56\x78'*4,
-            (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
-            0x12345678123456781234567812345678,
-        ]
-        for index, value in enumerate(values):
-            rec = TestModel(i=index, f=value)
-            self.database.insert([rec])
-        for rec in TestModel.objects_in(self.database):
-            self.assertEqual(rec.f, UUID(values[0]))
-        # Check that ClickHouse encoding functions are supported
-        for rec in self.database.select("SELECT i, UUIDNumToString(f) AS f FROM testmodel", TestModel):
-            self.assertEqual(rec.f, UUID(values[0]))
-        for rec in self.database.select("SELECT 1 as i, UUIDStringToNum('12345678-1234-5678-1234-567812345678') AS f", TestModel):
-            self.assertEqual(rec.f, UUID(values[0]))
-        # Check invalid values
-        for value in [None, 'zzz', -1, '123']:
-            with self.assertRaises(ValueError):
-                TestModel(i=1, f=value)
-
 
 class BooleanField(Field):
 
@@ -88,32 +54,3 @@ class BooleanField(Field):
         # The value was already converted by to_python, so it's a bool
         return '1' if value else '0'
 
-
-class UUIDField(Field):
-
-    # The ClickHouse column type to use
-    db_type = 'FixedString(16)'
-
-    # The default value if empty
-    class_default = UUID(int=0)
-
-    def to_python(self, value, timezone_in_use):
-        # Convert valid values to UUID instance
-        if isinstance(value, UUID):
-            return value
-        elif isinstance(value, six.string_types):
-            return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
-        elif isinstance(value, six.integer_types):
-            return UUID(int=value)
-        elif isinstance(value, tuple):
-            return UUID(fields=value)
-        else:
-            raise ValueError('Invalid value for UUIDField: %r' % value)
-
-    def to_db_string(self, value, quote=True):
-        # The value was already converted by to_python, so it's a UUID instance
-        val = value.bytes
-        if six.PY3:
-            val = str(val, 'latin1')
-        return escape(val, quote)
-

From cb91971fd911ffafec804bfd7c8aef05ca026ad4 Mon Sep 17 00:00:00 2001
From: Kuzma Leshakov <leshakov.kuzma@clover.global>
Date: Fri, 22 Feb 2019 14:38:12 +0300
Subject: [PATCH 13/30] Fix #110

---
 docs/field_types.md               | 2 +-
 docs/models_and_databases.md      | 2 +-
 src/infi/clickhouse_orm/fields.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/field_types.md b/docs/field_types.md
index d94782e..cf73646 100644
--- a/docs/field_types.md
+++ b/docs/field_types.md
@@ -9,7 +9,7 @@ Currently the following field types are supported:
 | ------------------ | ---------- | ------------------- | -----------------------------------------------------
 | StringField        | String     | unicode             | Encoded as UTF-8 when written to ClickHouse
 | FixedStringField   | String     | unicode             | Encoded as UTF-8 when written to ClickHouse
-| DateField          | Date       | datetime.date       | Range 1970-01-01 to 2038-01-19
+| DateField          | Date       | datetime.date       | Range 1970-01-01 to 2105-12-31
 | DateTimeField      | DateTime   | datetime.datetime   | Minimal value is 1970-01-01 00:00:00; Always in UTC
 | Int8Field          | Int8       | int                 | Range -128 to 127
 | Int16Field         | Int16      | int                 | Range -32768 to 32767
diff --git a/docs/models_and_databases.md b/docs/models_and_databases.md
index b062608..59947db 100644
--- a/docs/models_and_databases.md
+++ b/docs/models_and_databases.md
@@ -89,7 +89,7 @@ When values are assigned to model fields, they are immediately converted to thei
     >>> suzy.birthday = 0.5
     ValueError: Invalid value for DateField - 0.5
     >>> suzy.birthday = '1922-05-31'
-    ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2038-01-19
+    ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2105-12-31
 
 Inserting to the Database
 -------------------------
diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py
index 8c11805..b2cddfb 100644
--- a/src/infi/clickhouse_orm/fields.py
+++ b/src/infi/clickhouse_orm/fields.py
@@ -131,7 +131,7 @@ class FixedStringField(StringField):
 class DateField(Field):
 
     min_value = datetime.date(1970, 1, 1)
-    max_value = datetime.date(2038, 1, 19)
+    max_value = datetime.date(2105, 12, 31)
     class_default = min_value
     db_type = 'Date'
 

From fb5577bedd5f8db6902cf6ed7dadb445e05fae6f Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Tue, 26 Feb 2019 21:40:55 +0200
Subject: [PATCH 14/30] Update changelog

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e8efbb..588125b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 Change Log
 ==========
 
+Unreleased
+----------
+- Extend date field range (trthhrtz)
+
 v1.0.4
 ------
 - Added `timeout` parameter to database initializer (SUHAR1K)

From bec45b53fa93c8154d6d5a5370f1207c32dcfdcc Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Tue, 26 Feb 2019 22:46:00 +0200
Subject: [PATCH 15/30] Fix parsing of server errors in ClickHouse v19.3.3+

---
 CHANGELOG.md                        |  1 +
 src/infi/clickhouse_orm/database.py | 27 ++++++++++++++++--------
 tests/test_server_errors.py         | 32 +++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 9 deletions(-)
 create mode 100644 tests/test_server_errors.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 588125b..3abc3d6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ Change Log
 Unreleased
 ----------
 - Extend date field range (trthhrtz)
+- Fix parsing of server errors in ClickHouse v19.3.3+
 
 v1.0.4
 ------
diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py
index 06d43ba..7e72e99 100644
--- a/src/infi/clickhouse_orm/database.py
+++ b/src/infi/clickhouse_orm/database.py
@@ -40,11 +40,19 @@ class ServerError(DatabaseException):
             self.message = message
             super(ServerError, self).__init__(message)
 
-    ERROR_PATTERN = re.compile(r'''
-        Code:\ (?P<code>\d+),
-        \ e\.displayText\(\)\ =\ (?P<type1>[^ \n]+):\ (?P<msg>.+?),
-        \ e.what\(\)\ =\ (?P<type2>[^ \n]+)
-    ''', re.VERBOSE | re.DOTALL)
+    ERROR_PATTERNS = (
+        # ClickHouse prior to v19.3.3
+        re.compile(r'''
+            Code:\ (?P<code>\d+),
+            \ e\.displayText\(\)\ =\ (?P<type1>[^ \n]+):\ (?P<msg>.+?),
+            \ e.what\(\)\ =\ (?P<type2>[^ \n]+)
+        ''', re.VERBOSE | re.DOTALL),
+        # ClickHouse v19.3.3+
+        re.compile(r'''
+            Code:\ (?P<code>\d+),
+            \ e\.displayText\(\)\ =\ (?P<type1>[^ \n]+):\ (?P<msg>.+)
+        ''', re.VERBOSE | re.DOTALL),
+    )
 
     @classmethod
     def get_error_code_msg(cls, full_error_message):
@@ -54,10 +62,11 @@ class ServerError(DatabaseException):
         See the list of error codes here:
         https://github.com/yandex/ClickHouse/blob/master/dbms/src/Common/ErrorCodes.cpp
         """
-        match = cls.ERROR_PATTERN.match(full_error_message)
-        if match:
-            # assert match.group('type1') == match.group('type2')
-            return int(match.group('code')), match.group('msg')
+        for pattern in cls.ERROR_PATTERNS:
+            match = pattern.match(full_error_message)
+            if match:
+                # assert match.group('type1') == match.group('type2')
+                return int(match.group('code')), match.group('msg').strip()
 
         return 0, full_error_message
 
diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py
new file mode 100644
index 0000000..8159369
--- /dev/null
+++ b/tests/test_server_errors.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+import unittest
+
+from infi.clickhouse_orm.database import ServerError
+
+
+class ServerErrorTest(unittest.TestCase):
+
+    def test_old_format(self):
+
+        code, msg = ServerError.get_error_code_msg("Code: 81, e.displayText() = DB::Exception: Database db_not_here doesn't exist, e.what() = DB::Exception (from [::1]:33458)")
+        self.assertEquals(code, 81)
+        self.assertEquals(msg, "Database db_not_here doesn't exist")
+
+        code, msg = ServerError.get_error_code_msg("Code: 161, e.displayText() = DB::Exception: Limit for number of columns to read exceeded. Requested: 11, maximum: 1, e.what() = DB::Exception\n")
+        self.assertEquals(code, 161)
+        self.assertEquals(msg, "Limit for number of columns to read exceeded. Requested: 11, maximum: 1")
+
+
+    def test_new_format(self):
+
+        code, msg = ServerError.get_error_code_msg("Code: 164, e.displayText() = DB::Exception: Cannot drop table in readonly mode")
+        self.assertEquals(code, 164)
+        self.assertEquals(msg, "Cannot drop table in readonly mode")
+
+        code, msg = ServerError.get_error_code_msg("Code: 48, e.displayText() = DB::Exception: Method write is not supported by storage Merge")
+        self.assertEquals(code, 48)
+        self.assertEquals(msg, "Method write is not supported by storage Merge")
+
+        code, msg = ServerError.get_error_code_msg("Code: 60, e.displayText() = DB::Exception: Table default.zuzu doesn't exist.\n")
+        self.assertEquals(code, 60)
+        self.assertEquals(msg, "Table default.zuzu doesn't exist.")

From 81e942a4d2cff23fb991228a5257ad9d89f4be46 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Tue, 26 Feb 2019 22:58:59 +0200
Subject: [PATCH 16/30] Fix pagination when asking for the last page on a query
 that matches no records

---
 CHANGELOG.md                        | 1 +
 src/infi/clickhouse_orm/database.py | 4 ++--
 tests/test_database.py              | 8 ++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3abc3d6..a784f73 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ Unreleased
 ----------
 - Extend date field range (trthhrtz)
 - Fix parsing of server errors in ClickHouse v19.3.3+
+- Fix pagination when asking for the last page on a query that matches no records
 
 v1.0.4
 ------
diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py
index 7e72e99..f7b6e87 100644
--- a/src/infi/clickhouse_orm/database.py
+++ b/src/infi/clickhouse_orm/database.py
@@ -285,7 +285,7 @@ class Database(object):
         count = self.count(model_class, conditions)
         pages_total = int(ceil(count / float(page_size)))
         if page_num == -1:
-            page_num = pages_total
+            page_num = max(pages_total, 1)
         elif page_num < 1:
             raise ValueError('Invalid page number: %d' % page_num)
         offset = (page_num - 1) * page_size
@@ -296,7 +296,7 @@ class Database(object):
         query += ' LIMIT %d, %d' % (offset, page_size)
         query = self._substitute(query, model_class)
         return Page(
-            objects=list(self.select(query, model_class, settings)),
+            objects=list(self.select(query, model_class, settings)) if count else [],
             number_of_objects=count,
             pages_total=pages_total,
             number=page_num,
diff --git a/tests/test_database.py b/tests/test_database.py
index dd8776e..c563862 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -112,6 +112,14 @@ class DatabaseTestCase(TestCaseWithData):
             self.assertEqual([obj.to_tsv() for obj in page_a.objects],
                               [obj.to_tsv() for obj in page_b.objects])
 
+    def test_pagination_empty_page(self):
+        for page_num in (-1, 1, 2):
+            page = self.database.paginate(Person, 'first_name, last_name', page_num, 10, conditions="first_name = 'Ziggy'")
+            self.assertEqual(page.number_of_objects, 0)
+            self.assertEqual(page.objects, [])
+            self.assertEqual(page.pages_total, 0)
+            self.assertEqual(page.number, max(page_num, 1))
+
     def test_pagination_invalid_page(self):
         self._insert_and_check(self._sample_data(), len(data))
         for page_num in (0, -2, -100):

From b8fd39c6a6b5fcd5a28a9b757c5164fb447a7a48 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Tue, 26 Feb 2019 23:12:32 +0200
Subject: [PATCH 17/30] Use HTTP Basic Authentication instead of passing the
 credentials in the URL

---
 CHANGELOG.md                        | 1 +
 src/infi/clickhouse_orm/database.py | 8 ++------
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a784f73..66ce749 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ Unreleased
 - Extend date field range (trthhrtz)
 - Fix parsing of server errors in ClickHouse v19.3.3+
 - Fix pagination when asking for the last page on a query that matches no records
+- Use HTTP Basic Authentication instead of passing the credentials in the URL
 
 v1.0.4
 ------
diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py
index f7b6e87..b7379da 100644
--- a/src/infi/clickhouse_orm/database.py
+++ b/src/infi/clickhouse_orm/database.py
@@ -99,12 +99,12 @@ class Database(object):
         '''
         self.db_name = db_name
         self.db_url = db_url
-        self.username = username
-        self.password = password
         self.readonly = False
         self.timeout = timeout
         self.request_session = requests.Session()
         self.request_session.verify = verify_ssl_cert
+        if username:
+            self.request_session.auth = (username, password or '')
         self.settings = {}
         self.db_exists = False # this is required before running _is_existing_database
         self.db_exists = self._is_existing_database()
@@ -345,10 +345,6 @@ class Database(object):
         params.update(self.settings)
         if self.db_exists:
             params['database'] = self.db_name
-        if self.username:
-            params['user'] = self.username
-        if self.password:
-            params['password'] = self.password
         # Send the readonly flag, unless the connection is already readonly (to prevent db error)
         if self.readonly and not self.connection_readonly:
             params['readonly'] = '1'

From 3bbf06a70ac13cefebdcf743a4bb863e42366d9c Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Tue, 26 Feb 2019 23:24:45 +0200
Subject: [PATCH 18/30] TRIVIAL assertEquals ==> assertEqual

---
 tests/test_server_errors.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py
index 8159369..7445e5f 100644
--- a/tests/test_server_errors.py
+++ b/tests/test_server_errors.py
@@ -9,24 +9,24 @@ class ServerErrorTest(unittest.TestCase):
     def test_old_format(self):
 
         code, msg = ServerError.get_error_code_msg("Code: 81, e.displayText() = DB::Exception: Database db_not_here doesn't exist, e.what() = DB::Exception (from [::1]:33458)")
-        self.assertEquals(code, 81)
-        self.assertEquals(msg, "Database db_not_here doesn't exist")
+        self.assertEqual(code, 81)
+        self.assertEqual(msg, "Database db_not_here doesn't exist")
 
         code, msg = ServerError.get_error_code_msg("Code: 161, e.displayText() = DB::Exception: Limit for number of columns to read exceeded. Requested: 11, maximum: 1, e.what() = DB::Exception\n")
-        self.assertEquals(code, 161)
-        self.assertEquals(msg, "Limit for number of columns to read exceeded. Requested: 11, maximum: 1")
+        self.assertEqual(code, 161)
+        self.assertEqual(msg, "Limit for number of columns to read exceeded. Requested: 11, maximum: 1")
 
 
     def test_new_format(self):
 
         code, msg = ServerError.get_error_code_msg("Code: 164, e.displayText() = DB::Exception: Cannot drop table in readonly mode")
-        self.assertEquals(code, 164)
-        self.assertEquals(msg, "Cannot drop table in readonly mode")
+        self.assertEqual(code, 164)
+        self.assertEqual(msg, "Cannot drop table in readonly mode")
 
         code, msg = ServerError.get_error_code_msg("Code: 48, e.displayText() = DB::Exception: Method write is not supported by storage Merge")
-        self.assertEquals(code, 48)
-        self.assertEquals(msg, "Method write is not supported by storage Merge")
+        self.assertEqual(code, 48)
+        self.assertEqual(msg, "Method write is not supported by storage Merge")
 
         code, msg = ServerError.get_error_code_msg("Code: 60, e.displayText() = DB::Exception: Table default.zuzu doesn't exist.\n")
-        self.assertEquals(code, 60)
-        self.assertEquals(msg, "Table default.zuzu doesn't exist.")
+        self.assertEqual(code, 60)
+        self.assertEqual(msg, "Table default.zuzu doesn't exist.")

From 6e30ce0130769fe2deb06f2c1145094f5381c54f Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Tue, 26 Feb 2019 23:39:38 +0200
Subject: [PATCH 19/30] Replace enum34 with enum-compat

---
 buildout.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildout.cfg b/buildout.cfg
index 6ef77b1..67f93bb 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -49,7 +49,7 @@ eggs = ${project:name}
 	ipython<6
 	nose
 	coverage
-	enum34
+	enum-compat
 	infi.unittest
 	infi.traceback
 	memory_profiler

From ad81a811fcb1197e4c2800fb80b4b8ca3b079cf4 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Wed, 27 Feb 2019 08:41:16 +0200
Subject: [PATCH 20/30] Memory engine does not support PREWHERE

---
 src/infi/clickhouse_orm/query.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 9eee645..10dfe97 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -344,10 +344,10 @@ class QuerySet(object):
         params = (distinct, self.select_fields_as_sql(), self._model_cls.table_name(), final)
         sql = u'SELECT %s%s\nFROM `%s`%s' % params
 
-        if self._prewhere_q:
+        if self._prewhere_q and not self._prewhere_q.is_empty:
             sql += '\nPREWHERE ' + self.conditions_as_sql(prewhere=True)
 
-        if self._where_q:
+        if self._where_q and not self._where_q.is_empty:
             sql += '\nWHERE ' + self.conditions_as_sql(prewhere=False)
 
         if self._grouping_fields:
@@ -410,7 +410,7 @@ class QuerySet(object):
         return qs
 
     def _filter_or_exclude(self, *q, **kwargs):
-        reverse = kwargs.pop('reverse', False)
+        inverse = kwargs.pop('_inverse', False)
         prewhere = kwargs.pop('prewhere', False)
 
         qs = copy(self)
@@ -422,7 +422,7 @@ class QuerySet(object):
         if kwargs:
             condition &= Q(**kwargs)
 
-        if reverse:
+        if inverse:
             condition = ~condition
 
         condition = copy(self._prewhere_q if prewhere else self._where_q) & condition
@@ -436,15 +436,16 @@ class QuerySet(object):
     def filter(self, *q, **kwargs):
         """
         Returns a copy of this queryset that includes only rows matching the conditions.
-        Add q object to query if it specified.
+        Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
         """
         return self._filter_or_exclude(*q, **kwargs)
 
     def exclude(self, *q, **kwargs):
         """
         Returns a copy of this queryset that excludes all rows matching the conditions.
+        Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
         """
-        return self._filter_or_exclude(*q, reverse=True, **kwargs)
+        return self._filter_or_exclude(*q, _inverse=True, **kwargs)
 
     def paginate(self, page_num=1, page_size=100):
         """

From ba8bb1d5f499592e44dddb889145188465e74463 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Wed, 27 Feb 2019 08:42:09 +0200
Subject: [PATCH 21/30] Update docs

---
 CHANGELOG.md            |  1 +
 docs/class_reference.md | 24 ++++++++++++++++--------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 66ce749..158e22d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@ Change Log
 
 Unreleased
 ----------
+- Add PREWHERE support to querysets (M1hacka)
 - Extend date field range (trthhrtz)
 - Fix parsing of server errors in ClickHouse v19.3.3+
 - Fix pagination when asking for the last page on a query that matches no records
diff --git a/docs/class_reference.md b/docs/class_reference.md
index 6eb2353..4876817 100644
--- a/docs/class_reference.md
+++ b/docs/class_reference.md
@@ -835,7 +835,7 @@ is equivalent to:
 Returns the whole query as a SQL string.
 
 
-#### conditions_as_sql(prewhere=True)
+#### conditions_as_sql(prewhere=False)
 
 
 Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
@@ -854,17 +854,18 @@ Adds a DISTINCT clause to the query, meaning that any duplicate rows
 in the results will be omitted.
 
 
-#### exclude(**filter_fields)
+#### exclude(*q, **kwargs)
 
 
 Returns a copy of this queryset that excludes all rows matching the conditions.
+Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
 
 
-#### filter(*q, **filter_fields)
+#### filter(*q, **kwargs)
 
 
 Returns a copy of this queryset that includes only rows matching the conditions.
-Add q object to query if it specified.
+Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
 
 
 #### final()
@@ -908,6 +909,9 @@ The result is a namedtuple containing `objects` (list), `number_of_objects`,
 `pages_total`, `number` (of the current page), and `page_size`.
 
 
+#### select_fields_as_sql()
+
+
 ### AggregateQuerySet
 
 Extends QuerySet
@@ -943,7 +947,7 @@ This method is not supported on `AggregateQuerySet`.
 Returns the whole query as a SQL string.
 
 
-#### conditions_as_sql(prewhere=True)
+#### conditions_as_sql(prewhere=False)
 
 
 Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
@@ -962,17 +966,18 @@ Adds a DISTINCT clause to the query, meaning that any duplicate rows
 in the results will be omitted.
 
 
-#### exclude(**filter_fields)
+#### exclude(*q, **kwargs)
 
 
 Returns a copy of this queryset that excludes all rows matching the conditions.
+Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
 
 
-#### filter(*q, **filter_fields)
+#### filter(*q, **kwargs)
 
 
 Returns a copy of this queryset that includes only rows matching the conditions.
-Add q object to query if it specified.
+Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
 
 
 #### final()
@@ -1022,3 +1027,6 @@ The result is a namedtuple containing `objects` (list), `number_of_objects`,
 `pages_total`, `number` (of the current page), and `page_size`.
 
 
+#### select_fields_as_sql()
+
+

From 191eac44243db25b6e8bf108454956136a4c7527 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Wed, 27 Feb 2019 08:56:04 +0200
Subject: [PATCH 22/30] Remove unused method

---
 src/infi/clickhouse_orm/query.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 083c595..728b977 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -576,9 +576,6 @@ class AggregateQuerySet(QuerySet):
     def select_fields_as_sql(self):
         return comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
 
-    def group_by_as_sql(self):
-        return 'GROUP BY'
-
     def __iter__(self):
         return self._database.select(self.as_sql()) # using an ad-hoc model
 

From 7946a2a2725ff33706534cdcac1e1eacc13e2ec3 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Wed, 27 Feb 2019 08:58:41 +0200
Subject: [PATCH 23/30] Update docs

---
 CHANGELOG.md                     |  1 +
 docs/class_reference.md          | 14 ++++++++++++++
 docs/querysets.md                | 20 ++++++++++----------
 src/infi/clickhouse_orm/query.py |  6 ++++++
 4 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 158e22d..400cb6c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ Change Log
 Unreleased
 ----------
 - Add PREWHERE support to querysets (M1hacka)
+- Add WITH TOTALS support to querysets (M1hacka)
 - Extend date field range (trthhrtz)
 - Fix parsing of server errors in ClickHouse v19.3.3+
 - Fix pagination when asking for the last page on a query that matches no records
diff --git a/docs/class_reference.md b/docs/class_reference.md
index 4876817..49e71f3 100644
--- a/docs/class_reference.md
+++ b/docs/class_reference.md
@@ -912,6 +912,9 @@ The result is a namedtuple containing `objects` (list), `number_of_objects`,
 #### select_fields_as_sql()
 
 
+Returns the selected fields or expressions as a SQL string.
+
+
 ### AggregateQuerySet
 
 Extends QuerySet
@@ -1030,3 +1033,14 @@ The result is a namedtuple containing `objects` (list), `number_of_objects`,
 #### select_fields_as_sql()
 
 
+Returns the selected fields or expressions as a SQL string.
+
+
+#### with_totals()
+
+
+Adds WITH TOTALS modifier ot GROUP BY, making query return extra row
+with aggregate function calculated across all the rows. More information:
+https://clickhouse.yandex/docs/en/query_language/select/#with-totals-modifier
+
+
diff --git a/docs/querysets.md b/docs/querysets.md
index 09465df..056e794 100644
--- a/docs/querysets.md
+++ b/docs/querysets.md
@@ -32,14 +32,14 @@ For filters with compound conditions you can use `Q` objects inside `filter` wit
     >>> qs.conditions_as_sql()
     u"((first_name = 'Ciaran' AND last_name = 'Carver') OR height <= 1.8) AND (NOT (first_name = 'David'))"
 
-By default conditions from `filter` and `exclude` methods are add to `WHERE` clause. 
+By default conditions from `filter` and `exclude` methods are add to `WHERE` clause.
 For better aggregation performance you can add them to `PREWHERE` section using `prewhere=True` parameter
 
     >>> qs = Person.objects_in(database)
     >>> qs = qs.filter(first_name__startswith='V', prewhere=True)
     >>> qs.conditions_as_sql(prewhere=True)
     u"first_name LIKE 'V%'"
-    
+
 There are different operators that can be used, by passing `<fieldname>__<operator>=<value>` (two underscores separate the field name from the operator). In case no operator is given, `eq` is used by default. Below are all the supported operators.
 
 | Operator       | Equivalent SQL                               | Comments                           |
@@ -128,14 +128,14 @@ Adds a DISTINCT clause to the query, meaning that any duplicate rows in the resu
 Final
 --------
 
-This method can be used only with CollapsingMergeTree engine.  
+This method can be used only with CollapsingMergeTree engine.
 Adds a FINAL modifier to the query, meaning data is selected fully "collapsed" by sign field.
 
     >>> Person.objects_in(database).count()
     100
     >>> Person.objects_in(database).final().count()
     94
-    
+
 Slicing
 -------
 
@@ -214,14 +214,14 @@ If you limit aggregation results, it might be useful to get total aggregation va
 To achieve this, you can use `with_totals` method. It will return extra row (last) with
 values aggregated for all rows suitable for filters.
 
-    qs = Person.objects_in(database).aggregate('first_name' num='count()').with_totals().order_by('-count')[:3]
+    qs = Person.objects_in(database).aggregate('first_name', num='count()').with_totals().order_by('-count')[:3]
     >>> print qs.count()
     4
-    >>> for row in qs: 
-    >>>     print(row.first_name, row.count)
-    'Cassandra' 2
-    'Alexandra' 2
-    '' 100
+    >>> for row in qs:
+    >>>     print("'{}': {}".format(row.first_name, row.count))
+    'Cassandra': 2
+    'Alexandra': 2
+    '': 100
 
 ---
 
diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index 728b977..ab7a705 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -333,6 +333,9 @@ class QuerySet(object):
             return qs
 
     def select_fields_as_sql(self):
+        """
+        Returns the selected fields or expressions as a SQL string.
+        """
         return comma_join('`%s`' % field for field in self._fields) if self._fields else '*'
 
     def as_sql(self):
@@ -574,6 +577,9 @@ class AggregateQuerySet(QuerySet):
         raise NotImplementedError('Cannot re-aggregate an AggregateQuerySet')
 
     def select_fields_as_sql(self):
+        """
+        Returns the selected fields or expressions as a SQL string.
+        """
         return comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
 
     def __iter__(self):

From a765b5fe5fc5a7c9a6e41f369162b13731cc3226 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Sun, 12 May 2019 10:27:10 +0300
Subject: [PATCH 24/30] Format of database error messages changed, update
 failing tests

---
 tests/test_database.py | 4 ++--
 tests/test_engines.py  | 2 +-
 tests/test_readonly.py | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_database.py b/tests/test_database.py
index c563862..2a50590 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -150,7 +150,7 @@ class DatabaseTestCase(TestCaseWithData):
 
         exc = cm.exception
         self.assertEqual(exc.code, 193)
-        self.assertEqual(exc.message, 'Wrong password for user default')
+        self.assertTrue(exc.message.startswith('Wrong password for user default'))
 
     def test_nonexisting_db(self):
         db = Database('db_not_here', autocreate=False)
@@ -158,7 +158,7 @@ class DatabaseTestCase(TestCaseWithData):
             db.create_table(Person)
         exc = cm.exception
         self.assertEqual(exc.code, 81)
-        self.assertEqual(exc.message, "Database db_not_here doesn't exist")
+        self.assertTrue(exc.message.startswith("Database db_not_here doesn't exist"))
         # Create and delete the db twice, to ensure db_exists gets updated
         for i in range(2):
             # Now create the database - should succeed
diff --git a/tests/test_engines.py b/tests/test_engines.py
index 7e29426..846e825 100644
--- a/tests/test_engines.py
+++ b/tests/test_engines.py
@@ -209,7 +209,7 @@ class DistributedTestCase(_EnginesHelperTestCase):
 
         exc = cm.exception
         self.assertEqual(exc.code, 170)
-        self.assertEqual(exc.message, "Requested cluster 'cluster_name' not found")
+        self.assertTrue(exc.message.startswith("Requested cluster 'cluster_name' not found"))
 
     def test_verbose_engine_two_superclasses(self):
         class TestModel2(SampleModel):
diff --git a/tests/test_readonly.py b/tests/test_readonly.py
index b9333af..816c701 100644
--- a/tests/test_readonly.py
+++ b/tests/test_readonly.py
@@ -36,9 +36,9 @@ class ReadonlyTestCase(TestCaseWithData):
     def _check_db_readonly_err(self, exc, drop_table=None):
         self.assertEqual(exc.code, 164)
         if drop_table:
-            self.assertEqual(exc.message, 'Cannot drop table in readonly mode')
+            self.assertTrue(exc.message.startswith('Cannot drop table in readonly mode'))
         else:
-            self.assertEqual(exc.message, 'Cannot insert into table in readonly mode')
+            self.assertTrue(exc.message.startswith('Cannot insert into table in readonly mode'))
 
     def test_readonly_db_with_default_user(self):
         self._test_readonly_db('default')

From cadba6f6aabafd22011f4fb7e413b310e83d9237 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Thu, 13 Jun 2019 05:07:56 +0300
Subject: [PATCH 25/30] Support default/alias/materialized for nullable fields

---
 CHANGELOG.md                      |  1 +
 src/infi/clickhouse_orm/fields.py | 11 ++++++++++-
 tests/test_nullable_fields.py     | 23 ++++++++++++++++++++---
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 400cb6c..e679bd8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ Unreleased
 - Fix parsing of server errors in ClickHouse v19.3.3+
 - Fix pagination when asking for the last page on a query that matches no records
 - Use HTTP Basic Authentication instead of passing the credentials in the URL
+- Support default/alias/materialized for nullable fields
 
 v1.0.4
 ------
diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py
index b2cddfb..88fd7bb 100644
--- a/src/infi/clickhouse_orm/fields.py
+++ b/src/infi/clickhouse_orm/fields.py
@@ -478,4 +478,13 @@ class NullableField(Field):
         return self.inner_field.to_db_string(value, quote=quote)
 
     def get_sql(self, with_default_expression=True):
-        return 'Nullable(%s)' % self.inner_field.get_sql(with_default_expression=False)
+        s = 'Nullable(%s)' % self.inner_field.get_sql(with_default_expression=False)
+        if with_default_expression:
+            if self.alias:
+                s = '%s ALIAS %s' % (s, self.alias)
+            elif self.materialized:
+                s = '%s MATERIALIZED %s' % (s, self.materialized)
+            elif self.default:
+                default = self.to_db_string(self.default)
+                s = '%s DEFAULT %s' % (s, default)
+        return s
diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py
index 8234977..74bc66e 100644
--- a/tests/test_nullable_fields.py
+++ b/tests/test_nullable_fields.py
@@ -6,6 +6,7 @@ from infi.clickhouse_orm.database import Database
 from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
+from infi.clickhouse_orm.utils import comma_join
 
 from datetime import date, datetime
 
@@ -95,14 +96,19 @@ class NullableFieldsTest(unittest.TestCase):
             ModelWithNullable(date_field='2016-08-30', null_str='', null_int=42, null_date=dt),
             ModelWithNullable(date_field='2016-08-30', null_str='nothing', null_int=None, null_date=None),
             ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=42, null_date=dt),
-            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None)
+            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None, null_default=None)
         ])
 
     def _assert_sample_data(self, results):
+        for r in results:
+            print(r.to_dict())
         dt = date(1970, 1, 1)
         self.assertEqual(len(results), 4)
         self.assertIsNone(results[0].null_str)
         self.assertEqual(results[0].null_int, 42)
+        self.assertEqual(results[0].null_default, 7)
+        self.assertEqual(results[0].null_alias, 21)
+        self.assertEqual(results[0].null_materialized, 420)
         self.assertEqual(results[0].null_date, dt)
         self.assertIsNone(results[1].null_date)
         self.assertEqual(results[1].null_str, 'nothing')
@@ -110,19 +116,27 @@ class NullableFieldsTest(unittest.TestCase):
         self.assertIsNone(results[2].null_str)
         self.assertEqual(results[2].null_date, dt)
         self.assertEqual(results[2].null_int, 42)
+        self.assertEqual(results[2].null_default, 7)
+        self.assertEqual(results[2].null_alias, 21)
+        self.assertEqual(results[0].null_materialized, 420)
         self.assertIsNone(results[3].null_int)
+        self.assertIsNone(results[3].null_default)
+        self.assertIsNone(results[3].null_alias)
+        self.assertIsNone(results[3].null_materialized)
         self.assertIsNone(results[3].null_str)
         self.assertIsNone(results[3].null_date)
 
     def test_insert_and_select(self):
         self._insert_sample_data()
-        query = 'SELECT * from $table ORDER BY date_field'
+        fields = comma_join(ModelWithNullable.fields().keys())
+        query = 'SELECT %s from $table ORDER BY date_field' % fields
         results = list(self.database.select(query, ModelWithNullable))
         self._assert_sample_data(results)
 
     def test_ad_hoc_model(self):
         self._insert_sample_data()
-        query = 'SELECT * from $db.modelwithnullable ORDER BY date_field'
+        fields = comma_join(ModelWithNullable.fields().keys())
+        query = 'SELECT %s from $db.modelwithnullable ORDER BY date_field' % fields
         results = list(self.database.select(query))
         self._assert_sample_data(results)
 
@@ -133,5 +147,8 @@ class ModelWithNullable(Model):
     null_str = NullableField(StringField(), extra_null_values={''})
     null_int = NullableField(Int32Field())
     null_date = NullableField(DateField())
+    null_default = NullableField(Int32Field(), default=7)
+    null_alias = NullableField(Int32Field(), alias='null_int/2')
+    null_materialized = NullableField(Int32Field(), alias='null_int*10')
 
     engine = MergeTree('date_field', ('date_field',))

From 7a2c8515a21d4abf258eabe0a4c684f2fcb80581 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Thu, 13 Jun 2019 05:19:16 +0300
Subject: [PATCH 26/30] Update docs

---
 CHANGELOG.md                      | 1 +
 docs/class_reference.md           | 7 +++++++
 docs/toc.md                       | 1 +
 src/infi/clickhouse_orm/fields.py | 1 +
 tests/test_uuid_fields.py         | 1 +
 5 files changed, 11 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e679bd8..8f82621 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ Unreleased
 - Fix pagination when asking for the last page on a query that matches no records
 - Use HTTP Basic Authentication instead of passing the credentials in the URL
 - Support default/alias/materialized for nullable fields
+- Add UUIDField (kpotehin)
 
 v1.0.4
 ------
diff --git a/docs/class_reference.md b/docs/class_reference.md
index 49e71f3..fb41cfd 100644
--- a/docs/class_reference.md
+++ b/docs/class_reference.md
@@ -701,6 +701,13 @@ Extends BaseIntField
 #### UInt8Field(default=None, alias=None, materialized=None, readonly=None)
 
 
+### UUIDField
+
+Extends Field
+
+#### UUIDField(default=None, alias=None, materialized=None, readonly=None)
+
+
 infi.clickhouse_orm.engines
 ---------------------------
 
diff --git a/docs/toc.md b/docs/toc.md
index 9192404..05f384b 100644
--- a/docs/toc.md
+++ b/docs/toc.md
@@ -92,6 +92,7 @@
          * [UInt32Field](class_reference.md#uint32field)
          * [UInt64Field](class_reference.md#uint64field)
          * [UInt8Field](class_reference.md#uint8field)
+         * [UUIDField](class_reference.md#uuidfield)
       * [infi.clickhouse_orm.engines](class_reference.md#infi.clickhouse_orm.engines)
          * [Engine](class_reference.md#engine)
          * [TinyLog](class_reference.md#tinylog)
diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py
index 28e4c31..e864e5f 100644
--- a/src/infi/clickhouse_orm/fields.py
+++ b/src/infi/clickhouse_orm/fields.py
@@ -454,6 +454,7 @@ class ArrayField(Field):
 
 
 class UUIDField(Field):
+
     class_default = UUID(int=0)
     db_type = 'UUID'
 
diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py
index 6fc2278..bea3d3c 100644
--- a/tests/test_uuid_fields.py
+++ b/tests/test_uuid_fields.py
@@ -31,6 +31,7 @@ class UUIDFieldsTest(unittest.TestCase):
             '\x12\x34\x56\x78'*4,
             (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
             0x12345678123456781234567812345678,
+            UUID(int=0x12345678123456781234567812345678),
         ]
         for index, value in enumerate(values):
             rec = TestModel(i=index, f=value)

From 9dd1a8f4090e6e090c702f16c26a598d18311c25 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Thu, 13 Jun 2019 08:12:56 +0300
Subject: [PATCH 27/30] - Add `log_statements` parameter to database
 initializer - Fix test_merge which fails on ClickHouse v19.8.3

---
 CHANGELOG.md                        |  2 ++
 src/infi/clickhouse_orm/database.py |  6 +++++-
 src/infi/clickhouse_orm/models.py   | 15 +++++++++++----
 tests/base_test_with_data.py        |  4 ++--
 tests/test_alias_fields.py          |  2 +-
 tests/test_array_fields.py          |  2 +-
 tests/test_custom_fields.py         |  2 +-
 tests/test_datetime_fields.py       |  2 +-
 tests/test_decimal_fields.py        |  2 +-
 tests/test_engines.py               |  6 +++---
 tests/test_enum_fields.py           |  2 +-
 tests/test_fixed_string_fields.py   |  2 +-
 tests/test_join.py                  |  2 +-
 tests/test_materialized_fields.py   |  2 +-
 tests/test_migrations.py            |  2 +-
 tests/test_nullable_fields.py       |  2 +-
 tests/test_system_models.py         |  4 ++--
 tests/test_uuid_fields.py           |  2 +-
 18 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8f82621..96e3724 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,8 @@ Unreleased
 - Use HTTP Basic Authentication instead of passing the credentials in the URL
 - Support default/alias/materialized for nullable fields
 - Add UUIDField (kpotehin)
+- Add `log_statements` parameter to database initializer
+- Fix test_merge which fails on ClickHouse v19.8.3
 
 v1.0.4
 ------
diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py
index b7379da..d9eac52 100644
--- a/src/infi/clickhouse_orm/database.py
+++ b/src/infi/clickhouse_orm/database.py
@@ -83,7 +83,7 @@ class Database(object):
 
     def __init__(self, db_name, db_url='http://localhost:8123/',
                  username=None, password=None, readonly=False, autocreate=True,
-                 timeout=60, verify_ssl_cert=True):
+                 timeout=60, verify_ssl_cert=True, log_statements=False):
         '''
         Initializes a database instance. Unless it's readonly, the database will be
         created on the ClickHouse server if it does not already exist.
@@ -96,6 +96,7 @@ class Database(object):
         - `autocreate`: automatically create the database if it does not exist (unless in readonly mode).
         - `timeout`: the connection timeout in seconds.
         - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS.
+        - `log_statements`: when True, all database statements are logged.
         '''
         self.db_name = db_name
         self.db_url = db_url
@@ -105,6 +106,7 @@ class Database(object):
         self.request_session.verify = verify_ssl_cert
         if username:
             self.request_session.auth = (username, password or '')
+        self.log_statements = log_statements
         self.settings = {}
         self.db_exists = False # this is required before running _is_existing_database
         self.db_exists = self._is_existing_database()
@@ -334,6 +336,8 @@ class Database(object):
     def _send(self, data, settings=None, stream=False):
         if isinstance(data, string_types):
             data = data.encode('utf-8')
+            if self.log_statements:
+                logger.info(data)
         params = self._build_params(settings)
         r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout)
         if r.status_code != 200:
diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py
index d008513..beae53f 100644
--- a/src/infi/clickhouse_orm/models.py
+++ b/src/infi/clickhouse_orm/models.py
@@ -311,9 +311,16 @@ class MergeModel(Model):
 
     @classmethod
     def create_table_sql(cls, db):
-        assert isinstance(cls.engine, Merge), "engine must be engines.Merge instance"
-        return super(MergeModel, cls).create_table_sql(db)
-
+        assert isinstance(cls.engine, Merge), "engine must be an instance of engines.Merge"
+        parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())]
+        cols = []
+        for name, field in iteritems(cls.fields()):
+            if name != '_table':
+                cols.append('    %s %s' % (name, field.get_sql()))
+        parts.append(',\n'.join(cols))
+        parts.append(')')
+        parts.append('ENGINE = ' + cls.engine.create_table_sql(db))
+        return '\n'.join(parts)
 
 # TODO: base class for models that require specific engine
 
@@ -324,7 +331,7 @@ class DistributedModel(Model):
     """
 
     def set_database(self, db):
-        assert isinstance(self.engine, Distributed), "engine must be engines.Distributed instance"
+        assert isinstance(self.engine, Distributed), "engine must be an instance of engines.Distributed"
         res = super(DistributedModel, self).set_database(db)
         return res
 
diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py
index c3fc376..f080f85 100644
--- a/tests/base_test_with_data.py
+++ b/tests/base_test_with_data.py
@@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class TestCaseWithData(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(Person)
 
     def tearDown(self):
@@ -46,7 +46,7 @@ class Person(Model):
 data = [
     {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63",
      "passport": 35052255},
-    
+
     {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74",
      "passport": 36052255},
 
diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py
index aa692c2..de33993 100644
--- a/tests/test_alias_fields.py
+++ b/tests/test_alias_fields.py
@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class MaterializedFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithAliasFields)
 
     def tearDown(self):
diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py
index 9aa352e..f63ac2f 100644
--- a/tests/test_array_fields.py
+++ b/tests/test_array_fields.py
@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class ArrayFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithArrays)
 
     def tearDown(self):
diff --git a/tests/test_custom_fields.py b/tests/test_custom_fields.py
index 76dbd15..c0b739c 100644
--- a/tests/test_custom_fields.py
+++ b/tests/test_custom_fields.py
@@ -9,7 +9,7 @@ from infi.clickhouse_orm.engines import Memory
 class CustomFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()
diff --git a/tests/test_datetime_fields.py b/tests/test_datetime_fields.py
index 1c11849..abb8c47 100644
--- a/tests/test_datetime_fields.py
+++ b/tests/test_datetime_fields.py
@@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import *
 class DateFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithDate)
 
     def tearDown(self):
diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py
index db87d62..3a862c1 100644
--- a/tests/test_decimal_fields.py
+++ b/tests/test_decimal_fields.py
@@ -12,7 +12,7 @@ from infi.clickhouse_orm.engines import *
 class DecimalFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.add_setting('allow_experimental_decimal_type', 1)
         try:
             self.database.create_table(DecimalModel)
diff --git a/tests/test_engines.py b/tests/test_engines.py
index 846e825..a05d359 100644
--- a/tests/test_engines.py
+++ b/tests/test_engines.py
@@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class _EnginesHelperTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()
@@ -115,8 +115,8 @@ class EnginesTestCase(_EnginesHelperTestCase):
             TestModel2(date='2017-01-02', event_id=2, event_group=2, event_count=2, event_version=2)
         ])
         # event_uversion is materialized field. So * won't select it and it will be zero
-        res = self.database.select('SELECT *, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel)
-        res = [row for row in res]
+        res = self.database.select('SELECT *, _table, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel)
+        res = list(res)
         self.assertEqual(2, len(res))
         self.assertDictEqual({
             '_table': 'testmodel1',
diff --git a/tests/test_enum_fields.py b/tests/test_enum_fields.py
index 239a9b6..c6b7b1f 100644
--- a/tests/test_enum_fields.py
+++ b/tests/test_enum_fields.py
@@ -15,7 +15,7 @@ except NameError:
 class EnumFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithEnum)
         self.database.create_table(ModelWithEnumArray)
 
diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py
index 4ed663e..e9e0124 100644
--- a/tests/test_fixed_string_fields.py
+++ b/tests/test_fixed_string_fields.py
@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class FixedStringFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(FixedStringModel)
 
     def tearDown(self):
diff --git a/tests/test_join.py b/tests/test_join.py
index 49c0ac6..e3d5d12 100644
--- a/tests/test_join.py
+++ b/tests/test_join.py
@@ -9,7 +9,7 @@ from infi.clickhouse_orm import database, engines, fields, models
 class JoinTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = database.Database('test-db')
+        self.database = database.Database('test-db', log_statements=True)
         self.database.create_table(Foo)
         self.database.create_table(Bar)
         self.database.insert([Foo(id=i) for i in range(3)])
diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py
index 54d2111..3bfadcd 100644
--- a/tests/test_materialized_fields.py
+++ b/tests/test_materialized_fields.py
@@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class MaterializedFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithMaterializedFields)
 
     def tearDown(self):
diff --git a/tests/test_migrations.py b/tests/test_migrations.py
index c421ac7..d84450f 100644
--- a/tests/test_migrations.py
+++ b/tests/test_migrations.py
@@ -24,7 +24,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class MigrationsTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.drop_table(MigrationHistory)
 
     def tearDown(self):
diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py
index 74bc66e..b395a9b 100644
--- a/tests/test_nullable_fields.py
+++ b/tests/test_nullable_fields.py
@@ -14,7 +14,7 @@ from datetime import date, datetime
 class NullableFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(ModelWithNullable)
 
     def tearDown(self):
diff --git a/tests/test_system_models.py b/tests/test_system_models.py
index b9576ac..01229c8 100644
--- a/tests/test_system_models.py
+++ b/tests/test_system_models.py
@@ -14,7 +14,7 @@ from infi.clickhouse_orm.system_models import SystemPart
 
 class SystemTest(unittest.TestCase):
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()
@@ -38,7 +38,7 @@ class SystemPartTest(unittest.TestCase):
     BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/']
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
         self.database.create_table(TestTable)
         self.database.create_table(CustomPartitionedTable)
         self.database.insert([TestTable(date_field=date.today())])
diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py
index bea3d3c..247b757 100644
--- a/tests/test_uuid_fields.py
+++ b/tests/test_uuid_fields.py
@@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import Memory
 class UUIDFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
 
     def tearDown(self):
         self.database.drop_database()

From da87a151df660c373e812d3ac82e37547494a71b Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Thu, 13 Jun 2019 08:38:06 +0300
Subject: [PATCH 28/30] Fix querysets using the SystemPart model

---
 CHANGELOG.md                             |  1 +
 src/infi/clickhouse_orm/database.py      |  5 ++++-
 src/infi/clickhouse_orm/query.py         |  7 +++++--
 src/infi/clickhouse_orm/system_models.py |  4 ++--
 tests/test_system_models.py              | 14 +++++++++++++-
 5 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96e3724..0063bc5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ Unreleased
 - Add UUIDField (kpotehin)
 - Add `log_statements` parameter to database initializer
 - Fix test_merge which fails on ClickHouse v19.8.3
+- Fix querysets using the SystemPart model
 
 v1.0.4
 ------
diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py
index d9eac52..8aaae47 100644
--- a/src/infi/clickhouse_orm/database.py
+++ b/src/infi/clickhouse_orm/database.py
@@ -361,7 +361,10 @@ class Database(object):
         if '$' in query:
             mapping = dict(db="`%s`" % self.db_name)
             if model_class:
-                mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
+                if model_class.is_system_model():
+                    mapping['table'] = model_class.table_name()
+                else:
+                    mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
             query = Template(query).safe_substitute(mapping)
         return query
 
diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py
index ab7a705..16de5ba 100644
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@@ -344,9 +344,12 @@ class QuerySet(object):
         """
         distinct = 'DISTINCT ' if self._distinct else ''
         final = ' FINAL' if self._final else ''
+        table_name = self._model_cls.table_name()
+        if not self._model_cls.is_system_model():
+            table_name = '`%s`' % table_name
 
-        params = (distinct, self.select_fields_as_sql(), self._model_cls.table_name(), final)
-        sql = u'SELECT %s%s\nFROM `%s`%s' % params
+        params = (distinct, self.select_fields_as_sql(), table_name, final)
+        sql = u'SELECT %s%s\nFROM %s%s' % params
 
         if self._prewhere_q and not self._prewhere_q.is_empty:
             sql += '\nPREWHERE ' + self.conditions_as_sql(prewhere=True)
diff --git a/src/infi/clickhouse_orm/system_models.py b/src/infi/clickhouse_orm/system_models.py
index 7341d14..d51ec3b 100644
--- a/src/infi/clickhouse_orm/system_models.py
+++ b/src/infi/clickhouse_orm/system_models.py
@@ -19,8 +19,8 @@ class SystemPart(Model):
     """
     OPERATIONS = frozenset({'DETACH', 'DROP', 'ATTACH', 'FREEZE', 'FETCH'})
 
-    readonly = True
-    system = True
+    _readonly = True
+    _system = True
 
     database = StringField()  # Name of the database where the table that this part belongs to is located.
     table = StringField()  # Name of the table that this part belongs to.
diff --git a/tests/test_system_models.py b/tests/test_system_models.py
index 01229c8..1d8b8cc 100644
--- a/tests/test_system_models.py
+++ b/tests/test_system_models.py
@@ -13,6 +13,7 @@ from infi.clickhouse_orm.system_models import SystemPart
 
 
 class SystemTest(unittest.TestCase):
+
     def setUp(self):
         self.database = Database('test-db', log_statements=True)
 
@@ -54,6 +55,12 @@ class SystemPartTest(unittest.TestCase):
                 return dirnames
         raise unittest.SkipTest('Cannot find backups dir')
 
+    def test_is_read_only(self):
+        self.assertTrue(SystemPart.is_read_only())
+
+    def test_is_system_model(self):
+        self.assertTrue(SystemPart.is_system_model())
+
     def test_get_all(self):
         parts = SystemPart.get(self.database)
         self.assertEqual(len(list(parts)), 2)
@@ -62,7 +69,8 @@ class SystemPartTest(unittest.TestCase):
         parts = list(SystemPart.get_active(self.database))
         self.assertEqual(len(parts), 2)
         parts[0].detach()
-        self.assertEqual(len(list(SystemPart.get_active(self.database))), 1)
+        parts = list(SystemPart.get_active(self.database))
+        self.assertEqual(len(parts), 1)
 
     def test_get_conditions(self):
         parts = list(SystemPart.get(self.database, conditions="table='testtable'"))
@@ -101,6 +109,10 @@ class SystemPartTest(unittest.TestCase):
         # TODO Not tested, as I have no replication set
         pass
 
+    def test_query(self):
+        SystemPart.objects_in(self.database).count()
+        list(SystemPart.objects_in(self.database).filter(table='testtable'))
+
 
 class TestTable(Model):
     date_field = DateField()

From 840b29499a0b83acea77ee23461ab64f294b2fb5 Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Thu, 13 Jun 2019 08:58:39 +0300
Subject: [PATCH 29/30] UUIDField compatibility with Python 3

---
 src/infi/clickhouse_orm/fields.py | 4 +++-
 tests/test_uuid_fields.py         | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py
index e864e5f..25ae554 100644
--- a/src/infi/clickhouse_orm/fields.py
+++ b/src/infi/clickhouse_orm/fields.py
@@ -461,8 +461,10 @@ class UUIDField(Field):
     def to_python(self, value, timezone_in_use):
         if isinstance(value, UUID):
             return value
+        elif isinstance(value, binary_type):
+            return UUID(bytes=value)
         elif isinstance(value, string_types):
-            return UUID(bytes=value) if len(value) == 16 else UUID(value)
+            return UUID(value)
         elif isinstance(value, integer_types):
             return UUID(int=value)
         elif isinstance(value, tuple):
diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py
index 247b757..37bee2c 100644
--- a/tests/test_uuid_fields.py
+++ b/tests/test_uuid_fields.py
@@ -28,7 +28,7 @@ class UUIDFieldsTest(unittest.TestCase):
             '{12345678-1234-5678-1234-567812345678}',
             '12345678123456781234567812345678',
             'urn:uuid:12345678-1234-5678-1234-567812345678',
-            '\x12\x34\x56\x78'*4,
+            b'\x12\x34\x56\x78'*4,
             (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
             0x12345678123456781234567812345678,
             UUID(int=0x12345678123456781234567812345678),

From 74128e0f261e917cdb0f212212aefc452d84f47a Mon Sep 17 00:00:00 2001
From: Itai Shirav <itais@infinidat.com>
Date: Thu, 13 Jun 2019 17:20:35 +0300
Subject: [PATCH 30/30] Releasing v1.1.0

---
 CHANGELOG.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0063bc5..b4b646f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,8 @@
 Change Log
 ==========
 
-Unreleased
-----------
+v1.1.0
+------
 - Add PREWHERE support to querysets (M1hacka)
 - Add WITH TOTALS support to querysets (M1hacka)
 - Extend date field range (trthhrtz)