Merge branch 'develop' into funcs

# Conflicts: # src/infi/clickhouse_orm/query.py # tests/test_decimal_fields.py # tests/test_querysets.py
2024-11-22 00:56:34 +03:00 · 2019-07-13 11:51:10 +03:00 · 2019-07-13 11:51:10 +03:00 · 342f06e7b0
commit 342f06e7b0
parent 88510aaa42 4749918014
38 changed files with 1926 additions and 324 deletions
--- a/.noseids
+++ b/.noseids
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,11 +1,31 @@
 Change Log
 ==========
-Unreleased
+v1.2.0
----------
+------
 - Add support for per-field compression codecs (rbelio, Chocorean)
 - Add support for low cardinality fields (rbelio)
 v1.1.0
 ------
 - Add PREWHERE support to querysets (M1hacka)
 - Add WITH TOTALS support to querysets (M1hacka)
 - Extend date field range (trthhrtz)
 - Fix parsing of server errors in ClickHouse v19.3.3+
 - Fix pagination when asking for the last page on a query that matches no records
 - Use HTTP Basic Authentication instead of passing the credentials in the URL
 - Support default/alias/materialized for nullable fields
 - Add UUIDField (kpotehin)
 - Add `log_statements` parameter to database initializer
 - Fix test_merge which fails on ClickHouse v19.8.3
 - Fix querysets using the SystemPart model
 v1.0.4
 ------
 - Added `timeout` parameter to database initializer (SUHAR1K)
 - Added `verify_ssl_cert` parameter to database initializer
 - Added `final()` method to querysets (M1hacka)
 - Fixed a migrations problem - cannot add a new materialized field after a regular field
 v1.0.3
 ------
--- a/buildout.cfg
+++ b/buildout.cfg
@ -49,7 +49,7 @@ eggs = ${project:name}
 	ipython<6
 	nose
 	coverage
-	enum34
+	enum-compat
 	infi.unittest
 	infi.traceback
 	memory_profiler
--- a/docs/class_reference.md
+++ b/docs/class_reference.md
@ -10,7 +10,7 @@ infi.clickhouse_orm.database
 Database instances connect to a specific ClickHouse database for running queries,
 inserting data and other operations.
-#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False, autocreate=True, timeout=60, verify_ssl_cert=True)
+#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False, autocreate=True, timeout=60, verify_ssl_cert=True, log_statements=False)
 Initializes a database instance. Unless it's readonly, the database will be
@ -24,6 +24,7 @@ created on the ClickHouse server if it does not already exist.
 - `autocreate`: automatically create the database if it does not exist (unless in readonly mode).
 - `timeout`: the connection timeout in seconds.
 - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS.
 - `log_statements`: when True, all database statements are logged.
 #### add_setting(name, value)
@ -510,7 +511,7 @@ infi.clickhouse_orm.fields
 Extends Field
-#### ArrayField(inner_field, default=None, alias=None, materialized=None, readonly=None)
+#### ArrayField(inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### BaseEnumField
@ -520,7 +521,7 @@ Extends Field
 Abstract base class for all enum-type fields.
-#### BaseEnumField(enum_cls, default=None, alias=None, materialized=None, readonly=None)
+#### BaseEnumField(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### BaseFloatField
@ -530,7 +531,7 @@ Extends Field
 Abstract base class for all float-type fields.
-#### BaseFloatField(default=None, alias=None, materialized=None, readonly=None)
+#### BaseFloatField(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### BaseIntField
@ -540,21 +541,21 @@ Extends Field
 Abstract base class for all integer-type fields.
-#### BaseIntField(default=None, alias=None, materialized=None, readonly=None)
+#### BaseIntField(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### DateField
 Extends Field
-#### DateField(default=None, alias=None, materialized=None, readonly=None)
+#### DateField(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### DateTimeField
 Extends Field
-#### DateTimeField(default=None, alias=None, materialized=None, readonly=None)
+#### DateTimeField(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Decimal128Field
@ -592,14 +593,14 @@ Base class for all decimal fields. Can also be used directly.
 Extends BaseEnumField
-#### Enum16Field(enum_cls, default=None, alias=None, materialized=None, readonly=None)
+#### Enum16Field(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Enum8Field
 Extends BaseEnumField
-#### Enum8Field(enum_cls, default=None, alias=None, materialized=None, readonly=None)
+#### Enum8Field(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Field
@ -607,7 +608,7 @@ Extends BaseEnumField
 Abstract base class for all field types.
-#### Field(default=None, alias=None, materialized=None, readonly=None)
+#### Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### FixedStringField
@ -621,84 +622,98 @@ Extends StringField
 Extends BaseFloatField
-#### Float32Field(default=None, alias=None, materialized=None, readonly=None)
+#### Float32Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Float64Field
 Extends BaseFloatField
-#### Float64Field(default=None, alias=None, materialized=None, readonly=None)
+#### Float64Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Int16Field
 Extends BaseIntField
-#### Int16Field(default=None, alias=None, materialized=None, readonly=None)
+#### Int16Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Int32Field
 Extends BaseIntField
-#### Int32Field(default=None, alias=None, materialized=None, readonly=None)
+#### Int32Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Int64Field
 Extends BaseIntField
-#### Int64Field(default=None, alias=None, materialized=None, readonly=None)
+#### Int64Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### Int8Field
 Extends BaseIntField
-#### Int8Field(default=None, alias=None, materialized=None, readonly=None)
+#### Int8Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### LowCardinalityField
 Extends Field
 #### LowCardinalityField(inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### NullableField
 Extends Field
-#### NullableField(inner_field, default=None, alias=None, materialized=None, extra_null_values=None)
+#### NullableField(inner_field, default=None, alias=None, materialized=None, extra_null_values=None, codec=None)
 ### StringField
 Extends Field
-#### StringField(default=None, alias=None, materialized=None, readonly=None)
+#### StringField(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### UInt16Field
 Extends BaseIntField
-#### UInt16Field(default=None, alias=None, materialized=None, readonly=None)
+#### UInt16Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### UInt32Field
 Extends BaseIntField
-#### UInt32Field(default=None, alias=None, materialized=None, readonly=None)
+#### UInt32Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### UInt64Field
 Extends BaseIntField
-#### UInt64Field(default=None, alias=None, materialized=None, readonly=None)
+#### UInt64Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### UInt8Field
 Extends BaseIntField
-#### UInt8Field(default=None, alias=None, materialized=None, readonly=None)
+#### UInt8Field(default=None, alias=None, materialized=None, readonly=None, codec=None)
 ### UUIDField
 Extends Field
 #### UUIDField(default=None, alias=None, materialized=None, readonly=None, codec=None)
 infi.clickhouse_orm.engines
@ -835,10 +850,10 @@ is equivalent to:
 Returns the whole query as a SQL string.
-#### conditions_as_sql()
+#### conditions_as_sql(prewhere=False)
-Returns the contents of the query's `WHERE` clause as a string.
+Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
 #### count()
@ -854,17 +869,18 @@ Adds a DISTINCT clause to the query, meaning that any duplicate rows
 in the results will be omitted.
-#### exclude(**filter_fields)
+#### exclude(*q, **kwargs)
 Returns a copy of this queryset that excludes all rows matching the conditions.
 Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
-#### filter(*q, **filter_fields)
+#### filter(*q, **kwargs)
 Returns a copy of this queryset that includes only rows matching the conditions.
-Add q object to query if it specified.
+Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
 #### final()
@ -908,6 +924,12 @@ The result is a namedtuple containing `objects` (list), `number_of_objects`,
 `pages_total`, `number` (of the current page), and `page_size`.
 #### select_fields_as_sql()
 Returns the selected fields or expressions as a SQL string.
 ### AggregateQuerySet
 Extends QuerySet
@ -943,10 +965,10 @@ This method is not supported on `AggregateQuerySet`.
 Returns the whole query as a SQL string.
-#### conditions_as_sql()
+#### conditions_as_sql(prewhere=False)
-Returns the contents of the query's `WHERE` clause as a string.
+Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
 #### count()
@ -962,17 +984,18 @@ Adds a DISTINCT clause to the query, meaning that any duplicate rows
 in the results will be omitted.
-#### exclude(**filter_fields)
+#### exclude(*q, **kwargs)
 Returns a copy of this queryset that excludes all rows matching the conditions.
 Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
-#### filter(*q, **filter_fields)
+#### filter(*q, **kwargs)
 Returns a copy of this queryset that includes only rows matching the conditions.
-Add q object to query if it specified.
+Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
 #### final()
@ -1022,3 +1045,17 @@ The result is a namedtuple containing `objects` (list), `number_of_objects`,
 `pages_total`, `number` (of the current page), and `page_size`.
 #### select_fields_as_sql()
 Returns the selected fields or expressions as a SQL string.
 #### with_totals()
 Adds WITH TOTALS modifier ot GROUP BY, making query return extra row
 with aggregate function calculated across all the rows. More information:
 https://clickhouse.yandex/docs/en/query_language/select/#with-totals-modifier
--- a/docs/field_types.md
+++ b/docs/field_types.md
@ -9,7 +9,7 @@ Currently the following field types are supported:
 | ------------------ | ---------- | ------------------- | -----------------------------------------------------
 | StringField        | String     | unicode             | Encoded as UTF-8 when written to ClickHouse
 | FixedStringField   | String     | unicode             | Encoded as UTF-8 when written to ClickHouse
-| DateField          | Date       | datetime.date       | Range 1970-01-01 to 2038-01-19
+| DateField          | Date       | datetime.date       | Range 1970-01-01 to 2105-12-31
 | DateTimeField      | DateTime   | datetime.datetime   | Minimal value is 1970-01-01 00:00:00; Always in UTC
 | Int8Field          | Int8       | int                 | Range -128 to 127
 | Int16Field         | Int16      | int                 | Range -32768 to 32767
@ -25,6 +25,7 @@ Currently the following field types are supported:
 | Decimal32Field     | Decimal32  | Decimal             | Ditto
 | Decimal64Field     | Decimal64  | Decimal             | Ditto
 | Decimal128Field    | Decimal128 | Decimal             | Ditto
 | UUIDField          | UUID       | Decimal             |
 | Enum8Field         | Enum8      | Enum                | See below
 | Enum16Field        | Enum16     | Enum                | See below
 | ArrayField         | Array      | list                | See below
@ -120,8 +121,7 @@ db.select('SELECT * FROM $db.event', model_class=Event)
 Working with nullable fields
 ----------------------------
-From [some time](https://github.com/yandex/ClickHouse/pull/70) ClickHouse provides a NULL value support.
+[ClickHouse provides a NULL value support](https://clickhouse.yandex/docs/en/data_types/nullable).
 Also see some information [here](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00395_nullable.sql).
 Wrapping another field in a `NullableField` makes it possible to assign `None` to that field. For example:
@ -147,6 +147,79 @@ to `None`.
 NOTE: `ArrayField` of `NullableField` is not supported. Also `EnumField` cannot be nullable.
 NOTE: Using `Nullable` almost always negatively affects performance, keep this in mind when designing your databases.
 Working with field compression codecs
 -------------------------------------
 Besides default data compression, defined in server settings, per-field specification is also available.
 Supported compression algorithms:
 | Codec                | Argument                                   | Comment
 | -------------------- | -------------------------------------------| ----------------------------------------------------
 | NONE                 | None                                       | No compression.
 | LZ4                  | None                                       | LZ4 compression.
 | LZ4HC(`level`)       | Possible `level` range: [3, 12].           | Default value: 9. Greater values stands for better compression and higher CPU usage. Recommended value range: [4,9].
 | ZSTD(`level`)        | Possible `level`range: [1, 22].            | Default value: 1. Greater values stands for better compression and higher CPU usage. Levels >= 20, should be used with caution, as they require more memory.
 | Delta(`delta_bytes`) | Possible `delta_bytes` range: 1, 2, 4 , 8. | Default value for `delta_bytes` is `sizeof(type)` if it is equal to 1, 2,4 or 8 and equals to 1 otherwise.
 Codecs can be combined in a pipeline. Default table codec is not included into pipeline (if it should be applied to a field, you have to specify it explicitly in pipeline).
 Recommended usage for codecs:
 - Usually, values for particular metric, stored in path does not differ significantly from point to point. Using delta-encoding allows to reduce disk space usage significantly.
 - DateTime works great with pipeline of Delta, ZSTD and the column size can be compressed to 2-3% of its original size (given a smooth datetime data)
 - Numeric types usually enjoy best compression rates with ZSTD
 - String types enjoy good compression rates with LZ4HC
 Usage:
 ```python
 class Stats(models.Model):
    id                  = fields.UInt64Field(codec='ZSTD(10)')
    timestamp           = fields.DateTimeField(codec='Delta,ZSTD')
    timestamp_date      = fields.DateField(codec='Delta(4),ZSTD(22)')
    metadata_id         = fields.Int64Field(codec='LZ4')
    status              = fields.StringField(codec='LZ4HC(10)')
    calculation         = fields.NullableField(fields.Float32Field(), codec='ZSTD')
    alerts              = fields.ArrayField(fields.FixedStringField(length=15), codec='Delta(2),LZ4HC')
    engine = MergeTree('timestamp_date', ('id', 'timestamp'))
 ```
 Note: This feature is supported on ClickHouse version 19.1.16 and above. Codec arguments will be ignored by the ORM for older versions of ClickHouse.
 Working with LowCardinality fields
 ----------------------------------
 Starting with version 19.0 ClickHouse offers a new type of field to improve the performance of queries
 and compaction of columns for low entropy data.
 [More specifically](https://github.com/yandex/ClickHouse/issues/4074) LowCardinality data type builds dictionaries automatically. It can use multiple different dictionaries if necessarily.
 If the number of distinct values is pretty large, the dictionaries become local, several different dictionaries will be used for different ranges of data. For example, if you have too many distinct values in total, but only less than about a million values each day - then the queries by day will be processed efficiently, and queries for larger ranges will be processed rather efficiently.
 LowCardinality works independently of (generic) fields compression.
 LowCardinality fields are subsequently compressed as usual.
 The compression ratios of LowCardinality fields for text data may be significantly better than without LowCardinality.
 LowCardinality will give performance boost, in the form of processing speed, if the number of distinct values is less than a few millions. This is because data is processed in dictionary encoded form.
 You can find further information about LowCardinality in [this presentation](https://github.com/yandex/clickhouse-presentations/blob/master/meetup19/string_optimization.pdf).
 Usage example:
 ```python
 class LowCardinalityModel(models.Model):
    date       = fields.DateField()
    int32      = fields.LowCardinalityField(fields.Int32Field())
    float32    = fields.LowCardinalityField(fields.Float32Field())
    string     = fields.LowCardinalityField(fields.StringField())
    nullable   = fields.LowCardinalityField(fields.NullableField(fields.StringField()))
    array      = fields.ArrayField(fields.LowCardinalityField(fields.UInt64Field()))
    engine = MergeTree('date', ('date',))
 ```
 Note: `LowCardinality` field with an inner array field is not supported. Use an `ArrayField` with a `LowCardinality` inner field as seen in the example.
 Creating custom field types
 ---------------------------
 Sometimes it is convenient to use data types that are supported in Python, but have no corresponding column type in ClickHouse. In these cases it is possible to define a custom field class that knows how to convert the Pythonic object to a suitable representation in the database, and vice versa.
@ -183,46 +256,6 @@ class BooleanField(Field):
        return '1' if value else '0'
 ```
 Here's another example - a field for storing UUIDs in the database as 16-byte strings. We'll use Python's built-in `UUID` class to handle the conversion from strings, ints and tuples into UUID instances. So in our Python code we'll have the convenience of working with UUID objects, but they will be stored in the database as efficiently as possible:
 ```python
 from infi.clickhouse_orm.fields import Field
 from infi.clickhouse_orm.utils import escape
 from uuid import UUID
 import six
 class UUIDField(Field):
    # The ClickHouse column type to use
    db_type = 'FixedString(16)'
    # The default value if empty
    class_default = UUID(int=0)
    def to_python(self, value, timezone_in_use):
        # Convert valid values to UUID instance
        if isinstance(value, UUID):
            return value
        elif isinstance(value, six.string_types):
            return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
        elif isinstance(value, six.integer_types):
            return UUID(int=value)
        elif isinstance(value, tuple):
            return UUID(fields=value)
        else:
            raise ValueError('Invalid value for UUIDField: %r' % value)
    def to_db_string(self, value, quote=True):
        # The value was already converted by to_python, so it's a UUID instance
        val = value.bytes
        if six.PY3:
            val = str(val, 'latin1')
        return escape(val, quote)
 ```
 Note that the latin-1 encoding is used as an identity encoding for converting between raw bytes and strings. This is required in Python 3, where `str` and `bytes` are different types.
 ---
 [<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md)
--- a/docs/models_and_databases.md
+++ b/docs/models_and_databases.md
@ -89,7 +89,7 @@ When values are assigned to model fields, they are immediately converted to thei
    >>> suzy.birthday = 0.5
    ValueError: Invalid value for DateField - 0.5
    >>> suzy.birthday = '1922-05-31'
-    ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2038-01-19
+    ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2105-12-31
 Inserting to the Database
 -------------------------
--- a/docs/querysets.md
+++ b/docs/querysets.md
@ -32,6 +32,14 @@ For filters with compound conditions you can use `Q` objects inside `filter` wit
    >>> qs.conditions_as_sql()
    u"((first_name = 'Ciaran' AND last_name = 'Carver') OR height <= 1.8) AND (NOT (first_name = 'David'))"
 By default conditions from `filter` and `exclude` methods are add to `WHERE` clause.
 For better aggregation performance you can add them to `PREWHERE` section using `prewhere=True` parameter
    >>> qs = Person.objects_in(database)
    >>> qs = qs.filter(first_name__startswith='V', prewhere=True)
    >>> qs.conditions_as_sql(prewhere=True)
    u"first_name LIKE 'V%'"
 There are different operators that can be used, by passing `<fieldname>__<operator>=<value>` (two underscores separate the field name from the operator). In case no operator is given, `eq` is used by default. Below are all the supported operators.
 | Operator       | Equivalent SQL                               | Comments                           |
@ -202,6 +210,19 @@ This queryset is translated to:
 After calling `aggregate` you can still use most of the regular queryset methods, such as `count`, `order_by` and `paginate`. It is not possible, however, to call `only` or `aggregate`. It is also not possible to filter the queryset on calculated fields, only on fields that exist in the model.
 If you limit aggregation results, it might be useful to get total aggregation values for all rows.
 To achieve this, you can use `with_totals` method. It will return extra row (last) with
 values aggregated for all rows suitable for filters.
    qs = Person.objects_in(database).aggregate('first_name', num='count()').with_totals().order_by('-count')[:3]
    >>> print qs.count()
    4
    >>> for row in qs:
    >>>     print("'{}': {}".format(row.first_name, row.count))
    'Cassandra': 2
    'Alexandra': 2
    '': 100
 ---
 [<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md)
--- a/docs/ref.md
+++ b/docs/ref.md
@ -482,9 +482,9 @@ infi.clickhouse_orm.query
 #### QuerySet(model_cls, database)
-#### conditions_as_sql()
+#### conditions_as_sql(prewhere=True)
-Return the contents of the queryset's WHERE clause.
+Return the contents of the queryset's WHERE or `PREWHERE` clause.
 #### count()
--- a/docs/toc.md
+++ b/docs/toc.md
@ -36,6 +36,8 @@
      * [Working with array fields](field_types.md#working-with-array-fields)
      * [Working with materialized and alias fields](field_types.md#working-with-materialized-and-alias-fields)
      * [Working with nullable fields](field_types.md#working-with-nullable-fields)
      * [Working with field compression codecs](field_types.md#working-with-field-compression-codecs)
      * [Working with LowCardinality fields](field_types.md#working-with-lowcardinality-fields)
      * [Creating custom field types](field_types.md#creating-custom-field-types)
   * [Table Engines](table_engines.md#table-engines)
@ -86,12 +88,14 @@
         * [Int32Field](class_reference.md#int32field)
         * [Int64Field](class_reference.md#int64field)
         * [Int8Field](class_reference.md#int8field)
         * [LowCardinalityField](class_reference.md#lowcardinalityfield)
         * [NullableField](class_reference.md#nullablefield)
         * [StringField](class_reference.md#stringfield)
         * [UInt16Field](class_reference.md#uint16field)
         * [UInt32Field](class_reference.md#uint32field)
         * [UInt64Field](class_reference.md#uint64field)
         * [UInt8Field](class_reference.md#uint8field)
         * [UUIDField](class_reference.md#uuidfield)
      * [infi.clickhouse_orm.engines](class_reference.md#infi.clickhouse_orm.engines)
         * [Engine](class_reference.md#engine)
         * [TinyLog](class_reference.md#tinylog)
--- a/src/infi/clickhouse_orm/database.py
+++ b/src/infi/clickhouse_orm/database.py
@ -40,11 +40,19 @@ class ServerError(DatabaseException):
            self.message = message
            super(ServerError, self).__init__(message)
-    ERROR_PATTERN = re.compile(r'''
+    ERROR_PATTERNS = (
        # ClickHouse prior to v19.3.3
        re.compile(r'''
            Code:\ (?P<code>\d+),
            \ e\.displayText\(\)\ =\ (?P<type1>[^ \n]+):\ (?P<msg>.+?),
            \ e.what\(\)\ =\ (?P<type2>[^ \n]+)
-    ''', re.VERBOSE | re.DOTALL)
+        ''', re.VERBOSE | re.DOTALL),
        # ClickHouse v19.3.3+
        re.compile(r'''
            Code:\ (?P<code>\d+),
            \ e\.displayText\(\)\ =\ (?P<type1>[^ \n]+):\ (?P<msg>.+)
        ''', re.VERBOSE | re.DOTALL),
    )
    @classmethod
    def get_error_code_msg(cls, full_error_message):
@ -54,10 +62,11 @@ class ServerError(DatabaseException):
        See the list of error codes here:
        https://github.com/yandex/ClickHouse/blob/master/dbms/src/Common/ErrorCodes.cpp
        """
-        match = cls.ERROR_PATTERN.match(full_error_message)
+        for pattern in cls.ERROR_PATTERNS:
            match = pattern.match(full_error_message)
            if match:
                # assert match.group('type1') == match.group('type2')
-            return int(match.group('code')), match.group('msg')
+                return int(match.group('code')), match.group('msg').strip()
        return 0, full_error_message
@ -74,7 +83,7 @@ class Database(object):
    def __init__(self, db_name, db_url='http://localhost:8123/',
                 username=None, password=None, readonly=False, autocreate=True,
-                 timeout=60, verify_ssl_cert=True):
+                 timeout=60, verify_ssl_cert=True, log_statements=False):
        '''
        Initializes a database instance. Unless it's readonly, the database will be
        created on the ClickHouse server if it does not already exist.
@ -87,15 +96,17 @@ class Database(object):
        - `autocreate`: automatically create the database if it does not exist (unless in readonly mode).
        - `timeout`: the connection timeout in seconds.
        - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS.
        - `log_statements`: when True, all database statements are logged.
        '''
        self.db_name = db_name
        self.db_url = db_url
        self.username = username
        self.password = password
        self.readonly = False
        self.timeout = timeout
        self.request_session = requests.Session()
        self.request_session.verify = verify_ssl_cert
        if username:
            self.request_session.auth = (username, password or '')
        self.log_statements = log_statements
        self.settings = {}
        self.db_exists = False # this is required before running _is_existing_database
        self.db_exists = self._is_existing_database()
@ -109,6 +120,10 @@ class Database(object):
        self.server_version = self._get_server_version()
        # Versions 1.1.53981 and below don't have timezone function
        self.server_timezone = self._get_server_timezone() if self.server_version > (1, 1, 53981) else pytz.utc
        # Versions 19.1.16 and above support codec compression
        self.has_codec_support = self.server_version >= (19, 1, 16)
        # Version 19.0 and above support LowCardinality
        self.has_low_cardinality_support = self.server_version >= (19, 0)
    def create_database(self):
        '''
@ -276,7 +291,7 @@ class Database(object):
        count = self.count(model_class, conditions)
        pages_total = int(ceil(count / float(page_size)))
        if page_num == -1:
-            page_num = pages_total
+            page_num = max(pages_total, 1)
        elif page_num < 1:
            raise ValueError('Invalid page number: %d' % page_num)
        offset = (page_num - 1) * page_size
@ -287,7 +302,7 @@ class Database(object):
        query += ' LIMIT %d, %d' % (offset, page_size)
        query = self._substitute(query, model_class)
        return Page(
-            objects=list(self.select(query, model_class, settings)),
+            objects=list(self.select(query, model_class, settings)) if count else [],
            number_of_objects=count,
            pages_total=pages_total,
            number=page_num,
@ -325,6 +340,8 @@ class Database(object):
    def _send(self, data, settings=None, stream=False):
        if isinstance(data, string_types):
            data = data.encode('utf-8')
            if self.log_statements:
                logger.info(data)
        params = self._build_params(settings)
        r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout)
        if r.status_code != 200:
@ -336,10 +353,6 @@ class Database(object):
        params.update(self.settings)
        if self.db_exists:
            params['database'] = self.db_name
        if self.username:
            params['user'] = self.username
        if self.password:
            params['password'] = self.password
        # Send the readonly flag, unless the connection is already readonly (to prevent db error)
        if self.readonly and not self.connection_readonly:
            params['readonly'] = '1'
@ -352,6 +365,9 @@ class Database(object):
        if '$' in query:
            mapping = dict(db="`%s`" % self.db_name)
            if model_class:
                if model_class.is_system_model():
                    mapping['table'] = model_class.table_name()
                else:
                    mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
            query = Template(query).safe_substitute(mapping)
        return query
--- a/src/infi/clickhouse_orm/engines.py
+++ b/src/infi/clickhouse_orm/engines.py
@ -40,7 +40,7 @@ class MergeTree(Engine):
        assert date_col is None or isinstance(date_col, six.string_types), 'date_col must be string if present'
        assert partition_key is None or type(partition_key) in (list, tuple),\
            'partition_key must be tuple or list if present'
-        assert (replica_table_path is None) == (replica_name == None), \
+        assert (replica_table_path is None) == (replica_name is None), \
            'both replica_table_path and replica_name must be specified'
        # These values conflict with each other (old and new syntax of table engines.
--- a/src/infi/clickhouse_orm/fields.py
+++ b/src/infi/clickhouse_orm/fields.py
@ -1,15 +1,16 @@
 from __future__ import unicode_literals
-from six import string_types, text_type, binary_type
+from six import string_types, text_type, binary_type, integer_types
 import datetime
 import iso8601
 import pytz
 import time
 from calendar import timegm
 from decimal import Decimal, localcontext
-
+from uuid import UUID
 from logging import getLogger
 from .utils import escape, parse_array, comma_join
 from .funcs import F, FunctionOperatorsMixin
 logger = getLogger('clickhouse_orm')
 class Field(FunctionOperatorsMixin):
    '''
@ -21,14 +22,16 @@ class Field(FunctionOperatorsMixin):
    class_default    = 0    # should be overridden by concrete subclasses
    db_type          = None # should be overridden by concrete subclasses
-    def __init__(self, default=None, alias=None, materialized=None, readonly=None):
+    def __init__(self, default=None, alias=None, materialized=None, readonly=None, codec=None):
        assert (None, None) in {(default, alias), (alias, materialized), (default, materialized)}, \
            "Only one of default, alias and materialized parameters can be given"
        assert alias is None or isinstance(alias, string_types) and alias != "",\
-            "Alias field must be string field name, if given"
+            "Alias field must be a string, if given"
-        assert materialized is None or isinstance(materialized, string_types) and alias != "",\
+        assert materialized is None or isinstance(materialized, string_types) and materialized != "",\
            "Materialized field must be string, if given"
        assert readonly is None or type(readonly) is bool, "readonly parameter must be bool if given"
        assert codec is None or isinstance(codec, string_types) and codec != "", \
            "Codec field must be string, if given"
        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1
@ -36,6 +39,7 @@ class Field(FunctionOperatorsMixin):
        self.alias = alias
        self.materialized = materialized
        self.readonly = bool(self.alias or self.materialized or readonly)
        self.codec = codec
    def to_python(self, value, timezone_in_use):
        '''
@ -66,22 +70,30 @@ class Field(FunctionOperatorsMixin):
        '''
        return escape(value, quote)
-    def get_sql(self, with_default_expression=True):
+    def get_sql(self, with_default_expression=True, db=None):
        '''
        Returns an SQL expression describing the field (e.g. for CREATE TABLE).
        :param with_default_expression: If True, adds default value to sql.
            It doesn't affect fields with alias and materialized values.
        :param db: Database, used for checking supported features.
        '''
        sql = self.db_type
        if with_default_expression:
            sql += self._extra_params(db)
        return sql
    def _extra_params(self, db):
        sql = ''
        if self.alias:
-                return '%s ALIAS %s' % (self.db_type, self.alias)
+            sql += ' ALIAS %s' % self.alias
        elif self.materialized:
-                return '%s MATERIALIZED %s' % (self.db_type, self.materialized)
+            sql += ' MATERIALIZED %s' % self.materialized
-            else:
+        elif self.default:
            default = self.to_db_string(self.default)
-                return '%s DEFAULT %s' % (self.db_type, default)
+            sql += ' DEFAULT %s' % default
-        else:
+        if self.codec and db and db.has_codec_support:
-            return self.db_type
+            sql += ' CODEC(%s)' % self.codec
        return sql
    def isinstance(self, types):
        """
@ -154,7 +166,7 @@ class FixedStringField(StringField):
 class DateField(Field):
    min_value = datetime.date(1970, 1, 1)
-    max_value = datetime.date(2038, 1, 19)
+    max_value = datetime.date(2105, 12, 31)
    class_default = min_value
    db_type = 'Date'
@ -383,11 +395,11 @@ class BaseEnumField(Field):
    Abstract base class for all enum-type fields.
    '''
-    def __init__(self, enum_cls, default=None, alias=None, materialized=None, readonly=None):
+    def __init__(self, enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None):
        self.enum_cls = enum_cls
        if default is None:
            default = list(enum_cls)[0]
-        super(BaseEnumField, self).__init__(default, alias, materialized, readonly)
+        super(BaseEnumField, self).__init__(default, alias, materialized, readonly, codec)
    def to_python(self, value, timezone_in_use):
        if isinstance(value, self.enum_cls):
@ -406,12 +418,14 @@ class BaseEnumField(Field):
    def to_db_string(self, value, quote=True):
        return escape(value.name, quote)
-    def get_sql(self, with_default_expression=True):
+    def get_sql(self, with_default_expression=True, db=None):
        values = ['%s = %d' % (escape(item.name), item.value) for item in self.enum_cls]
        sql = '%s(%s)' % (self.db_type, ' ,'.join(values))
        if with_default_expression:
            default = self.to_db_string(self.default)
            sql = '%s DEFAULT %s' % (sql, default)
            if self.codec and db and db.has_codec_support:
                sql+= ' CODEC(%s)' % self.codec
        return sql
    @classmethod
@ -421,10 +435,7 @@ class BaseEnumField(Field):
        this method returns a matching enum field.
        '''
        import re
-        try:
+        from enum import Enum
            Enum # exists in Python 3.4+
        except NameError:
            from enum import Enum # use the enum34 library instead
        members = {}
        for match in re.finditer("'(\w+)' = (\d+)", db_type):
            members[match.group(1)] = int(match.group(2))
@ -447,11 +458,11 @@ class ArrayField(Field):
    class_default = []
-    def __init__(self, inner_field, default=None, alias=None, materialized=None, readonly=None):
+    def __init__(self, inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None):
        assert isinstance(inner_field, Field), "The first argument of ArrayField must be a Field instance"
        assert not isinstance(inner_field, ArrayField), "Multidimensional array fields are not supported by the ORM"
        self.inner_field = inner_field
-        super(ArrayField, self).__init__(default, alias, materialized, readonly)
+        super(ArrayField, self).__init__(default, alias, materialized, readonly, codec)
    def to_python(self, value, timezone_in_use):
        if isinstance(value, text_type):
@ -470,9 +481,34 @@ class ArrayField(Field):
        array = [self.inner_field.to_db_string(v, quote=True) for v in value]
        return '[' + comma_join(array) + ']'
-    def get_sql(self, with_default_expression=True):
+    def get_sql(self, with_default_expression=True, db=None):
-        from .utils import escape
+        sql = 'Array(%s)' % self.inner_field.get_sql(with_default_expression=False, db=db)
-        return 'Array(%s)' % self.inner_field.get_sql(with_default_expression=False)
+        if with_default_expression and self.codec and db and db.has_codec_support:
            sql+= ' CODEC(%s)' % self.codec
        return sql
 class UUIDField(Field):
    class_default = UUID(int=0)
    db_type = 'UUID'
    def to_python(self, value, timezone_in_use):
        if isinstance(value, UUID):
            return value
        elif isinstance(value, binary_type):
            return UUID(bytes=value)
        elif isinstance(value, string_types):
            return UUID(value)
        elif isinstance(value, integer_types):
            return UUID(int=value)
        elif isinstance(value, tuple):
            return UUID(fields=value)
        else:
            raise ValueError('Invalid value for UUIDField: %r' % value)
    def to_db_string(self, value, quote=True):
        return escape(str(value), quote)
 class NullableField(Field):
@ -480,12 +516,12 @@ class NullableField(Field):
    class_default = None
    def __init__(self, inner_field, default=None, alias=None, materialized=None,
-                 extra_null_values=None):
+                 extra_null_values=None, codec=None):
        self.inner_field = inner_field
        self._null_values = [None]
        if extra_null_values:
            self._null_values.extend(extra_null_values)
-        super(NullableField, self).__init__(default, alias, materialized, readonly=None)
+        super(NullableField, self).__init__(default, alias, materialized, readonly=None, codec=codec)
    def to_python(self, value, timezone_in_use):
        if value == '\\N' or value in self._null_values:
@ -500,5 +536,38 @@ class NullableField(Field):
            return '\\N'
        return self.inner_field.to_db_string(value, quote=quote)
-    def get_sql(self, with_default_expression=True):
+    def get_sql(self, with_default_expression=True, db=None):
-        return 'Nullable(%s)' % self.inner_field.get_sql(with_default_expression=False)
+        sql = 'Nullable(%s)' % self.inner_field.get_sql(with_default_expression=False, db=db)
        if with_default_expression:
            sql += self._extra_params(db)
        return sql
 class LowCardinalityField(Field):
    def __init__(self, inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None):
        assert isinstance(inner_field, Field), "The first argument of LowCardinalityField must be a Field instance. Not: {}".format(inner_field)
        assert not isinstance(inner_field, LowCardinalityField), "LowCardinality inner fields are not supported by the ORM"
        assert not isinstance(inner_field, ArrayField), "Array field inside LowCardinality are not supported by the ORM. Use Array(LowCardinality) instead"
        self.inner_field = inner_field
        self.class_default = self.inner_field.class_default
        super(LowCardinalityField, self).__init__(default, alias, materialized, readonly, codec)
    def to_python(self, value, timezone_in_use):
        return self.inner_field.to_python(value, timezone_in_use)
    def validate(self, value):
        self.inner_field.validate(value)
    def to_db_string(self, value, quote=True):
        return self.inner_field.to_db_string(value, quote=quote)
    def get_sql(self, with_default_expression=True, db=None):
        if db and db.has_low_cardinality_support:
            sql = 'LowCardinality(%s)' % self.inner_field.get_sql(with_default_expression=False)
        else:
            sql = self.inner_field.get_sql(with_default_expression=False)
            logger.warning('LowCardinalityField not supported on clickhouse-server version < 19.0 using {} as fallback'.format(self.inner_field.__class__.__name__))
        if with_default_expression:
            sql += self._extra_params(db)
        return sql
--- a/src/infi/clickhouse_orm/migrations.py
+++ b/src/infi/clickhouse_orm/migrations.py
@ -75,13 +75,16 @@ class AlterTable(Operation):
        # Identify fields that were added to the model
        prev_name = None
        for name, field in iteritems(self.model_class.fields()):
            is_regular_field = not (field.materialized or field.alias)
            if name not in table_fields:
                logger.info('        Add column %s', name)
                assert prev_name, 'Cannot add a column to the beginning of the table'
-                cmd = 'ADD COLUMN %s %s AFTER %s' % (name, field.get_sql(), prev_name)
+                cmd = 'ADD COLUMN %s %s' % (name, field.get_sql(db=database))
                if is_regular_field:
                    cmd += ' AFTER %s' % prev_name
                self._alter_table(database, cmd)
-            if not field.materialized and not field.alias:
+            if is_regular_field:
                # ALIAS and MATERIALIZED fields are not stored in the database, and raise DatabaseError
                # (no AFTER column). So we will skip them
                prev_name = name
@ -90,7 +93,7 @@ class AlterTable(Operation):
        # The order of class attributes can be changed any time, so we can't count on it
        # Secondly, MATERIALIZED and ALIAS fields are always at the end of the DESC, so we can't expect them to save
        # attribute position. Watch https://github.com/Infinidat/infi.clickhouse_orm/issues/47
-        model_fields = {name: field.get_sql(with_default_expression=False)
+        model_fields = {name: field.get_sql(with_default_expression=False, db=database)
                        for name, field in iteritems(self.model_class.fields())}
        for field_name, field_sql in self._get_table_fields(database):
            # All fields must have been created and dropped by this moment
--- a/src/infi/clickhouse_orm/models.py
+++ b/src/infi/clickhouse_orm/models.py
@ -197,7 +197,7 @@ class Model(with_metaclass(ModelBase)):
        parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())]
        cols = []
        for name, field in iteritems(cls.fields()):
-            cols.append('    %s %s' % (name, field.get_sql()))
+            cols.append('    %s %s' % (name, field.get_sql(db=db)))
        parts.append(',\n'.join(cols))
        parts.append(')')
        parts.append('ENGINE = ' + cls.engine.create_table_sql(db))
@ -318,9 +318,16 @@ class MergeModel(Model):
    @classmethod
    def create_table_sql(cls, db):
-        assert isinstance(cls.engine, Merge), "engine must be engines.Merge instance"
+        assert isinstance(cls.engine, Merge), "engine must be an instance of engines.Merge"
-        return super(MergeModel, cls).create_table_sql(db)
+        parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())]
-
+        cols = []
        for name, field in iteritems(cls.fields()):
            if name != '_table':
                cols.append('    %s %s' % (name, field.get_sql(db=db)))
        parts.append(',\n'.join(cols))
        parts.append(')')
        parts.append('ENGINE = ' + cls.engine.create_table_sql(db))
        return '\n'.join(parts)
 # TODO: base class for models that require specific engine
@ -331,7 +338,7 @@ class DistributedModel(Model):
    """
    def set_database(self, db):
-        assert isinstance(self.engine, Distributed), "engine must be engines.Distributed instance"
+        assert isinstance(self.engine, Distributed), "engine must be an instance of engines.Distributed"
        res = super(DistributedModel, self).set_database(db)
        return res
--- a/src/infi/clickhouse_orm/query.py
+++ b/src/infi/clickhouse_orm/query.py
@ -2,7 +2,7 @@ from __future__ import unicode_literals
 import six
 import pytz
-from copy import copy
+from copy import copy, deepcopy
 from math import ceil
 from .engines import CollapsingMergeTree
 from datetime import date, datetime
@ -185,6 +185,11 @@ class FieldCond(Cond):
    def to_sql(self, model_cls):
        return self._operator.to_sql(model_cls, self._field_name, self._value)
    def __deepcopy__(self, memodict={}):
        res = copy(self)
        res._value = deepcopy(self._value)
        return res
 class Q(object):
@ -193,17 +198,32 @@ class Q(object):
    def __init__(self, *filter_funcs, **filter_fields):
        self._conds = list(filter_funcs) + [self._build_cond(k, v) for k, v in six.iteritems(filter_fields)]
-        self._l_child = None
+        self._children = []
        self._r_child = None
        self._negate = False
        self._mode = self.AND_MODE
    @property
    def is_empty(self):
        """
        Checks if there are any conditions in Q object
        :return: Boolean
        """
        return not bool(self._conds or self._children)
    @classmethod
    def _construct_from(cls, l_child, r_child, mode):
        if mode == l_child._mode:
            q = deepcopy(l_child)
            q._children.append(deepcopy(r_child))
        elif mode == r_child._mode:
            q = deepcopy(r_child)
            q._children.append(deepcopy(l_child))
        else:
            # Different modes
            q = Q()
-        q._l_child = l_child
+            q._children = [l_child, r_child]
        q._r_child = r_child
            q._mode = mode  # AND/OR
        return q
    def _build_cond(self, key, value):
@ -214,16 +234,27 @@ class Q(object):
        return FieldCond(field_name, operator, value)
    def to_sql(self, model_cls):
        condition_sql = []
        if self._conds:
-            sql = self._mode.join(cond.to_sql(model_cls) for cond in self._conds)
+            condition_sql.extend([cond.to_sql(model_cls) for cond in self._conds])
        if self._children:
            condition_sql.extend([child.to_sql(model_cls) for child in self._children if child])
        if not condition_sql:
            # Empty Q() object returns everything
            sql = '1'
        elif len(condition_sql) == 1:
            # Skip not needed brackets over single condition
            sql = condition_sql[0]
        else:
-            if self._l_child and self._r_child:
+            # Each condition must be enclosed in brackets, or order of operations may be wrong
-                sql = '({} {} {})'.format(
+            sql = '(%s)' % ') {} ('.format(self._mode).join(condition_sql)
-                        self._l_child.to_sql(model_cls), self._mode, self._r_child.to_sql(model_cls))
+
            else:
                return '1'
        if self._negate:
            sql = 'NOT (%s)' % sql
        return sql
    def __or__(self, other):
@ -237,6 +268,20 @@ class Q(object):
        q._negate = True
        return q
    def __bool__(self):
        return not self.is_empty
    def __deepcopy__(self, memodict={}):
        q = Q()
        q._conds = [deepcopy(cond) for cond in self._conds]
        q._negate = self._negate
        q._mode = self._mode
        if self._children:
            q._children = [deepcopy(child) for child in self._children]
        return q
@six.python_2_unicode_compatible
 class QuerySet(object):
@ -254,7 +299,10 @@ class QuerySet(object):
        self._model_cls = model_cls
        self._database = database
        self._order_by = []
-        self._q = []
+        self._where_q = Q()
        self._prewhere_q = Q()
        self._grouping_fields = []
        self._grouping_with_totals = False
        self._fields = model_cls.fields().keys()
        self._extra = {}
        self._limits = None
@ -297,22 +345,49 @@ class QuerySet(object):
            qs._limits = (start, stop - start)
            return qs
-    def as_sql(self):
+    def select_fields_as_sql(self):
        """
-        Returns the whole query as a SQL string.
+        Returns the selected fields or expressions as a SQL string.
        """
        distinct = 'DISTINCT ' if self._distinct else ''
        fields = '*'
        if self._fields:
            fields = comma_join('`%s`' % field for field in self._fields)
        for name, func in self._extra.items():
            fields += ', %s AS %s' % (func.to_sql(), name)
-        ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
+        return fields
-        limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
+
    def as_sql(self):
        """
        Returns the whole query as a SQL string.
        """
        distinct = 'DISTINCT ' if self._distinct else ''
        final = ' FINAL' if self._final else ''
-        params = (distinct, fields, self._model_cls.table_name(), final,
+        table_name = self._model_cls.table_name()
-                  self.conditions_as_sql(), ordering, limit)
+        if not self._model_cls.is_system_model():
-        return u'SELECT %s%s\nFROM `%s`%s\nWHERE %s%s%s' % params
+            table_name = '`%s`' % table_name
        params = (distinct, self.select_fields_as_sql(), table_name, final)
        sql = u'SELECT %s%s\nFROM %s%s' % params
        if self._prewhere_q and not self._prewhere_q.is_empty:
            sql += '\nPREWHERE ' + self.conditions_as_sql(prewhere=True)
        if self._where_q and not self._where_q.is_empty:
            sql += '\nWHERE ' + self.conditions_as_sql(prewhere=False)
        if self._grouping_fields:
            sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields)
            if self._grouping_with_totals:
                sql += ' WITH TOTALS'
        if self._order_by:
            sql += '\nORDER BY ' + self.order_by_as_sql()
        if self._limits:
            sql += '\nLIMIT %d, %d' % self._limits
        return sql
    def order_by_as_sql(self):
        """
@ -323,14 +398,12 @@ class QuerySet(object):
            for field in self._order_by
        ])
-    def conditions_as_sql(self):
+    def conditions_as_sql(self, prewhere=False):
        """
-        Returns the contents of the query's `WHERE` clause as a string.
+        Returns the contents of the query's `WHERE` or `PREWHERE` clause as a string.
        """
-        if self._q:
+        q_object = self._prewhere_q if prewhere else self._where_q
-            return u' AND '.join([q.to_sql(self._model_cls) for q in self._q])
+        return q_object.to_sql(self._model_cls)
        else:
            return u'1'
    def count(self):
        """
@ -341,8 +414,10 @@ class QuerySet(object):
            sql = u'SELECT count() FROM (%s)' % self.as_sql()
            raw = self._database.raw(sql)
            return int(raw) if raw else 0
        # Simple case
-        return self._database.count(self._model_cls, self.conditions_as_sql())
+        conditions = (self._where_q & self._prewhere_q).to_sql(self._model_cls)
        return self._database.count(self._model_cls, conditions)
    def order_by(self, *field_names):
        """
@ -367,32 +442,50 @@ class QuerySet(object):
        qs._extra = kwargs
        return qs
-    def filter(self, *q, **filter_fields):
+    def _filter_or_exclude(self, *q, **kwargs):
-        """
+        from .funcs import F
-        Returns a copy of this queryset that includes only rows matching the conditions.
+
-        Add q object to query if it specified.
+        inverse = kwargs.pop('_inverse', False)
-        """
+        prewhere = kwargs.pop('prewhere', False)
-        from infi.clickhouse_orm.funcs import F
+
        qs = copy(self)
-        qs._q = list(self._q)
+
        condition = Q()
        for arg in q:
            if isinstance(arg, Q):
-                qs._q.append(arg)
+                condition &= arg
            elif isinstance(arg, F):
-                qs._q.append(Q(arg))
+                condition &= Q(arg)
            else:
                raise TypeError('Invalid argument "%r" to queryset filter' % arg)
-        if filter_fields:
+
-            qs._q += [Q(**filter_fields)]
+        if kwargs:
            condition &= Q(**kwargs)
        if inverse:
            condition = ~condition
        condition = copy(self._prewhere_q if prewhere else self._where_q) & condition
        if prewhere:
            qs._prewhere_q = condition
        else:
            qs._where_q = condition
        return qs
-    def exclude(self, **filter_fields):
+    def filter(self, *q, **kwargs):
        """
        Returns a copy of this queryset that includes only rows matching the conditions.
        Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
        """
        return self._filter_or_exclude(*q, **kwargs)
    def exclude(self, *q, **kwargs):
        """
        Returns a copy of this queryset that excludes all rows matching the conditions.
        Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE.
        """
-        qs = copy(self)
+        return self._filter_or_exclude(*q, _inverse=True, **kwargs)
        qs._q = list(self._q) + [~Q(**filter_fields)]
        return qs
    def paginate(self, page_num=1, page_size=100):
        """
@ -486,7 +579,8 @@ class AggregateQuerySet(QuerySet):
        self._grouping_fields = grouping_fields
        self._calculated_fields = calculated_fields
        self._order_by = list(base_qs._order_by)
-        self._q = list(base_qs._q)
+        self._where_q = base_qs._where_q
        self._prewhere_q = base_qs._prewhere_q
        self._limits = base_qs._limits
        self._distinct = base_qs._distinct
@ -515,26 +609,11 @@ class AggregateQuerySet(QuerySet):
        """
        raise NotImplementedError('Cannot re-aggregate an AggregateQuerySet')
-    def as_sql(self):
+    def select_fields_as_sql(self):
        """
-        Returns the whole query as a SQL string.
+        Returns the selected fields or expressions as a SQL string.
        """
-        distinct = 'DISTINCT ' if self._distinct else ''
+        return comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
        grouping = comma_join('`%s`' % field for field in self._grouping_fields)
        fields = comma_join(list(self._fields) + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()])
        params = dict(
            distinct=distinct,
            grouping=grouping or "''",
            fields=fields,
            table=self._model_cls.table_name(),
            conds=self.conditions_as_sql()
        )
        sql = u'SELECT %(distinct)s%(fields)s\nFROM `%(table)s`\nWHERE %(conds)s\nGROUP BY %(grouping)s' % params
        if self._order_by:
            sql += '\nORDER BY ' + self.order_by_as_sql()
        if self._limits:
            sql += '\nLIMIT %d, %d' % self._limits
        return sql
    def __iter__(self):
        return self._database.select(self.as_sql()) # using an ad-hoc model
@ -547,4 +626,12 @@ class AggregateQuerySet(QuerySet):
        raw = self._database.raw(sql)
        return int(raw) if raw else 0
-
+    def with_totals(self):
        """
        Adds WITH TOTALS modifier ot GROUP BY, making query return extra row
        with aggregate function calculated across all the rows. More information:
        https://clickhouse.yandex/docs/en/query_language/select/#with-totals-modifier
        """
        qs = copy(self)
        qs._grouping_with_totals = True
        return qs
--- a/src/infi/clickhouse_orm/system_models.py
+++ b/src/infi/clickhouse_orm/system_models.py
@ -19,8 +19,8 @@ class SystemPart(Model):
    """
    OPERATIONS = frozenset({'DETACH', 'DROP', 'ATTACH', 'FREEZE', 'FETCH'})
-    readonly = True
+    _readonly = True
-    system = True
+    _system = True
    database = StringField()  # Name of the database where the table that this part belongs to is located.
    table = StringField()  # Name of the table that this part belongs to.
--- a/tests/base_test_with_data.py
+++ b/tests/base_test_with_data.py
@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class TestCaseWithData(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(Person)
    def tearDown(self):
--- a/tests/sample_migrations/0015.py
+++ b/tests/sample_migrations/0015.py
@ -0,0 +1,7 @@
 from infi.clickhouse_orm import migrations
 from ..test_migrations import *
 operations = [
    migrations.AlterTable(Model4_compressed),
    migrations.AlterTable(Model2LowCardinality)
 ]
--- a/tests/test_alias_fields.py
+++ b/tests/test_alias_fields.py
@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class MaterializedFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(ModelWithAliasFields)
    def tearDown(self):
--- a/tests/test_array_fields.py
+++ b/tests/test_array_fields.py
@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class ArrayFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(ModelWithArrays)
    def tearDown(self):
--- a/tests/test_compressed_fields.py
+++ b/tests/test_compressed_fields.py
@ -0,0 +1,123 @@
 from __future__ import unicode_literals
 import unittest
 import datetime
 import pytz
 from infi.clickhouse_orm.database import Database
 from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
 from infi.clickhouse_orm.utils import parse_tsv
 class CompressedFieldsTestCase(unittest.TestCase):
    def setUp(self):
        self.database = Database('test-db', log_statements=True)
        self.database.create_table(CompressedModel)
    def tearDown(self):
        self.database.drop_database()
    def test_defaults(self):
        # Check that all fields have their explicit or implicit defaults
        instance = CompressedModel()
        self.database.insert([instance])
        self.assertEqual(instance.date_field, datetime.date(1970, 1, 1))
        self.assertEqual(instance.datetime_field, datetime.datetime(1970, 1, 1, tzinfo=pytz.utc))
        self.assertEqual(instance.string_field, 'dozo')
        self.assertEqual(instance.int64_field, 42)
        self.assertEqual(instance.float_field, 0)
        self.assertEqual(instance.nullable_field, None)
        self.assertEqual(instance.array_field, [])
    def test_assignment(self):
        # Check that all fields are assigned during construction
        kwargs = dict(
            uint64_field=217,
            date_field=datetime.date(1973, 12, 6),
            datetime_field=datetime.datetime(2000, 5, 24, 10, 22, tzinfo=pytz.utc),
            string_field='aloha',
            int64_field=-50,
            float_field=3.14,
            nullable_field=-2.718281,
            array_field=['123456789123456','','a']
        )
        instance = CompressedModel(**kwargs)
        self.database.insert([instance])
        for name, value in kwargs.items():
            self.assertEqual(kwargs[name], getattr(instance, name))
    def test_string_conversion(self):
        # Check field conversion from string during construction
        instance = CompressedModel(date_field='1973-12-06', int64_field='100', float_field='7', nullable_field=None, array_field='[a,b,c]')
        self.assertEqual(instance.date_field, datetime.date(1973, 12, 6))
        self.assertEqual(instance.int64_field, 100)
        self.assertEqual(instance.float_field, 7)
        self.assertEqual(instance.nullable_field, None)
        self.assertEqual(instance.array_field, ['a', 'b', 'c'])
        # Check field conversion from string during assignment
        instance.int64_field = '99'
        self.assertEqual(instance.int64_field, 99)
    def test_to_dict(self):
        instance = CompressedModel(date_field='1973-12-06', int64_field='100', float_field='7', array_field='[a,b,c]')
        self.assertDictEqual(instance.to_dict(), {
            "date_field": datetime.date(1973, 12, 6),
            "int64_field": 100,
            "float_field": 7.0,
            "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc),
            "alias_field": 0.0,
            'string_field': 'dozo',
            'nullable_field': None,
            'uint64_field': 0,
            'array_field': ['a','b','c']
        })
        self.assertDictEqual(instance.to_dict(include_readonly=False), {
            "date_field": datetime.date(1973, 12, 6),
            "int64_field": 100,
            "float_field": 7.0,
            "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc),
            'string_field': 'dozo',
            'nullable_field': None,
            'uint64_field': 0,
            'array_field': ['a', 'b', 'c']
        })
        self.assertDictEqual(
            instance.to_dict(include_readonly=False, field_names=('int64_field', 'alias_field', 'datetime_field')), {
                "int64_field": 100,
                "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)
            })
    # This test will fail on clickhouse version < 19.1.16, use skip test
    def test_confirm_compression_codec(self):
        instance = CompressedModel(date_field='1973-12-06', int64_field='100', float_field='7', array_field='[a,b,c]')
        self.database.insert([instance])
        r = self.database.raw("select name, compression_codec from system.columns where table = '{}' and database='{}' FORMAT TabSeparatedWithNamesAndTypes".format(instance.table_name(), self.database.db_name))
        lines = r.splitlines()
        field_names = parse_tsv(lines[0])
        field_types = parse_tsv(lines[1])
        data = [tuple(parse_tsv(line)) for line in lines[2:]]
        self.assertListEqual(data, [('uint64_field', 'CODEC(ZSTD(10))'),
                                    ('datetime_field', 'CODEC(Delta(4), ZSTD(1))'),
                                    ('date_field', 'CODEC(Delta(4), ZSTD(22))'),
                                    ('int64_field', 'CODEC(LZ4)'),
                                    ('string_field', 'CODEC(LZ4HC(10))'),
                                    ('nullable_field', 'CODEC(ZSTD(1))'),
                                    ('array_field', 'CODEC(Delta(2), LZ4HC(0))'),
                                    ('float_field', 'CODEC(NONE)'),
                                    ('alias_field', 'CODEC(ZSTD(4))')])
 class CompressedModel(Model):
    uint64_field        = UInt64Field(codec='ZSTD(10)')
    datetime_field      = DateTimeField(codec='Delta,ZSTD')
    date_field          = DateField(codec='Delta(4),ZSTD(22)')
    int64_field         = Int64Field(default=42, codec='LZ4')
    string_field        = StringField(default='dozo', codec='LZ4HC(10)')
    nullable_field      = NullableField(Float32Field(), codec='ZSTD')
    array_field         = ArrayField(FixedStringField(length=15), codec='Delta(2),LZ4HC')
    float_field         = Float32Field(codec='NONE')
    alias_field         = Float32Field(alias='float_field', codec='ZSTD(4)')
    engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field'))
--- a/tests/test_custom_fields.py
+++ b/tests/test_custom_fields.py
@ -1,18 +1,15 @@
 from __future__ import unicode_literals
 import unittest
 import six
 from uuid import UUID
 from infi.clickhouse_orm.database import Database
 from infi.clickhouse_orm.fields import Field, Int16Field
 from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.engines import Memory
 from infi.clickhouse_orm.utils import escape
 class CustomFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
    def tearDown(self):
        self.database.drop_database()
@ -35,37 +32,6 @@ class CustomFieldsTest(unittest.TestCase):
            with self.assertRaises(ValueError):
                TestModel(i=1, f=value)
    def test_uuid_field(self):
        # Create a model
        class TestModel(Model):
            i = Int16Field()
            f = UUIDField()
            engine = Memory()
        self.database.create_table(TestModel)
        # Check valid values (all values are the same UUID)
        values = [
            '{12345678-1234-5678-1234-567812345678}',
            '12345678123456781234567812345678',
            'urn:uuid:12345678-1234-5678-1234-567812345678',
            '\x12\x34\x56\x78'*4,
            (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
            0x12345678123456781234567812345678,
        ]
        for index, value in enumerate(values):
            rec = TestModel(i=index, f=value)
            self.database.insert([rec])
        for rec in TestModel.objects_in(self.database):
            self.assertEqual(rec.f, UUID(values[0]))
        # Check that ClickHouse encoding functions are supported
        for rec in self.database.select("SELECT i, UUIDNumToString(f) AS f FROM testmodel", TestModel):
            self.assertEqual(rec.f, UUID(values[0]))
        for rec in self.database.select("SELECT 1 as i, UUIDStringToNum('12345678-1234-5678-1234-567812345678') AS f", TestModel):
            self.assertEqual(rec.f, UUID(values[0]))
        # Check invalid values
        for value in [None, 'zzz', -1, '123']:
            with self.assertRaises(ValueError):
                TestModel(i=1, f=value)
 class BooleanField(Field):
@ -88,32 +54,3 @@ class BooleanField(Field):
        # The value was already converted by to_python, so it's a bool
        return '1' if value else '0'
 class UUIDField(Field):
    # The ClickHouse column type to use
    db_type = 'FixedString(16)'
    # The default value if empty
    class_default = UUID(int=0)
    def to_python(self, value, timezone_in_use):
        # Convert valid values to UUID instance
        if isinstance(value, UUID):
            return value
        elif isinstance(value, six.string_types):
            return UUID(bytes=value.encode('latin1')) if len(value) == 16 else UUID(value)
        elif isinstance(value, six.integer_types):
            return UUID(int=value)
        elif isinstance(value, tuple):
            return UUID(fields=value)
        else:
            raise ValueError('Invalid value for UUIDField: %r' % value)
    def to_db_string(self, value, quote=True):
        # The value was already converted by to_python, so it's a UUID instance
        val = value.bytes
        if six.PY3:
            val = str(val, 'latin1')
        return escape(val, quote)
--- a/tests/test_database.py
+++ b/tests/test_database.py
@ -112,6 +112,14 @@ class DatabaseTestCase(TestCaseWithData):
            self.assertEqual([obj.to_tsv() for obj in page_a.objects],
                              [obj.to_tsv() for obj in page_b.objects])
    def test_pagination_empty_page(self):
        for page_num in (-1, 1, 2):
            page = self.database.paginate(Person, 'first_name, last_name', page_num, 10, conditions="first_name = 'Ziggy'")
            self.assertEqual(page.number_of_objects, 0)
            self.assertEqual(page.objects, [])
            self.assertEqual(page.pages_total, 0)
            self.assertEqual(page.number, max(page_num, 1))
    def test_pagination_invalid_page(self):
        self._insert_and_check(self._sample_data(), len(data))
        for page_num in (0, -2, -100):
@ -142,7 +150,7 @@ class DatabaseTestCase(TestCaseWithData):
        exc = cm.exception
        self.assertEqual(exc.code, 193)
-        self.assertEqual(exc.message, 'Wrong password for user default')
+        self.assertTrue(exc.message.startswith('Wrong password for user default'))
    def test_nonexisting_db(self):
        db = Database('db_not_here', autocreate=False)
@ -150,7 +158,7 @@ class DatabaseTestCase(TestCaseWithData):
            db.create_table(Person)
        exc = cm.exception
        self.assertEqual(exc.code, 81)
-        self.assertEqual(exc.message, "Database db_not_here doesn't exist")
+        self.assertTrue(exc.message.startswith("Database db_not_here doesn't exist"))
        # Create and delete the db twice, to ensure db_exists gets updated
        for i in range(2):
            # Now create the database - should succeed
--- a/tests/test_datetime_fields.py
+++ b/tests/test_datetime_fields.py
@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import *
 class DateFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(ModelWithDate)
    def tearDown(self):
--- a/tests/test_decimal_fields.py
+++ b/tests/test_decimal_fields.py
@ -12,7 +12,7 @@ from infi.clickhouse_orm.engines import *
 class DecimalFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        try:
            self.database.create_table(DecimalModel)
        except ServerError as e:
--- a/tests/test_engines.py
+++ b/tests/test_engines.py
@ -14,7 +14,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class _EnginesHelperTestCase(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
    def tearDown(self):
        self.database.drop_database()
@ -115,8 +115,8 @@ class EnginesTestCase(_EnginesHelperTestCase):
            TestModel2(date='2017-01-02', event_id=2, event_group=2, event_count=2, event_version=2)
        ])
        # event_uversion is materialized field. So * won't select it and it will be zero
-        res = self.database.select('SELECT *, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel)
+        res = self.database.select('SELECT *, _table, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel)
-        res = [row for row in res]
+        res = list(res)
        self.assertEqual(2, len(res))
        self.assertDictEqual({
            '_table': 'testmodel1',
@ -161,9 +161,9 @@ class EnginesTestCase(_EnginesHelperTestCase):
        self.assertEqual(2, len(parts))
        self.assertEqual('testcollapsemodel', parts[0].table)
-        self.assertEqual('(201701, 13)', parts[0].partition)
+        self.assertEqual('(201701, 13)'.replace(' ', ''), parts[0].partition.replace(' ', ''))
        self.assertEqual('testmodel', parts[1].table)
-        self.assertEqual('(201701, 13)', parts[1].partition)
+        self.assertEqual('(201701, 13)'.replace(' ', ''), parts[1].partition.replace(' ', ''))
 class SampleModel(Model):
@ -209,7 +209,7 @@ class DistributedTestCase(_EnginesHelperTestCase):
        exc = cm.exception
        self.assertEqual(exc.code, 170)
-        self.assertEqual(exc.message, "Requested cluster 'cluster_name' not found")
+        self.assertTrue(exc.message.startswith("Requested cluster 'cluster_name' not found"))
    def test_verbose_engine_two_superclasses(self):
        class TestModel2(SampleModel):
--- a/tests/test_enum_fields.py
+++ b/tests/test_enum_fields.py
@ -6,16 +6,13 @@ from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
-try:
+from enum import Enum
    Enum # exists in Python 3.4+
 except NameError:
    from enum import Enum # use the enum34 library instead
 class EnumFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(ModelWithEnum)
        self.database.create_table(ModelWithEnumArray)
--- a/tests/test_fixed_string_fields.py
+++ b/tests/test_fixed_string_fields.py
@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class FixedStringFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(FixedStringModel)
    def tearDown(self):
--- a/tests/test_funcs.py
+++ b/tests/test_funcs.py
@ -24,7 +24,7 @@ class FuncsTestCase(TestCaseWithData):
        sql = 'SELECT %s AS value' % func.to_sql()
        logger.info(sql)
        result = list(self.database.select(sql))
-        logger.info('\t==> %s', result[0].value)
+        logger.info('\t==> %s', result[0].value if result else '<empty>')
        if expected_value is not None:
            self.assertEqual(result[0].value, expected_value)
@ -255,7 +255,7 @@ class FuncsTestCase(TestCaseWithData):
        try:
            self._test_func(F.base64Decode(F.base64Encode('Hello')), 'Hello')
            self._test_func(F.tryBase64Decode(F.base64Encode('Hello')), 'Hello')
-            self._test_func(F.tryBase64Decode('zzz'), '')
+            self._test_func(F.tryBase64Decode('zzz'), None)
        except ServerError as e:
            # ClickHouse version that doesn't support these functions
            raise unittest.SkipTest(e.message)
--- a/tests/test_join.py
+++ b/tests/test_join.py
@ -9,7 +9,7 @@ from infi.clickhouse_orm import database, engines, fields, models
 class JoinTest(unittest.TestCase):
    def setUp(self):
-        self.database = database.Database('test-db')
+        self.database = database.Database('test-db', log_statements=True)
        self.database.create_table(Foo)
        self.database.create_table(Bar)
        self.database.insert([Foo(id=i) for i in range(3)])
--- a/tests/test_materialized_fields.py
+++ b/tests/test_materialized_fields.py
@ -11,7 +11,7 @@ from infi.clickhouse_orm.engines import *
 class MaterializedFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(ModelWithMaterializedFields)
    def tearDown(self):
--- a/tests/test_migrations.py
+++ b/tests/test_migrations.py
@ -7,14 +7,11 @@ from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
 from infi.clickhouse_orm.migrations import MigrationHistory
 from enum import Enum
 # Add tests to path so that migrations will be importable
 import sys, os
 sys.path.append(os.path.dirname(__file__))
 try:
    Enum # exists in Python 3.4+
 except NameError:
    from enum import Enum # use the enum34 library instead
 import logging
 logging.basicConfig(level=logging.DEBUG, format='%(message)s')
@ -24,7 +21,7 @@ logging.getLogger("requests").setLevel(logging.WARNING)
 class MigrationsTestCase(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.drop_table(MigrationHistory)
    def tearDown(self):
@ -93,10 +90,21 @@ class MigrationsTestCase(unittest.TestCase):
        self.database.migrate('tests.sample_migrations', 14)
        self.assertTrue(self.tableExists(MaterializedModel1))
        self.assertEqual(self.getTableFields(MaterializedModel1),
-                          [('date_time', "DateTime"), ('int_field', 'Int8'), ('date', 'Date')])
+                          [('date_time', 'DateTime'), ('int_field', 'Int8'), ('date', 'Date'), ('int_field_plus_one', 'Int8')])
        self.assertTrue(self.tableExists(AliasModel1))
        self.assertEqual(self.getTableFields(AliasModel1),
-                          [('date', 'Date'), ('int_field', 'Int8'), ('date_alias', "Date")])
+                          [('date', 'Date'), ('int_field', 'Int8'), ('date_alias', 'Date'), ('int_field_plus_one', 'Int8')])
        self.database.migrate('tests.sample_migrations', 15)
        self.assertTrue(self.tableExists(Model4_compressed))
        if self.database.has_low_cardinality_support:
            self.assertEqual(self.getTableFields(Model2LowCardinality),
                             [('date', 'Date'), ('f1', 'LowCardinality(Int32)'), ('f3', 'LowCardinality(Float32)'),
                              ('f2', 'LowCardinality(String)'), ('f4', 'LowCardinality(Nullable(String))'), ('f5', 'Array(LowCardinality(UInt64))')])
        else:
            logging.warning('No support for low cardinality')
            self.assertEqual(self.getTableFields(Model2),
                             [('date', 'Date'), ('f1', 'Int32'), ('f3', 'Float32'), ('f2', 'String'), ('f4', 'Nullable(String)'),
                              ('f5', 'Array(UInt64)')])
 # Several different models with the same table name, to simulate a table that changes over time
@ -183,6 +191,7 @@ class MaterializedModel1(Model):
    date_time = DateTimeField()
    date = DateField(materialized='toDate(date_time)')
    int_field = Int8Field()
    int_field_plus_one = Int8Field(materialized='int_field + 1')
    engine = MergeTree('date', ('date',))
@ -206,6 +215,7 @@ class AliasModel1(Model):
    date = DateField()
    date_alias = DateField(alias='date')
    int_field = Int8Field()
    int_field_plus_one = Int8Field(alias='int_field + 1')
    engine = MergeTree('date', ('date',))
@ -256,3 +266,31 @@ class Model4Buffer_changed(BufferModel, Model4_changed):
    @classmethod
    def table_name(cls):
        return 'model4buffer'
 class Model4_compressed(Model):
    date = DateField()
    f3 = DateTimeField(codec='Delta,ZSTD(10)')
    f2 = StringField(codec='LZ4HC')
    engine = MergeTree('date', ('date',))
    @classmethod
    def table_name(cls):
        return 'model4'
 class Model2LowCardinality(Model):
    date = DateField()
    f1 = LowCardinalityField(Int32Field())
    f3 = LowCardinalityField(Float32Field())
    f2 = LowCardinalityField(StringField())
    f4 = LowCardinalityField(NullableField(StringField()))
    f5 = ArrayField(LowCardinalityField(UInt64Field()))
    engine = MergeTree('date', ('date',))
    @classmethod
    def table_name(cls):
        return 'mig'
--- a/tests/test_nullable_fields.py
+++ b/tests/test_nullable_fields.py
@ -6,6 +6,7 @@ from infi.clickhouse_orm.database import Database
 from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
 from infi.clickhouse_orm.utils import comma_join
 from datetime import date, datetime
@ -13,7 +14,7 @@ from datetime import date, datetime
 class NullableFieldsTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(ModelWithNullable)
    def tearDown(self):
@ -95,14 +96,19 @@ class NullableFieldsTest(unittest.TestCase):
            ModelWithNullable(date_field='2016-08-30', null_str='', null_int=42, null_date=dt),
            ModelWithNullable(date_field='2016-08-30', null_str='nothing', null_int=None, null_date=None),
            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=42, null_date=dt),
-            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None)
+            ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None, null_default=None)
        ])
    def _assert_sample_data(self, results):
        for r in results:
            print(r.to_dict())
        dt = date(1970, 1, 1)
        self.assertEqual(len(results), 4)
        self.assertIsNone(results[0].null_str)
        self.assertEqual(results[0].null_int, 42)
        self.assertEqual(results[0].null_default, 7)
        self.assertEqual(results[0].null_alias, 21)
        self.assertEqual(results[0].null_materialized, 420)
        self.assertEqual(results[0].null_date, dt)
        self.assertIsNone(results[1].null_date)
        self.assertEqual(results[1].null_str, 'nothing')
@ -110,19 +116,27 @@ class NullableFieldsTest(unittest.TestCase):
        self.assertIsNone(results[2].null_str)
        self.assertEqual(results[2].null_date, dt)
        self.assertEqual(results[2].null_int, 42)
        self.assertEqual(results[2].null_default, 7)
        self.assertEqual(results[2].null_alias, 21)
        self.assertEqual(results[0].null_materialized, 420)
        self.assertIsNone(results[3].null_int)
        self.assertIsNone(results[3].null_default)
        self.assertIsNone(results[3].null_alias)
        self.assertIsNone(results[3].null_materialized)
        self.assertIsNone(results[3].null_str)
        self.assertIsNone(results[3].null_date)
    def test_insert_and_select(self):
        self._insert_sample_data()
-        query = 'SELECT * from $table ORDER BY date_field'
+        fields = comma_join(ModelWithNullable.fields().keys())
        query = 'SELECT %s from $table ORDER BY date_field' % fields
        results = list(self.database.select(query, ModelWithNullable))
        self._assert_sample_data(results)
    def test_ad_hoc_model(self):
        self._insert_sample_data()
-        query = 'SELECT * from $db.modelwithnullable ORDER BY date_field'
+        fields = comma_join(ModelWithNullable.fields().keys())
        query = 'SELECT %s from $db.modelwithnullable ORDER BY date_field' % fields
        results = list(self.database.select(query))
        self._assert_sample_data(results)
@ -133,5 +147,8 @@ class ModelWithNullable(Model):
    null_str = NullableField(StringField(), extra_null_values={''})
    null_int = NullableField(Int32Field())
    null_date = NullableField(DateField())
    null_default = NullableField(Int32Field(), default=7)
    null_alias = NullableField(Int32Field(), alias='null_int/2')
    null_materialized = NullableField(Int32Field(), alias='null_int*10')
    engine = MergeTree('date_field', ('date_field',))
--- a/tests/test_querysets.py
+++ b/tests/test_querysets.py
@ -3,17 +3,15 @@ from __future__ import unicode_literals, print_function
 import unittest
 from infi.clickhouse_orm.database import Database
-from infi.clickhouse_orm.query import Q, F
+from infi.clickhouse_orm.query import Q
 from infi.clickhouse_orm.funcs import F
 from .base_test_with_data import *
 from datetime import date, datetime
 from enum import Enum
 from logging import getLogger
 logger = getLogger('tests')
 try:
    Enum # exists in Python 3.4+
 except NameError:
    from enum import Enum # use the enum34 library instead
 class QuerySetTestCase(TestCaseWithData):
@ -31,6 +29,13 @@ class QuerySetTestCase(TestCaseWithData):
        self.assertEqual(count, expected_count)
        self.assertEqual(qs.count(), expected_count)
    def test_prewhere(self):
        # We can't distinguish prewhere and where results, it affects performance only.
        # So let's control prewhere acts like where does
        qs = Person.objects_in(self.database)
        self.assertTrue(qs.filter(first_name='Connor', prewhere=True))
        self.assertFalse(qs.filter(first_name='Willy', prewhere=True))
    def test_no_filtering(self):
        qs = Person.objects_in(self.database)
        self._test_qs(qs, len(data))
@ -222,7 +227,7 @@ class QuerySetTestCase(TestCaseWithData):
        qs = Person.objects_in(self.database).order_by('first_name', 'last_name')
        # Try different page sizes
        for page_size in (1, 2, 7, 10, 30, 100, 150):
-            # Iterate over pages and collect all intances
+            # Iterate over pages and collect all instances
            page_num = 1
            instances = set()
            while True:
@ -296,7 +301,7 @@ class QuerySetTestCase(TestCaseWithData):
        qs = Person.objects_in(self.database)
        qs = qs.filter(Q(first_name='a'), F('greater', Person.height, 1.7), last_name='b')
        self.assertEqual(qs.conditions_as_sql(),
-                         "first_name = 'a' AND greater(`height`, 1.7) AND last_name = 'b'")
+                         "(first_name = 'a') AND (greater(`height`, 1.7)) AND (last_name = 'b')")
    def test_invalid_filter(self):
        qs = Person.objects_in(self.database)
@ -417,6 +422,17 @@ class AggregateTestCase(TestCaseWithData):
        print(qs.as_sql())
        self.assertEqual(qs.count(), 1)
    def test_aggregate_with_totals(self):
        qs = Person.objects_in(self.database).aggregate('first_name', count='count()').\
            with_totals().order_by('-count')[:5]
        print(qs.as_sql())
        result = list(qs)
        self.assertEqual(len(result), 6)
        for row in result[:-1]:
            self.assertEqual(2, row.count)
        self.assertEqual(100, result[-1].count)
    def test_double_underscore_field(self):
        class Mdl(Model):
            the__number = Int32Field()
@ -463,9 +479,9 @@ class FuncsTestCase(TestCaseWithData):
        # Numeric args
        self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)")
        # Date args
-        self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func('2018-12-31')")
+        self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func(toDate('2018-12-31'))")
        # Datetime args
-        self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func('1546214400')")
+        self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func(toDateTime('1546214400'))")
        # Boolean args
        self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)")
        # Null args
--- a/tests/test_readonly.py
+++ b/tests/test_readonly.py
@ -36,9 +36,9 @@ class ReadonlyTestCase(TestCaseWithData):
    def _check_db_readonly_err(self, exc, drop_table=None):
        self.assertEqual(exc.code, 164)
        if drop_table:
-            self.assertEqual(exc.message, 'Cannot drop table in readonly mode')
+            self.assertTrue(exc.message.startswith('Cannot drop table in readonly mode'))
        else:
-            self.assertEqual(exc.message, 'Cannot insert into table in readonly mode')
+            self.assertTrue(exc.message.startswith('Cannot insert into table in readonly mode'))
    def test_readonly_db_with_default_user(self):
        self._test_readonly_db('default')
--- a/tests/test_server_errors.py
+++ b/tests/test_server_errors.py
@ -0,0 +1,32 @@
 from __future__ import unicode_literals
 import unittest
 from infi.clickhouse_orm.database import ServerError
 class ServerErrorTest(unittest.TestCase):
    def test_old_format(self):
        code, msg = ServerError.get_error_code_msg("Code: 81, e.displayText() = DB::Exception: Database db_not_here doesn't exist, e.what() = DB::Exception (from [::1]:33458)")
        self.assertEqual(code, 81)
        self.assertEqual(msg, "Database db_not_here doesn't exist")
        code, msg = ServerError.get_error_code_msg("Code: 161, e.displayText() = DB::Exception: Limit for number of columns to read exceeded. Requested: 11, maximum: 1, e.what() = DB::Exception\n")
        self.assertEqual(code, 161)
        self.assertEqual(msg, "Limit for number of columns to read exceeded. Requested: 11, maximum: 1")
    def test_new_format(self):
        code, msg = ServerError.get_error_code_msg("Code: 164, e.displayText() = DB::Exception: Cannot drop table in readonly mode")
        self.assertEqual(code, 164)
        self.assertEqual(msg, "Cannot drop table in readonly mode")
        code, msg = ServerError.get_error_code_msg("Code: 48, e.displayText() = DB::Exception: Method write is not supported by storage Merge")
        self.assertEqual(code, 48)
        self.assertEqual(msg, "Method write is not supported by storage Merge")
        code, msg = ServerError.get_error_code_msg("Code: 60, e.displayText() = DB::Exception: Table default.zuzu doesn't exist.\n")
        self.assertEqual(code, 60)
        self.assertEqual(msg, "Table default.zuzu doesn't exist.")
--- a/tests/test_system_models.py
+++ b/tests/test_system_models.py
@ -13,8 +13,9 @@ from infi.clickhouse_orm.system_models import SystemPart
 class SystemTest(unittest.TestCase):
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
    def tearDown(self):
        self.database.drop_database()
@ -38,7 +39,7 @@ class SystemPartTest(unittest.TestCase):
    BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/']
    def setUp(self):
-        self.database = Database('test-db')
+        self.database = Database('test-db', log_statements=True)
        self.database.create_table(TestTable)
        self.database.create_table(CustomPartitionedTable)
        self.database.insert([TestTable(date_field=date.today())])
@ -54,6 +55,12 @@ class SystemPartTest(unittest.TestCase):
                return dirnames
        raise unittest.SkipTest('Cannot find backups dir')
    def test_is_read_only(self):
        self.assertTrue(SystemPart.is_read_only())
    def test_is_system_model(self):
        self.assertTrue(SystemPart.is_system_model())
    def test_get_all(self):
        parts = SystemPart.get(self.database)
        self.assertEqual(len(list(parts)), 2)
@ -62,7 +69,8 @@ class SystemPartTest(unittest.TestCase):
        parts = list(SystemPart.get_active(self.database))
        self.assertEqual(len(parts), 2)
        parts[0].detach()
-        self.assertEqual(len(list(SystemPart.get_active(self.database))), 1)
+        parts = list(SystemPart.get_active(self.database))
        self.assertEqual(len(parts), 1)
    def test_get_conditions(self):
        parts = list(SystemPart.get(self.database, conditions="table='testtable'"))
@ -101,6 +109,10 @@ class SystemPartTest(unittest.TestCase):
        # TODO Not tested, as I have no replication set
        pass
    def test_query(self):
        SystemPart.objects_in(self.database).count()
        list(SystemPart.objects_in(self.database).filter(table='testtable'))
 class TestTable(Model):
    date_field = DateField()
--- a/tests/test_uuid_fields.py
+++ b/tests/test_uuid_fields.py
@ -0,0 +1,45 @@
 from __future__ import unicode_literals
 import unittest
 from uuid import UUID
 from infi.clickhouse_orm.database import Database
 from infi.clickhouse_orm.fields import Int16Field, UUIDField
 from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.engines import Memory
 class UUIDFieldsTest(unittest.TestCase):
    def setUp(self):
        self.database = Database('test-db', log_statements=True)
    def tearDown(self):
        self.database.drop_database()
    def test_uuid_field(self):
        # Create a model
        class TestModel(Model):
            i = Int16Field()
            f = UUIDField()
            engine = Memory()
        self.database.create_table(TestModel)
        # Check valid values (all values are the same UUID)
        values = [
            '12345678-1234-5678-1234-567812345678',
            '{12345678-1234-5678-1234-567812345678}',
            '12345678123456781234567812345678',
            'urn:uuid:12345678-1234-5678-1234-567812345678',
            b'\x12\x34\x56\x78'*4,
            (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678),
            0x12345678123456781234567812345678,
            UUID(int=0x12345678123456781234567812345678),
        ]
        for index, value in enumerate(values):
            rec = TestModel(i=index, f=value)
            self.database.insert([rec])
        for rec in TestModel.objects_in(self.database):
            self.assertEqual(rec.f, UUID(values[0]))
        # Check invalid values
        for value in [None, 'zzz', -1, '123']:
            with self.assertRaises(ValueError):
                TestModel(i=1, f=value)