Functions WIP

src/infi/clickhouse_orm/fields.py

@@ -8,10 +8,10 @@ from calendar import timegm
 from decimal import Decimal, localcontext
 
 from .utils import escape, parse_array, comma_join
-from .query import F
+from .funcs import F, FunctionOperatorsMixin
 
 
-class Field(object):
+class Field(FunctionOperatorsMixin):
     '''
     Abstract base class for all field types.
     '''
@@ -99,26 +99,6 @@ class Field(object):
             inner_field = getattr(inner_field, 'inner_field', None)
         return False
 
-    # Support comparison operators (for use in querysets)
-
-    def __lt__(self, other):
-        return F.less(self, other)
-
-    def __le__(self, other):
-        return F.lessOrEquals(self, other)
-
-    def __eq__(self, other):
-        return F.equals(self, other)
-
-    def __ne__(self, other):
-        return F.notEquals(self, other)
-
-    def __gt__(self, other):
-        return F.greater(self, other)
-
-    def __ge__(self, other):
-        return F.greaterOrEquals(self, other)
-
 
 class StringField(Field):
 

src/infi/clickhouse_orm/funcs.py

@@ -0,0 +1,578 @@
+import six
+from datetime import date, datetime, tzinfo
+import functools
+
+from .utils import is_iterable, comma_join
+from .query import Cond
+
+
+def binary_operator(func):
+    """
+    Decorates a function to mark it as a binary operator.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        ret = func(*args, **kwargs)
+        ret.is_binary_operator = True
+        return ret
+    return wrapper
+
+
+class FunctionOperatorsMixin(object):
+    """
+    A mixin for implementing Python operators using F objects.
+    """
+
+    # Comparison operators
+
+    def __lt__(self, other):
+        return F.less(self, other)
+
+    def __le__(self, other):
+        return F.lessOrEquals(self, other)
+
+    def __eq__(self, other):
+        return F.equals(self, other)
+
+    def __ne__(self, other):
+        return F.notEquals(self, other)
+
+    def __gt__(self, other):
+        return F.greater(self, other)
+
+    def __ge__(self, other):
+        return F.greaterOrEquals(self, other)
+
+    # Arithmetic operators
+
+    def __add__(self, other):
+        return F.plus(self, other)
+
+    def __radd__(self, other):
+        return F.plus(other, self)
+
+    def __sub__(self, other):
+        return F.minus(self, other)
+
+    def __rsub__(self, other):
+        return F.minus(other, self)
+
+    def __mul__(self, other):
+        return F.multiply(self, other)
+
+    def __rmul__(self, other):
+        return F.multiply(other, self)
+
+    def __div__(self, other):
+        return F.divide(self, other)
+
+    def __rdiv__(self, other):
+        return F.divide(other, self)
+
+    def __mod__(self, other):
+        return F.modulo(self, other)
+
+    def __rmod__(self, other):
+        return F.modulo(other, self)
+
+    def __neg__(self):
+        return F.negate(self)
+
+    def __pos__(self):
+        return self
+
+    # Logical operators
+
+    def __and__(self, other):
+        return F._and(self, other)
+
+    def __rand__(self, other):
+        return F._and(other, self)
+
+    def __or__(self, other):
+        return F._or(self, other)
+
+    def __ror__(self, other):
+        return F._or(other, self)
+
+    def __xor__(self, other):
+        return F._xor(self, other)
+
+    def __rxor__(self, other):
+        return F._xor(other, self)
+
+    def __invert__(self):
+        return F._not(self)
+
+
+class F(Cond, FunctionOperatorsMixin):
+    """
+    Represents a database function call and its arguments.
+    It doubles as a query condition when the function returns a boolean result.
+    """
+    def __init__(self, name, *args):
+        self.name = name
+        self.args = args
+        self.is_binary_operator = False
+
+    def to_sql(self, *args):
+        """
+        Generates an SQL string for this function and its arguments.
+        For example if the function name is a symbol of a binary operator:
+            (2.54 * `height`)
+        For other functions:
+            gcd(12, 300)
+        """
+        if self.is_binary_operator:
+            prefix = ''
+            sep = ' ' + self.name + ' '
+        else:
+            prefix = self.name
+            sep = ', '
+        arg_strs = (self.arg_to_sql(arg) for arg in self.args)
+        return prefix + '(' + sep.join(arg_strs) + ')'
+
+    def arg_to_sql(self, arg):
+        """
+        Converts a function argument to SQL string according to its type.
+        Supports functions, model fields, strings, dates, datetimes, booleans,
+        None, numbers, timezones, arrays/iterables.
+        """
+        from .fields import Field, StringField, DateTimeField, DateField
+        if isinstance(arg, F):
+            return arg.to_sql()
+        if isinstance(arg, Field):
+            return "`%s`" % arg.name
+        if isinstance(arg, six.string_types):
+            return StringField().to_db_string(arg)
+        if isinstance(arg, datetime):
+            return "toDateTime(%s)" % DateTimeField().to_db_string(arg)
+        if isinstance(arg, date):
+            return "toDate('%s')" % arg.isoformat()
+        if isinstance(arg, bool):
+            return six.text_type(int(arg))
+        if isinstance(arg, tzinfo):
+            return StringField().to_db_string(arg.tzname(None))
+        if arg is None:
+            return 'NULL'
+        if is_iterable(arg):
+            return '[' + comma_join(self.arg_to_sql(x) for x in arg) + ']'
+        return six.text_type(arg)
+
+    # Arithmetic functions
+
+    @staticmethod
+    @binary_operator
+    def plus(a, b):
+        return F('+', a, b)
+
+    @staticmethod
+    @binary_operator
+    def minus(a, b):
+        return F('-', a, b)
+
+    @staticmethod
+    @binary_operator
+    def multiply(a, b):
+        return F('*', a, b)
+
+    @staticmethod
+    @binary_operator
+    def divide(a, b):
+        return F('/', a, b)
+
+    @staticmethod
+    def intDiv(a, b):
+        return F('intDiv', a, b)
+
+    @staticmethod
+    def intDivOrZero(a, b):
+        return F('intDivOrZero', a, b)
+
+    @staticmethod
+    @binary_operator
+    def modulo(a, b):
+        return F('%', a, b)
+
+    @staticmethod
+    def negate(a):
+        return F('negate', a)
+
+    @staticmethod
+    def abs(a):
+        return F('abs', a)
+
+    @staticmethod
+    def gcd(a, b):
+        return F('gcd',a, b)
+
+    @staticmethod
+    def lcm(a, b):
+        return F('lcm', a, b)
+
+    # Comparison functions
+
+    @staticmethod
+    @binary_operator
+    def equals(a, b):
+        return F('=', a, b)
+
+    @staticmethod
+    @binary_operator
+    def notEquals(a, b):
+        return F('!=', a, b)
+
+    @staticmethod
+    @binary_operator
+    def less(a, b):
+        return F('<', a, b)
+
+    @staticmethod
+    @binary_operator
+    def greater(a, b):
+        return F('>', a, b)
+
+    @staticmethod
+    @binary_operator
+    def lessOrEquals(a, b):
+        return F('<=', a, b)
+
+    @staticmethod
+    @binary_operator
+    def greaterOrEquals(a, b):
+        return F('>=', a, b)
+
+    # Logical functions (should be used as python operators: & | ^ ~)
+
+    @staticmethod
+    @binary_operator
+    def _and(a, b):
+        return F('AND', a, b)
+
+    @staticmethod
+    @binary_operator
+    def _or(a, b):
+        return F('OR', a, b)
+
+    @staticmethod
+    def _xor(a, b):
+        return F('xor', a, b)
+
+    @staticmethod
+    def _not(a):
+        return F('not', a)
+
+    # Functions for working with dates and times
+
+    @staticmethod
+    def toYear(d):
+        return F('toYear', d)
+
+    @staticmethod
+    def toMonth(d):
+        return F('toMonth', d)
+
+    @staticmethod
+    def toDayOfMonth(d):
+        return F('toDayOfMonth', d)
+
+    @staticmethod
+    def toDayOfWeek(d):
+        return F('toDayOfWeek', d)
+
+    @staticmethod
+    def toHour(d):
+        return F('toHour', d)
+
+    @staticmethod
+    def toMinute(d):
+        return F('toMinute', d)
+
+    @staticmethod
+    def toSecond(d):
+        return F('toSecond', d)
+
+    @staticmethod
+    def toMonday(d):
+        return F('toMonday', d)
+
+    @staticmethod
+    def toStartOfMonth(d):
+        return F('toStartOfMonth', d)
+
+    @staticmethod
+    def toStartOfQuarter(d):
+        return F('toStartOfQuarter', d)
+
+    @staticmethod
+    def toStartOfYear(d):
+        return F('toStartOfYear', d)
+
+    @staticmethod
+    def toStartOfMinute(d):
+        return F('toStartOfMinute', d)
+
+    @staticmethod
+    def toStartOfFiveMinute(d):
+        return F('toStartOfFiveMinute', d)
+
+    @staticmethod
+    def toStartOfFifteenMinutes(d):
+        return F('toStartOfFifteenMinutes', d)
+
+    @staticmethod
+    def toStartOfHour(d):
+        return F('toStartOfHour', d)
+
+    @staticmethod
+    def toStartOfDay(d):
+        return F('toStartOfDay', d)
+
+    @staticmethod
+    def toTime(d, timezone=''):
+        return F('toTime', d, timezone)
+
+    @staticmethod
+    def toRelativeYearNum(d, timezone=''):
+        return F('toRelativeYearNum', d, timezone)
+
+    @staticmethod
+    def toRelativeMonthNum(d, timezone=''):
+        return F('toRelativeMonthNum', d, timezone)
+
+    @staticmethod
+    def toRelativeWeekNum(d, timezone=''):
+        return F('toRelativeWeekNum', d, timezone)
+
+    @staticmethod
+    def toRelativeDayNum(d, timezone=''):
+        return F('toRelativeDayNum', d, timezone)
+
+    @staticmethod
+    def toRelativeHourNum(d, timezone=''):
+        return F('toRelativeHourNum', d, timezone)
+
+    @staticmethod
+    def toRelativeMinuteNum(d, timezone=''):
+        return F('toRelativeMinuteNum', d, timezone)
+
+    @staticmethod
+    def toRelativeSecondNum(d, timezone=''):
+        return F('toRelativeSecondNum', d, timezone)
+
+    @staticmethod
+    def now():
+        return F('now')
+
+    @staticmethod
+    def today():
+        return F('today')
+
+    @staticmethod
+    def yesterday():
+        return F('yesterday')
+
+    @staticmethod
+    def timeSlot(d):
+        return F('timeSlot', d)
+
+    @staticmethod
+    def timeSlots(start_time, duration):
+        return F('timeSlots', start_time, F.toUInt32(duration))
+
+    @staticmethod
+    def formatDateTime(d, format, timezone=''):
+        return F('formatDateTime', d, format, timezone)
+
+    # Type conversion functions
+
+    @staticmethod
+    def toUInt8(x):
+        return F('toUInt8', x)
+
+    @staticmethod
+    def toUInt16(x):
+        return F('toUInt16', x)
+
+    @staticmethod
+    def toUInt32(x):
+        return F('toUInt32', x)
+
+    @staticmethod
+    def toUInt64(x):
+        return F('toUInt64', x)
+
+    @staticmethod
+    def toInt8(x):
+        return F('toInt8', x)
+
+    @staticmethod
+    def toInt16(x):
+        return F('toInt16', x)
+
+    @staticmethod
+    def toInt32(x):
+        return F('toInt32', x)
+
+    @staticmethod
+    def toInt64(x):
+        return F('toInt64', x)
+
+    @staticmethod
+    def toFloat32(x):
+        return F('toFloat32', x)
+
+    @staticmethod
+    def toFloat64(x):
+        return F('toFloat64', x)
+
+    @staticmethod
+    def toUInt8OrZero(x):
+        return F('toUInt8OrZero', x)
+
+    @staticmethod
+    def toUInt16OrZero(x):
+        return F('toUInt16OrZero', x)
+
+    @staticmethod
+    def toUInt32OrZero(x):
+        return F('toUInt32OrZero', x)
+
+    @staticmethod
+    def toUInt64OrZero(x):
+        return F('toUInt64OrZero', x)
+
+    @staticmethod
+    def toInt8OrZero(x):
+        return F('toInt8OrZero', x)
+
+    @staticmethod
+    def toInt16OrZero(x):
+        return F('toInt16OrZero', x)
+
+    @staticmethod
+    def toInt32OrZero(x):
+        return F('toInt32OrZero', x)
+
+    @staticmethod
+    def toInt64OrZero(x):
+        return F('toInt64OrZero', x)
+
+    @staticmethod
+    def toFloat32OrZero(x):
+        return F('toFloat32OrZero', x)
+
+    @staticmethod
+    def toFloat64OrZero(x):
+        return F('toFloat64OrZero', x)
+
+    @staticmethod
+    def toDecimal32(x, scale):
+        return F('toDecimal32', x, scale)
+
+    @staticmethod
+    def toDecimal64(x, scale):
+        return F('toDecimal64', x, scale)
+
+    @staticmethod
+    def toDecimal128(x, scale):
+        return F('toDecimal128', x, scale)
+
+    @staticmethod
+    def toDate(x):
+        return F('toDate', x)
+
+    @staticmethod
+    def toDateTime(x):
+        return F('toDateTime', x)
+
+    @staticmethod
+    def toString(x):
+        return F('toString', x)
+
+    @staticmethod
+    def toFixedString(s, length):
+        return F('toFixedString', s, length)
+
+    @staticmethod
+    def toStringCutToZero(s):
+        return F('toStringCutToZero', s)
+
+    @staticmethod
+    def CAST(x, type):
+        return F('CAST', x, type)
+
+    # Functions for working with strings
+
+    @staticmethod
+    def empty(s):
+        return F('empty', s)
+
+    @staticmethod
+    def notEmpty(s):
+        return F('notEmpty', s)
+
+    @staticmethod
+    def length(s):
+        return F('length', s)
+
+    @staticmethod
+    def lengthUTF8(s):
+        return F('lengthUTF8', s)
+
+    @staticmethod
+    def lower(s):
+        return F('lower', s)
+
+    @staticmethod
+    def upper(s):
+        return F('upper', s)
+
+    @staticmethod
+    def lowerUTF8(s):
+        return F('lowerUTF8', s)
+
+    @staticmethod
+    def upperUTF8(s):
+        return F('upperUTF8', s)
+
+    @staticmethod
+    def reverse(s):
+        return F('reverse', s)
+
+    @staticmethod
+    def reverseUTF8(s):
+        return F('reverseUTF8', s)
+
+    @staticmethod
+    def concat(*args):
+        return F('concat', *args)
+
+    @staticmethod
+    def substring(s, offset, length):
+        return F('substring', s, offset, length)
+
+    @staticmethod
+    def substringUTF8(s, offset, length):
+        return F('substringUTF8', s, offset, length)
+
+    @staticmethod
+    def appendTrailingCharIfAbsent(s, c):
+        return F('appendTrailingCharIfAbsent', s, c)
+
+    @staticmethod
+    def convertCharset(s, from_charset, to_charset):
+        return F('convertCharset', s, from_charset, to_charset)
+
+    @staticmethod
+    def base64Encode(s):
+        return F('base64Encode', s)
+
+    @staticmethod
+    def base64Decode(s):
+        return F('base64Decode', s)
+
+    @staticmethod
+    def tryBase64Decode(s):
+        return F('tryBase64Decode', s)
+

src/infi/clickhouse_orm/query.py

@@ -5,8 +5,8 @@ import pytz
 from copy import copy
 from math import ceil
 from .engines import CollapsingMergeTree
-from datetime import date, datetime
+from datetime import date, datetime, tzinfo
-from .utils import comma_join, is_iterable
+from .utils import comma_join
 
 
 # TODO
@@ -26,6 +26,7 @@ class Operator(object):
         raise NotImplementedError   # pragma: no cover
 
     def _value_to_sql(self, field, value, quote=True):
+        from infi.clickhouse_orm.funcs import F
         if isinstance(value, F):
             return value.to_sql()
         return field.to_db_string(field.to_python(value, pytz.utc), quote)
@@ -185,293 +186,6 @@ class FieldCond(Cond):
         return self._operator.to_sql(model_cls, self._field_name, self._value)
 
 
-class F(Cond):
-    """
-    Represents a database function call and its arguments.
-    It doubles as a query condition when the function returns a boolean result.
-    """
-
-    def __init__(self, name, *args):
-        self.name = name
-        self.args = args
-
-    def to_sql(self, *args):
-        args_sql = comma_join(self.arg_to_sql(arg) for arg in self.args)
-        return self.name + '(' + args_sql + ')'
-
-    def arg_to_sql(self, arg):
-        from .fields import Field, StringField, DateTimeField, DateField
-        if isinstance(arg, F):
-            return arg.to_sql()
-        if isinstance(arg, Field):
-            return "`%s`" % arg.name
-        if isinstance(arg, six.string_types):
-            return StringField().to_db_string(arg)
-        if isinstance(arg, datetime):
-            return DateTimeField().to_db_string(arg)
-        if isinstance(arg, date):
-            return DateField().to_db_string(arg)
-        if isinstance(arg, bool):
-            return six.text_type(int(arg))
-        if arg is None:
-            return 'NULL'
-        if is_iterable(arg):
-            return '[' + comma_join(self.arg_to_sql(x) for x in arg) + ']'
-        return six.text_type(arg)
-
-    # Support comparison operators with F objects
-
-    def __lt__(self, other):
-        return F.less(self, other)
-
-    def __le__(self, other):
-        return F.lessOrEquals(self, other)
-
-    def __eq__(self, other):
-        return F.equals(self, other)
-
-    def __ne__(self, other):
-        return F.notEquals(self, other)
-
-    def __gt__(self, other):
-        return F.greater(self, other)
-
-    def __ge__(self, other):
-        return F.greaterOrEquals(self, other)
-
-    # Support arithmetic operations on F objects
-
-    def __add__(self, other):
-        return F.plus(self, other)
-
-    def __radd__(self, other):
-        return F.plus(other, self)
-
-    def __sub__(self, other):
-        return F.minus(self, other)
-
-    def __rsub__(self, other):
-        return F.minus(other, self)
-
-    def __mul__(self, other):
-        return F.multiply(self, other)
-
-    def __rmul__(self, other):
-        return F.multiply(other, self)
-
-    def __div__(self, other):
-        return F.divide(self, other)
-
-    def __rdiv__(self, other):
-        return F.divide(other, self)
-
-    def __mod__(self, other):
-        return F.modulo(self, other)
-
-    def __rmod__(self, other):
-        return F.modulo(other, self)
-
-    def __neg__(self):
-        return F.negate(self)
-
-    def __pos__(self):
-        return self
-
-    # Arithmetic functions
-
-    @staticmethod
-    def plus(a, b):
-        return F('plus', a, b)
-
-    @staticmethod
-    def minus(a, b):
-        return F('minus', a, b)
-
-    @staticmethod
-    def multiply(a, b):
-        return F('multiply', a, b)
-
-    @staticmethod
-    def divide(a, b):
-        return F('divide', a, b)
-
-    @staticmethod
-    def intDiv(a, b):
-        return F('intDiv', a, b)
-
-    @staticmethod
-    def intDivOrZero(a, b):
-        return F('intDivOrZero', a, b)
-
-    @staticmethod
-    def modulo(a, b):
-        return F('modulo', a, b)
-
-    @staticmethod
-    def negate(a):
-        return F('negate', a)
-
-    @staticmethod
-    def abs(a):
-        return F('abs', a)
-
-    @staticmethod
-    def gcd(a, b):
-        return F('gcd',a, b)
-
-    @staticmethod
-    def lcm(a, b):
-        return F('lcm', a, b)
-
-    # Comparison functions
-
-    @staticmethod
-    def equals(a, b):
-        return F('equals', a, b)
-
-    @staticmethod
-    def notEquals(a, b):
-        return F('notEquals', a, b)
-
-    @staticmethod
-    def less(a, b):
-        return F('less', a, b)
-
-    @staticmethod
-    def greater(a, b):
-        return F('greater', a, b)
-
-    @staticmethod
-    def lessOrEquals(a, b):
-        return F('lessOrEquals', a, b)
-
-    @staticmethod
-    def greaterOrEquals(a, b):
-        return F('greaterOrEquals', a, b)
-
-    # Functions for working with dates and times
-
-    @staticmethod
-    def toYear(d):
-        return F('toYear', d)
-
-    @staticmethod
-    def toMonth(d):
-        return F('toMonth', d)
-
-    @staticmethod
-    def toDayOfMonth(d):
-        return F('toDayOfMonth', d)
-
-    @staticmethod
-    def toDayOfWeek(d):
-        return F('toDayOfWeek', d)
-
-    @staticmethod
-    def toHour(d):
-        return F('toHour', d)
-
-    @staticmethod
-    def toMinute(d):
-        return F('toMinute', d)
-
-    @staticmethod
-    def toSecond(d):
-        return F('toSecond', d)
-
-    @staticmethod
-    def toMonday(d):
-        return F('toMonday', d)
-
-    @staticmethod
-    def toStartOfMonth(d):
-        return F('toStartOfMonth', d)
-
-    @staticmethod
-    def toStartOfQuarter(d):
-        return F('toStartOfQuarter', d)
-
-    @staticmethod
-    def toStartOfYear(d):
-        return F('toStartOfYear', d)
-
-    @staticmethod
-    def toStartOfMinute(d):
-        return F('toStartOfMinute', d)
-
-    @staticmethod
-    def toStartOfFiveMinute(d):
-        return F('toStartOfFiveMinute', d)
-
-    @staticmethod
-    def toStartOfFifteenMinutes(d):
-        return F('toStartOfFifteenMinutes', d)
-
-    @staticmethod
-    def toStartOfHour(d):
-        return F('toStartOfHour', d)
-
-    @staticmethod
-    def toStartOfDay(d):
-        return F('toStartOfDay', d)
-
-    @staticmethod
-    def toTime(d):
-        return F('toTime', d)
-
-    @staticmethod
-    def toRelativeYearNum(d, timezone=''):
-        return F('toRelativeYearNum', d, timezone)
-
-    @staticmethod
-    def toRelativeMonthNum(d, timezone=''):
-        return F('toRelativeMonthNum', d, timezone)
-
-    @staticmethod
-    def toRelativeWeekNum(d, timezone=''):
-        return F('toRelativeWeekNum', d, timezone)
-
-    @staticmethod
-    def toRelativeDayNum(d, timezone=''):
-        return F('toRelativeDayNum', d, timezone)
-
-    @staticmethod
-    def toRelativeHourNum(d, timezone=''):
-        return F('toRelativeHourNum', d, timezone)
-
-    @staticmethod
-    def toRelativeMinuteNum(d, timezone=''):
-        return F('toRelativeMinuteNum', d, timezone)
-
-    @staticmethod
-    def toRelativeSecondNum(d, timezone=''):
-        return F('toRelativeSecondNum', d, timezone)
-
-    @staticmethod
-    def now():
-        return F('now')
-
-    @staticmethod
-    def today():
-        return F('today')
-
-    @staticmethod
-    def yesterday(d):
-        return F('yesterday')
-
-    @staticmethod
-    def timeSlot(d):
-        return F('timeSlot', d)
-
-    @staticmethod
-    def timeSlots(start_time, duration):
-        return F('timeSlots', start_time, duration)
-
-    @staticmethod
-    def formatDateTime(d, format, timezone=''):
-        return F('formatDateTime', d, format, timezone)
-
-
 class Q(object):
 
     AND_MODE = ' AND '
@@ -542,6 +256,7 @@ class QuerySet(object):
         self._order_by = []
         self._q = []
         self._fields = model_cls.fields().keys()
+        self._extra = {}
         self._limits = None
         self._distinct = False
         self._final = False
@@ -590,6 +305,8 @@ class QuerySet(object):
         fields = '*'
         if self._fields:
             fields = comma_join('`%s`' % field for field in self._fields)
+        for name, func in self._extra.items():
+            fields += ', %s AS %s' % (func.to_sql(), name)
         ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
         limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
         final = ' FINAL' if self._final else ''
@@ -645,11 +362,17 @@ class QuerySet(object):
         qs._fields = field_names
         return qs
 
+    def extra(self, **kwargs):
+        qs = copy(self)
+        qs._extra = kwargs
+        return qs
+
     def filter(self, *q, **filter_fields):
         """
         Returns a copy of this queryset that includes only rows matching the conditions.
         Add q object to query if it specified.
         """
+        from infi.clickhouse_orm.funcs import F
         qs = copy(self)
         qs._q = list(self._q)
         for arg in q:

tests/test_decimal_fields.py

@@ -13,15 +13,11 @@ class DecimalFieldsTest(unittest.TestCase):
 
     def setUp(self):
         self.database = Database('test-db')
-        self.database.add_setting('allow_experimental_decimal_type', 1)
         try:
             self.database.create_table(DecimalModel)
         except ServerError as e:
-            if 'Unknown setting' in e.message:
+            # This ClickHouse version does not support decimals yet
-                # This ClickHouse version does not support decimals yet
+            raise unittest.SkipTest(e.message)
-                raise unittest.SkipTest(e.message)
-            else:
-                raise
 
     def tearDown(self):
         self.database.drop_database()

tests/test_funcs.py

@@ -0,0 +1,261 @@
+import unittest
+from .base_test_with_data import *
+from .test_querysets import SampleModel
+from datetime import date, datetime, tzinfo, timedelta
+from infi.clickhouse_orm.database import ServerError
+
+
+class FuncsTestCase(TestCaseWithData):
+
+    def setUp(self):
+        super(FuncsTestCase, self).setUp()
+        self.database.insert(self._sample_data())
+
+    def _test_qs(self, qs, expected_count):
+        logger.info(qs.as_sql())
+        count = 0
+        for instance in qs:
+            count += 1
+            logger.info('\t[%d]\t%s' % (count, instance.to_dict()))
+        self.assertEqual(count, expected_count)
+        self.assertEqual(qs.count(), expected_count)
+
+    def _test_func(self, func, expected_value=None):
+        sql = 'SELECT %s AS value' % func.to_sql()
+        logger.info(sql)
+        result = list(self.database.select(sql))
+        logger.info('\t==> %s', result[0].value)
+        if expected_value is not None:
+            self.assertEqual(result[0].value, expected_value)
+
+    def test_func_to_sql(self):
+        # No args
+        self.assertEqual(F('func').to_sql(), 'func()')
+        # String args
+        self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')")
+        # Numeric args
+        self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)")
+        # Date args
+        self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func(toDate('2018-12-31'))")
+        # Datetime args
+        self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func(toDateTime('1546214400'))")
+        # Boolean args
+        self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)")
+        # Timezone args
+        self.assertEqual(F('func', pytz.utc).to_sql(), "func('UTC')")
+        self.assertEqual(F('func', pytz.timezone('Europe/Athens')).to_sql(), "func('Europe/Athens')")
+        # Null args
+        self.assertEqual(F('func', None).to_sql(), "func(NULL)")
+        # Fields as args
+        self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)")
+        # Funcs as args
+        self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))')
+        # Iterables as args
+        x = [1, 'z', F('foo', 17)]
+        for y in [x, tuple(x), iter(x)]:
+            self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)")
+        self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([[1, 2], [3, 4]])")
+        # Binary operator functions
+        self.assertEqual(F.plus(1, 2).to_sql(), "(1 + 2)")
+        self.assertEqual(F.lessOrEquals(1, 2).to_sql(), "(1 <= 2)")
+
+    def test_filter_float_field(self):
+        qs = Person.objects_in(self.database)
+        # Height > 2
+        self._test_qs(qs.filter(F.greater(Person.height, 2)), 0)
+        self._test_qs(qs.filter(Person.height > 2), 0)
+        # Height > 1.61
+        self._test_qs(qs.filter(F.greater(Person.height, 1.61)), 96)
+        self._test_qs(qs.filter(Person.height > 1.61), 96)
+        # Height < 1.61
+        self._test_qs(qs.filter(F.less(Person.height, 1.61)), 4)
+        self._test_qs(qs.filter(Person.height < 1.61), 4)
+
+    def test_filter_date_field(self):
+        qs = Person.objects_in(self.database)
+        # People born on the 30th
+        self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3)
+        self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3)
+        self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3)
+        # People born on Sunday
+        self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18)
+        self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18)
+        self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18)
+        # People born on 1976-10-01
+        self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1)
+        self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 01))), 1)
+        self._test_qs(qs.filter(Person.birthday == date(1976, 10, 01)), 1)
+
+    def test_func_as_field_value(self):
+        qs = Person.objects_in(self.database)
+        self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96)
+        self._test_qs(qs.exclude(birthday=F.today()), 100)
+        self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100)
+
+    def test_comparison_operators(self):
+        one = F.plus(1, 0)
+        two = F.plus(1, 1)
+        self._test_func(one > one, 0)
+        self._test_func(two > one, 1)
+        self._test_func(one >= two, 0)
+        self._test_func(one >= one, 1)
+        self._test_func(one < one, 0)
+        self._test_func(one < two, 1)
+        self._test_func(two <= one, 0)
+        self._test_func(one <= one, 1)
+        self._test_func(one == two, 0)
+        self._test_func(one == one, 1)
+        self._test_func(one != one, 0)
+        self._test_func(one != two, 1)
+
+    def test_arithmetic_operators(self):
+        one = F.plus(1, 0)
+        two = F.plus(1, 1)
+        # +
+        self._test_func(one + two, 3)
+        self._test_func(one + 2, 3)
+        self._test_func(2 + one, 3)
+        # -
+        self._test_func(one - two, -1)
+        self._test_func(one - 2, -1)
+        self._test_func(1 - two, -1)
+        # *
+        self._test_func(one * two, 2)
+        self._test_func(one * 2, 2)
+        self._test_func(1 * two, 2)
+        # /
+        self._test_func(one / two, 0.5)
+        self._test_func(one / 2, 0.5)
+        self._test_func(1 / two, 0.5)
+        # %
+        self._test_func(one % two, 1)
+        self._test_func(one % 2, 1)
+        self._test_func(1 % two, 1)
+        # sign
+        self._test_func(-one, -1)
+        self._test_func(--one, 1)
+        self._test_func(+one, 1)
+
+    def test_logical_operators(self):
+        one = F.plus(1, 0)
+        two = F.plus(1, 1)
+        # &
+        self._test_func(one & two, 1)
+        self._test_func(one & two, 1)
+        self._test_func(one & 0, 0)
+        self._test_func(0 & one, 0)
+        # |
+        self._test_func(one | two, 1)
+        self._test_func(one | 0, 1)
+        self._test_func(0 | one, 1)
+        # ^
+        self._test_func(one ^ one, 0)
+        self._test_func(one ^ 0, 1)
+        self._test_func(0 ^ one, 1)
+        # ~
+        self._test_func(~one, 0)
+        self._test_func(~~one, 1)
+        # compound
+        self._test_func(one & 0 | two, 1)
+        self._test_func(one & 0 & two, 0)
+        self._test_func(one & 0 | 0, 0)
+        self._test_func((one | 0) & two, 1)
+
+    def test_date_functions(self):
+        d = date(2018, 12, 31)
+        dt = datetime(2018, 12, 31, 11, 22, 33)
+        self._test_func(F.toYear(d), 2018)
+        self._test_func(F.toYear(dt), 2018)
+        self._test_func(F.toMonth(d), 12)
+        self._test_func(F.toMonth(dt), 12)
+        self._test_func(F.toDayOfMonth(d), 31)
+        self._test_func(F.toDayOfMonth(dt), 31)
+        self._test_func(F.toDayOfWeek(d), 1)
+        self._test_func(F.toDayOfWeek(dt), 1)
+        self._test_func(F.toHour(dt), 11)
+        self._test_func(F.toMinute(dt), 22)
+        self._test_func(F.toSecond(dt), 33)
+        self._test_func(F.toMonday(d), d)
+        self._test_func(F.toMonday(dt), d)
+        self._test_func(F.toStartOfMonth(d), date(2018, 12, 1))
+        self._test_func(F.toStartOfMonth(dt), date(2018, 12, 1))
+        self._test_func(F.toStartOfQuarter(d), date(2018, 10, 1))
+        self._test_func(F.toStartOfQuarter(dt), date(2018, 10, 1))
+        self._test_func(F.toStartOfYear(d), date(2018, 1, 1))
+        self._test_func(F.toStartOfYear(dt), date(2018, 1, 1))
+        self._test_func(F.toStartOfMinute(dt), datetime(2018, 12, 31, 11, 22, 0, tzinfo=pytz.utc))
+        self._test_func(F.toStartOfFiveMinute(dt), datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc))
+        self._test_func(F.toStartOfFifteenMinutes(dt), datetime(2018, 12, 31, 11, 15, 0, tzinfo=pytz.utc))
+        self._test_func(F.toStartOfHour(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc))
+        self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc))
+        self._test_func(F.toTime(dt), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toTime(dt, 'Europe/Athens'), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toTime(dt, pytz.timezone('Europe/Athens')), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toRelativeYearNum(dt), 2018)
+        self._test_func(F.toRelativeYearNum(dt, 'Europe/Athens'), 2018)
+        self._test_func(F.toRelativeMonthNum(dt), 2018 * 12 + 12)
+        self._test_func(F.toRelativeMonthNum(dt, 'Europe/Athens'), 2018 * 12 + 12)
+        self._test_func(F.toRelativeWeekNum(dt), 2557)
+        self._test_func(F.toRelativeWeekNum(dt, 'Europe/Athens'), 2557)
+        self._test_func(F.toRelativeDayNum(dt), 17896)
+        self._test_func(F.toRelativeDayNum(dt, 'Europe/Athens'), 17896)
+        self._test_func(F.toRelativeHourNum(dt), 429515)
+        self._test_func(F.toRelativeHourNum(dt, 'Europe/Athens'), 429515)
+        self._test_func(F.toRelativeMinuteNum(dt), 25770922)
+        self._test_func(F.toRelativeMinuteNum(dt, 'Europe/Athens'), 25770922)
+        self._test_func(F.toRelativeSecondNum(dt), 1546255353)
+        self._test_func(F.toRelativeSecondNum(dt, 'Europe/Athens'), 1546255353)
+        self._test_func(F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0))
+        self._test_func(F.today(), date.today())
+        self._test_func(F.yesterday(), date.today() - timedelta(days=1))
+        self._test_func(F.timeSlot(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc))
+        self._test_func(F.timeSlots(dt, 300), [datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)])
+        self._test_func(F.formatDateTime(dt, '%D %T'), '12/31/18 11:22:33')
+        self._test_func(F.formatDateTime(dt, '%D %T', 'Europe/Athens'), '12/31/18 13:22:33')
+
+    def test_type_conversion_functions(self):
+        for f in (F.toUInt8, F.toUInt16, F.toUInt32, F.toUInt64, F.toInt8, F.toInt16, F.toInt32, F.toInt64, F.toFloat32, F.toFloat64):
+            self._test_func(f(17), 17)
+            self._test_func(f('17'), 17)
+        for f in (F.toUInt8OrZero, F.toUInt16OrZero, F.toUInt32OrZero, F.toUInt64OrZero, F.toInt8OrZero, F.toInt16OrZero, F.toInt32OrZero, F.toInt64OrZero, F.toFloat32OrZero, F.toFloat64OrZero):
+            self._test_func(f('17'), 17)
+            self._test_func(f('a'), 0)
+        for f in (F.toDecimal32, F.toDecimal64, F.toDecimal128):
+            self._test_func(f(17.17, 2), Decimal('17.17'))
+            self._test_func(f('17.17', 2), Decimal('17.17'))
+        self._test_func(F.toDate('2018-12-31'), date(2018, 12, 31))
+        self._test_func(F.toDateTime('2018-12-31 11:22:33'), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc))
+        self._test_func(F.toString(123), '123')
+        self._test_func(F.toFixedString('123', 5), '123')
+        self._test_func(F.toStringCutToZero('123\0'), '123')
+        self._test_func(F.CAST(17, 'String'), '17')
+
+    def test_string_functions(self):
+        self._test_func(F.empty(''), 1)
+        self._test_func(F.empty('x'), 0)
+        self._test_func(F.notEmpty(''), 0)
+        self._test_func(F.notEmpty('x'), 1)
+        self._test_func(F.length('x'), 1)
+        self._test_func(F.lengthUTF8('x'), 1)
+        self._test_func(F.lower('Ab'), 'ab')
+        self._test_func(F.upper('Ab'), 'AB')
+        self._test_func(F.lowerUTF8('Ab'), 'ab')
+        self._test_func(F.upperUTF8('Ab'), 'AB')
+        self._test_func(F.reverse('Ab'), 'bA')
+        self._test_func(F.reverseUTF8('Ab'), 'bA')
+        self._test_func(F.concat('Ab', 'Cd', 'Ef'), 'AbCdEf')
+        self._test_func(F.substring('123456', 3, 2), '34')
+        self._test_func(F.substringUTF8('123456', 3, 2), '34')
+        self._test_func(F.appendTrailingCharIfAbsent('Hello', '!'), 'Hello!')
+        self._test_func(F.appendTrailingCharIfAbsent('Hello!', '!'), 'Hello!')
+        self._test_func(F.convertCharset(F.convertCharset('Hello', 'latin1', 'utf16'), 'utf16', 'latin1'), 'Hello')
+
+    def test_base64_functions(self):
+        try:
+            self._test_func(F.base64Decode(F.base64Encode('Hello')), 'Hello')
+            self._test_func(F.tryBase64Decode(F.base64Encode('Hello')), 'Hello')
+            self._test_func(F.tryBase64Decode('zzz'), '')
+        except ServerError as e:
+            # ClickHouse version that doesn't support these functions
+            raise unittest.SkipTest(e.message)

tests/test_querysets.py

@@ -3,9 +3,10 @@ from __future__ import unicode_literals, print_function
 import unittest
 
 from infi.clickhouse_orm.database import Database
-from infi.clickhouse_orm.query import Q, F
+from infi.clickhouse_orm.query import Q
+from infi.clickhouse_orm.funcs import F
 from .base_test_with_data import *
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 
 from logging import getLogger
 logger = getLogger('tests')
@@ -432,136 +433,6 @@ class AggregateTestCase(TestCaseWithData):
         self.assertEqual(qs.conditions_as_sql(), 'the__next__number > 1')
 
 
-class FuncsTestCase(TestCaseWithData):
-
-    def setUp(self):
-        super(FuncsTestCase, self).setUp()
-        self.database.insert(self._sample_data())
-
-    def _test_qs(self, qs, expected_count):
-        logger.info(qs.as_sql())
-        count = 0
-        for instance in qs:
-            count += 1
-            logger.info('\t[%d]\t%s' % (count, instance.to_dict()))
-        self.assertEqual(count, expected_count)
-        self.assertEqual(qs.count(), expected_count)
-
-    def _test_func(self, func, expected_value=None):
-        sql = 'SELECT %s AS value' % func.to_sql()
-        logger.info(sql)
-        result = list(self.database.select(sql))
-        logger.info('\t==> %s', result[0].value)
-        if expected_value is not None:
-            self.assertEqual(result[0].value, expected_value)
-
-    def test_func_to_sql(self):
-        # No args
-        self.assertEqual(F('func').to_sql(), 'func()')
-        # String args
-        self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')")
-        # Numeric args
-        self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)")
-        # Date args
-        self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func('2018-12-31')")
-        # Datetime args
-        self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func('1546214400')")
-        # Boolean args
-        self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)")
-        # Null args
-        self.assertEqual(F('func', None).to_sql(), "func(NULL)")
-        # Fields as args
-        self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)")
-        # Funcs as args
-        self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))')
-        # Iterables as args
-        x = [1, 'z', F('foo', 17)]
-        for y in [x, tuple(x), iter(x)]:
-            self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)")
-        self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([[1, 2], [3, 4]])")
-
-    def test_filter_float_field(self):
-        qs = Person.objects_in(self.database)
-        # Height > 2
-        self._test_qs(qs.filter(F.greater(Person.height, 2)), 0)
-        self._test_qs(qs.filter(Person.height > 2), 0)
-        # Height > 1.61
-        self._test_qs(qs.filter(F.greater(Person.height, 1.61)), 96)
-        self._test_qs(qs.filter(Person.height > 1.61), 96)
-        # Height < 1.61
-        self._test_qs(qs.filter(F.less(Person.height, 1.61)), 4)
-        self._test_qs(qs.filter(Person.height < 1.61), 4)
-
-    def test_filter_date_field(self):
-        qs = Person.objects_in(self.database)
-        # People born on the 30th
-        self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3)
-        self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3)
-        self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3)
-        # People born on Sunday
-        self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18)
-        self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18)
-        self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18)
-        # People born on 1976-10-01
-        self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1)
-        self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 01))), 1)
-        self._test_qs(qs.filter(Person.birthday == date(1976, 10, 01)), 1)
-
-    def test_func_as_field_value(self):
-        qs = Person.objects_in(self.database)
-        self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96)
-        self._test_qs(qs.exclude(birthday=F.today()), 100)
-        self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100)
-
-    def test_comparison_operators(self):
-        one = F.plus(1, 0)
-        two = F.plus(1, 1)
-        self._test_func(one > one, 0)
-        self._test_func(two > one, 1)
-        self._test_func(one >= two, 0)
-        self._test_func(one >= one, 1)
-        self._test_func(one < one, 0)
-        self._test_func(one < two, 1)
-        self._test_func(two <= one, 0)
-        self._test_func(one <= one, 1)
-        self._test_func(one == two, 0)
-        self._test_func(one == one, 1)
-        self._test_func(one != one, 0)
-        self._test_func(one != two, 1)
-
-    def test_arithmetic_operators(self):
-        one = F.plus(1, 0)
-        two = F.plus(1, 1)
-        # +
-        self._test_func(one + two, 3)
-        self._test_func(one + 2, 3)
-        self._test_func(2 + one, 3)
-        # -
-        self._test_func(one - two, -1)
-        self._test_func(one - 2, -1)
-        self._test_func(1 - two, -1)
-        # *
-        self._test_func(one * two, 2)
-        self._test_func(one * 2, 2)
-        self._test_func(1 * two, 2)
-        # /
-        self._test_func(one / two, 0.5)
-        self._test_func(one / 2, 0.5)
-        self._test_func(1 / two, 0.5)
-        # %
-        self._test_func(one % two, 1)
-        self._test_func(one % 2, 1)
-        self._test_func(1 % two, 1)
-        # sign
-        self._test_func(-one, -1)
-        self._test_func(--one, 1)
-        self._test_func(+one, 1)
-
-
-
-
-
-
 Color = Enum('Color', u'red blue green yellow brown white black')