diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index 83b738e..c768c0c 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -8,10 +8,10 @@ from calendar import timegm from decimal import Decimal, localcontext from .utils import escape, parse_array, comma_join -from .query import F +from .funcs import F, FunctionOperatorsMixin -class Field(object): +class Field(FunctionOperatorsMixin): ''' Abstract base class for all field types. ''' @@ -99,26 +99,6 @@ class Field(object): inner_field = getattr(inner_field, 'inner_field', None) return False - # Support comparison operators (for use in querysets) - - def __lt__(self, other): - return F.less(self, other) - - def __le__(self, other): - return F.lessOrEquals(self, other) - - def __eq__(self, other): - return F.equals(self, other) - - def __ne__(self, other): - return F.notEquals(self, other) - - def __gt__(self, other): - return F.greater(self, other) - - def __ge__(self, other): - return F.greaterOrEquals(self, other) - class StringField(Field): diff --git a/src/infi/clickhouse_orm/funcs.py b/src/infi/clickhouse_orm/funcs.py new file mode 100644 index 0000000..fadda4a --- /dev/null +++ b/src/infi/clickhouse_orm/funcs.py @@ -0,0 +1,578 @@ +import six +from datetime import date, datetime, tzinfo +import functools + +from .utils import is_iterable, comma_join +from .query import Cond + + +def binary_operator(func): + """ + Decorates a function to mark it as a binary operator. + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + ret = func(*args, **kwargs) + ret.is_binary_operator = True + return ret + return wrapper + + +class FunctionOperatorsMixin(object): + """ + A mixin for implementing Python operators using F objects. + """ + + # Comparison operators + + def __lt__(self, other): + return F.less(self, other) + + def __le__(self, other): + return F.lessOrEquals(self, other) + + def __eq__(self, other): + return F.equals(self, other) + + def __ne__(self, other): + return F.notEquals(self, other) + + def __gt__(self, other): + return F.greater(self, other) + + def __ge__(self, other): + return F.greaterOrEquals(self, other) + + # Arithmetic operators + + def __add__(self, other): + return F.plus(self, other) + + def __radd__(self, other): + return F.plus(other, self) + + def __sub__(self, other): + return F.minus(self, other) + + def __rsub__(self, other): + return F.minus(other, self) + + def __mul__(self, other): + return F.multiply(self, other) + + def __rmul__(self, other): + return F.multiply(other, self) + + def __div__(self, other): + return F.divide(self, other) + + def __rdiv__(self, other): + return F.divide(other, self) + + def __mod__(self, other): + return F.modulo(self, other) + + def __rmod__(self, other): + return F.modulo(other, self) + + def __neg__(self): + return F.negate(self) + + def __pos__(self): + return self + + # Logical operators + + def __and__(self, other): + return F._and(self, other) + + def __rand__(self, other): + return F._and(other, self) + + def __or__(self, other): + return F._or(self, other) + + def __ror__(self, other): + return F._or(other, self) + + def __xor__(self, other): + return F._xor(self, other) + + def __rxor__(self, other): + return F._xor(other, self) + + def __invert__(self): + return F._not(self) + + +class F(Cond, FunctionOperatorsMixin): + """ + Represents a database function call and its arguments. + It doubles as a query condition when the function returns a boolean result. + """ + def __init__(self, name, *args): + self.name = name + self.args = args + self.is_binary_operator = False + + def to_sql(self, *args): + """ + Generates an SQL string for this function and its arguments. + For example if the function name is a symbol of a binary operator: + (2.54 * `height`) + For other functions: + gcd(12, 300) + """ + if self.is_binary_operator: + prefix = '' + sep = ' ' + self.name + ' ' + else: + prefix = self.name + sep = ', ' + arg_strs = (self.arg_to_sql(arg) for arg in self.args) + return prefix + '(' + sep.join(arg_strs) + ')' + + def arg_to_sql(self, arg): + """ + Converts a function argument to SQL string according to its type. + Supports functions, model fields, strings, dates, datetimes, booleans, + None, numbers, timezones, arrays/iterables. + """ + from .fields import Field, StringField, DateTimeField, DateField + if isinstance(arg, F): + return arg.to_sql() + if isinstance(arg, Field): + return "`%s`" % arg.name + if isinstance(arg, six.string_types): + return StringField().to_db_string(arg) + if isinstance(arg, datetime): + return "toDateTime(%s)" % DateTimeField().to_db_string(arg) + if isinstance(arg, date): + return "toDate('%s')" % arg.isoformat() + if isinstance(arg, bool): + return six.text_type(int(arg)) + if isinstance(arg, tzinfo): + return StringField().to_db_string(arg.tzname(None)) + if arg is None: + return 'NULL' + if is_iterable(arg): + return '[' + comma_join(self.arg_to_sql(x) for x in arg) + ']' + return six.text_type(arg) + + # Arithmetic functions + + @staticmethod + @binary_operator + def plus(a, b): + return F('+', a, b) + + @staticmethod + @binary_operator + def minus(a, b): + return F('-', a, b) + + @staticmethod + @binary_operator + def multiply(a, b): + return F('*', a, b) + + @staticmethod + @binary_operator + def divide(a, b): + return F('/', a, b) + + @staticmethod + def intDiv(a, b): + return F('intDiv', a, b) + + @staticmethod + def intDivOrZero(a, b): + return F('intDivOrZero', a, b) + + @staticmethod + @binary_operator + def modulo(a, b): + return F('%', a, b) + + @staticmethod + def negate(a): + return F('negate', a) + + @staticmethod + def abs(a): + return F('abs', a) + + @staticmethod + def gcd(a, b): + return F('gcd',a, b) + + @staticmethod + def lcm(a, b): + return F('lcm', a, b) + + # Comparison functions + + @staticmethod + @binary_operator + def equals(a, b): + return F('=', a, b) + + @staticmethod + @binary_operator + def notEquals(a, b): + return F('!=', a, b) + + @staticmethod + @binary_operator + def less(a, b): + return F('<', a, b) + + @staticmethod + @binary_operator + def greater(a, b): + return F('>', a, b) + + @staticmethod + @binary_operator + def lessOrEquals(a, b): + return F('<=', a, b) + + @staticmethod + @binary_operator + def greaterOrEquals(a, b): + return F('>=', a, b) + + # Logical functions (should be used as python operators: & | ^ ~) + + @staticmethod + @binary_operator + def _and(a, b): + return F('AND', a, b) + + @staticmethod + @binary_operator + def _or(a, b): + return F('OR', a, b) + + @staticmethod + def _xor(a, b): + return F('xor', a, b) + + @staticmethod + def _not(a): + return F('not', a) + + # Functions for working with dates and times + + @staticmethod + def toYear(d): + return F('toYear', d) + + @staticmethod + def toMonth(d): + return F('toMonth', d) + + @staticmethod + def toDayOfMonth(d): + return F('toDayOfMonth', d) + + @staticmethod + def toDayOfWeek(d): + return F('toDayOfWeek', d) + + @staticmethod + def toHour(d): + return F('toHour', d) + + @staticmethod + def toMinute(d): + return F('toMinute', d) + + @staticmethod + def toSecond(d): + return F('toSecond', d) + + @staticmethod + def toMonday(d): + return F('toMonday', d) + + @staticmethod + def toStartOfMonth(d): + return F('toStartOfMonth', d) + + @staticmethod + def toStartOfQuarter(d): + return F('toStartOfQuarter', d) + + @staticmethod + def toStartOfYear(d): + return F('toStartOfYear', d) + + @staticmethod + def toStartOfMinute(d): + return F('toStartOfMinute', d) + + @staticmethod + def toStartOfFiveMinute(d): + return F('toStartOfFiveMinute', d) + + @staticmethod + def toStartOfFifteenMinutes(d): + return F('toStartOfFifteenMinutes', d) + + @staticmethod + def toStartOfHour(d): + return F('toStartOfHour', d) + + @staticmethod + def toStartOfDay(d): + return F('toStartOfDay', d) + + @staticmethod + def toTime(d, timezone=''): + return F('toTime', d, timezone) + + @staticmethod + def toRelativeYearNum(d, timezone=''): + return F('toRelativeYearNum', d, timezone) + + @staticmethod + def toRelativeMonthNum(d, timezone=''): + return F('toRelativeMonthNum', d, timezone) + + @staticmethod + def toRelativeWeekNum(d, timezone=''): + return F('toRelativeWeekNum', d, timezone) + + @staticmethod + def toRelativeDayNum(d, timezone=''): + return F('toRelativeDayNum', d, timezone) + + @staticmethod + def toRelativeHourNum(d, timezone=''): + return F('toRelativeHourNum', d, timezone) + + @staticmethod + def toRelativeMinuteNum(d, timezone=''): + return F('toRelativeMinuteNum', d, timezone) + + @staticmethod + def toRelativeSecondNum(d, timezone=''): + return F('toRelativeSecondNum', d, timezone) + + @staticmethod + def now(): + return F('now') + + @staticmethod + def today(): + return F('today') + + @staticmethod + def yesterday(): + return F('yesterday') + + @staticmethod + def timeSlot(d): + return F('timeSlot', d) + + @staticmethod + def timeSlots(start_time, duration): + return F('timeSlots', start_time, F.toUInt32(duration)) + + @staticmethod + def formatDateTime(d, format, timezone=''): + return F('formatDateTime', d, format, timezone) + + # Type conversion functions + + @staticmethod + def toUInt8(x): + return F('toUInt8', x) + + @staticmethod + def toUInt16(x): + return F('toUInt16', x) + + @staticmethod + def toUInt32(x): + return F('toUInt32', x) + + @staticmethod + def toUInt64(x): + return F('toUInt64', x) + + @staticmethod + def toInt8(x): + return F('toInt8', x) + + @staticmethod + def toInt16(x): + return F('toInt16', x) + + @staticmethod + def toInt32(x): + return F('toInt32', x) + + @staticmethod + def toInt64(x): + return F('toInt64', x) + + @staticmethod + def toFloat32(x): + return F('toFloat32', x) + + @staticmethod + def toFloat64(x): + return F('toFloat64', x) + + @staticmethod + def toUInt8OrZero(x): + return F('toUInt8OrZero', x) + + @staticmethod + def toUInt16OrZero(x): + return F('toUInt16OrZero', x) + + @staticmethod + def toUInt32OrZero(x): + return F('toUInt32OrZero', x) + + @staticmethod + def toUInt64OrZero(x): + return F('toUInt64OrZero', x) + + @staticmethod + def toInt8OrZero(x): + return F('toInt8OrZero', x) + + @staticmethod + def toInt16OrZero(x): + return F('toInt16OrZero', x) + + @staticmethod + def toInt32OrZero(x): + return F('toInt32OrZero', x) + + @staticmethod + def toInt64OrZero(x): + return F('toInt64OrZero', x) + + @staticmethod + def toFloat32OrZero(x): + return F('toFloat32OrZero', x) + + @staticmethod + def toFloat64OrZero(x): + return F('toFloat64OrZero', x) + + @staticmethod + def toDecimal32(x, scale): + return F('toDecimal32', x, scale) + + @staticmethod + def toDecimal64(x, scale): + return F('toDecimal64', x, scale) + + @staticmethod + def toDecimal128(x, scale): + return F('toDecimal128', x, scale) + + @staticmethod + def toDate(x): + return F('toDate', x) + + @staticmethod + def toDateTime(x): + return F('toDateTime', x) + + @staticmethod + def toString(x): + return F('toString', x) + + @staticmethod + def toFixedString(s, length): + return F('toFixedString', s, length) + + @staticmethod + def toStringCutToZero(s): + return F('toStringCutToZero', s) + + @staticmethod + def CAST(x, type): + return F('CAST', x, type) + + # Functions for working with strings + + @staticmethod + def empty(s): + return F('empty', s) + + @staticmethod + def notEmpty(s): + return F('notEmpty', s) + + @staticmethod + def length(s): + return F('length', s) + + @staticmethod + def lengthUTF8(s): + return F('lengthUTF8', s) + + @staticmethod + def lower(s): + return F('lower', s) + + @staticmethod + def upper(s): + return F('upper', s) + + @staticmethod + def lowerUTF8(s): + return F('lowerUTF8', s) + + @staticmethod + def upperUTF8(s): + return F('upperUTF8', s) + + @staticmethod + def reverse(s): + return F('reverse', s) + + @staticmethod + def reverseUTF8(s): + return F('reverseUTF8', s) + + @staticmethod + def concat(*args): + return F('concat', *args) + + @staticmethod + def substring(s, offset, length): + return F('substring', s, offset, length) + + @staticmethod + def substringUTF8(s, offset, length): + return F('substringUTF8', s, offset, length) + + @staticmethod + def appendTrailingCharIfAbsent(s, c): + return F('appendTrailingCharIfAbsent', s, c) + + @staticmethod + def convertCharset(s, from_charset, to_charset): + return F('convertCharset', s, from_charset, to_charset) + + @staticmethod + def base64Encode(s): + return F('base64Encode', s) + + @staticmethod + def base64Decode(s): + return F('base64Decode', s) + + @staticmethod + def tryBase64Decode(s): + return F('tryBase64Decode', s) + diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py index 73a45d1..6e2c82f 100644 --- a/src/infi/clickhouse_orm/query.py +++ b/src/infi/clickhouse_orm/query.py @@ -5,8 +5,8 @@ import pytz from copy import copy from math import ceil from .engines import CollapsingMergeTree -from datetime import date, datetime -from .utils import comma_join, is_iterable +from datetime import date, datetime, tzinfo +from .utils import comma_join # TODO @@ -26,6 +26,7 @@ class Operator(object): raise NotImplementedError # pragma: no cover def _value_to_sql(self, field, value, quote=True): + from infi.clickhouse_orm.funcs import F if isinstance(value, F): return value.to_sql() return field.to_db_string(field.to_python(value, pytz.utc), quote) @@ -185,293 +186,6 @@ class FieldCond(Cond): return self._operator.to_sql(model_cls, self._field_name, self._value) -class F(Cond): - """ - Represents a database function call and its arguments. - It doubles as a query condition when the function returns a boolean result. - """ - - def __init__(self, name, *args): - self.name = name - self.args = args - - def to_sql(self, *args): - args_sql = comma_join(self.arg_to_sql(arg) for arg in self.args) - return self.name + '(' + args_sql + ')' - - def arg_to_sql(self, arg): - from .fields import Field, StringField, DateTimeField, DateField - if isinstance(arg, F): - return arg.to_sql() - if isinstance(arg, Field): - return "`%s`" % arg.name - if isinstance(arg, six.string_types): - return StringField().to_db_string(arg) - if isinstance(arg, datetime): - return DateTimeField().to_db_string(arg) - if isinstance(arg, date): - return DateField().to_db_string(arg) - if isinstance(arg, bool): - return six.text_type(int(arg)) - if arg is None: - return 'NULL' - if is_iterable(arg): - return '[' + comma_join(self.arg_to_sql(x) for x in arg) + ']' - return six.text_type(arg) - - # Support comparison operators with F objects - - def __lt__(self, other): - return F.less(self, other) - - def __le__(self, other): - return F.lessOrEquals(self, other) - - def __eq__(self, other): - return F.equals(self, other) - - def __ne__(self, other): - return F.notEquals(self, other) - - def __gt__(self, other): - return F.greater(self, other) - - def __ge__(self, other): - return F.greaterOrEquals(self, other) - - # Support arithmetic operations on F objects - - def __add__(self, other): - return F.plus(self, other) - - def __radd__(self, other): - return F.plus(other, self) - - def __sub__(self, other): - return F.minus(self, other) - - def __rsub__(self, other): - return F.minus(other, self) - - def __mul__(self, other): - return F.multiply(self, other) - - def __rmul__(self, other): - return F.multiply(other, self) - - def __div__(self, other): - return F.divide(self, other) - - def __rdiv__(self, other): - return F.divide(other, self) - - def __mod__(self, other): - return F.modulo(self, other) - - def __rmod__(self, other): - return F.modulo(other, self) - - def __neg__(self): - return F.negate(self) - - def __pos__(self): - return self - - # Arithmetic functions - - @staticmethod - def plus(a, b): - return F('plus', a, b) - - @staticmethod - def minus(a, b): - return F('minus', a, b) - - @staticmethod - def multiply(a, b): - return F('multiply', a, b) - - @staticmethod - def divide(a, b): - return F('divide', a, b) - - @staticmethod - def intDiv(a, b): - return F('intDiv', a, b) - - @staticmethod - def intDivOrZero(a, b): - return F('intDivOrZero', a, b) - - @staticmethod - def modulo(a, b): - return F('modulo', a, b) - - @staticmethod - def negate(a): - return F('negate', a) - - @staticmethod - def abs(a): - return F('abs', a) - - @staticmethod - def gcd(a, b): - return F('gcd',a, b) - - @staticmethod - def lcm(a, b): - return F('lcm', a, b) - - # Comparison functions - - @staticmethod - def equals(a, b): - return F('equals', a, b) - - @staticmethod - def notEquals(a, b): - return F('notEquals', a, b) - - @staticmethod - def less(a, b): - return F('less', a, b) - - @staticmethod - def greater(a, b): - return F('greater', a, b) - - @staticmethod - def lessOrEquals(a, b): - return F('lessOrEquals', a, b) - - @staticmethod - def greaterOrEquals(a, b): - return F('greaterOrEquals', a, b) - - # Functions for working with dates and times - - @staticmethod - def toYear(d): - return F('toYear', d) - - @staticmethod - def toMonth(d): - return F('toMonth', d) - - @staticmethod - def toDayOfMonth(d): - return F('toDayOfMonth', d) - - @staticmethod - def toDayOfWeek(d): - return F('toDayOfWeek', d) - - @staticmethod - def toHour(d): - return F('toHour', d) - - @staticmethod - def toMinute(d): - return F('toMinute', d) - - @staticmethod - def toSecond(d): - return F('toSecond', d) - - @staticmethod - def toMonday(d): - return F('toMonday', d) - - @staticmethod - def toStartOfMonth(d): - return F('toStartOfMonth', d) - - @staticmethod - def toStartOfQuarter(d): - return F('toStartOfQuarter', d) - - @staticmethod - def toStartOfYear(d): - return F('toStartOfYear', d) - - @staticmethod - def toStartOfMinute(d): - return F('toStartOfMinute', d) - - @staticmethod - def toStartOfFiveMinute(d): - return F('toStartOfFiveMinute', d) - - @staticmethod - def toStartOfFifteenMinutes(d): - return F('toStartOfFifteenMinutes', d) - - @staticmethod - def toStartOfHour(d): - return F('toStartOfHour', d) - - @staticmethod - def toStartOfDay(d): - return F('toStartOfDay', d) - - @staticmethod - def toTime(d): - return F('toTime', d) - - @staticmethod - def toRelativeYearNum(d, timezone=''): - return F('toRelativeYearNum', d, timezone) - - @staticmethod - def toRelativeMonthNum(d, timezone=''): - return F('toRelativeMonthNum', d, timezone) - - @staticmethod - def toRelativeWeekNum(d, timezone=''): - return F('toRelativeWeekNum', d, timezone) - - @staticmethod - def toRelativeDayNum(d, timezone=''): - return F('toRelativeDayNum', d, timezone) - - @staticmethod - def toRelativeHourNum(d, timezone=''): - return F('toRelativeHourNum', d, timezone) - - @staticmethod - def toRelativeMinuteNum(d, timezone=''): - return F('toRelativeMinuteNum', d, timezone) - - @staticmethod - def toRelativeSecondNum(d, timezone=''): - return F('toRelativeSecondNum', d, timezone) - - @staticmethod - def now(): - return F('now') - - @staticmethod - def today(): - return F('today') - - @staticmethod - def yesterday(d): - return F('yesterday') - - @staticmethod - def timeSlot(d): - return F('timeSlot', d) - - @staticmethod - def timeSlots(start_time, duration): - return F('timeSlots', start_time, duration) - - @staticmethod - def formatDateTime(d, format, timezone=''): - return F('formatDateTime', d, format, timezone) - - class Q(object): AND_MODE = ' AND ' @@ -542,6 +256,7 @@ class QuerySet(object): self._order_by = [] self._q = [] self._fields = model_cls.fields().keys() + self._extra = {} self._limits = None self._distinct = False self._final = False @@ -590,6 +305,8 @@ class QuerySet(object): fields = '*' if self._fields: fields = comma_join('`%s`' % field for field in self._fields) + for name, func in self._extra.items(): + fields += ', %s AS %s' % (func.to_sql(), name) ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else '' limit = '\nLIMIT %d, %d' % self._limits if self._limits else '' final = ' FINAL' if self._final else '' @@ -645,11 +362,17 @@ class QuerySet(object): qs._fields = field_names return qs + def extra(self, **kwargs): + qs = copy(self) + qs._extra = kwargs + return qs + def filter(self, *q, **filter_fields): """ Returns a copy of this queryset that includes only rows matching the conditions. Add q object to query if it specified. """ + from infi.clickhouse_orm.funcs import F qs = copy(self) qs._q = list(self._q) for arg in q: diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py index db87d62..e285f1c 100644 --- a/tests/test_decimal_fields.py +++ b/tests/test_decimal_fields.py @@ -13,15 +13,11 @@ class DecimalFieldsTest(unittest.TestCase): def setUp(self): self.database = Database('test-db') - self.database.add_setting('allow_experimental_decimal_type', 1) try: self.database.create_table(DecimalModel) except ServerError as e: - if 'Unknown setting' in e.message: - # This ClickHouse version does not support decimals yet - raise unittest.SkipTest(e.message) - else: - raise + # This ClickHouse version does not support decimals yet + raise unittest.SkipTest(e.message) def tearDown(self): self.database.drop_database() diff --git a/tests/test_funcs.py b/tests/test_funcs.py new file mode 100644 index 0000000..d3c1ab7 --- /dev/null +++ b/tests/test_funcs.py @@ -0,0 +1,261 @@ +import unittest +from .base_test_with_data import * +from .test_querysets import SampleModel +from datetime import date, datetime, tzinfo, timedelta +from infi.clickhouse_orm.database import ServerError + + +class FuncsTestCase(TestCaseWithData): + + def setUp(self): + super(FuncsTestCase, self).setUp() + self.database.insert(self._sample_data()) + + def _test_qs(self, qs, expected_count): + logger.info(qs.as_sql()) + count = 0 + for instance in qs: + count += 1 + logger.info('\t[%d]\t%s' % (count, instance.to_dict())) + self.assertEqual(count, expected_count) + self.assertEqual(qs.count(), expected_count) + + def _test_func(self, func, expected_value=None): + sql = 'SELECT %s AS value' % func.to_sql() + logger.info(sql) + result = list(self.database.select(sql)) + logger.info('\t==> %s', result[0].value) + if expected_value is not None: + self.assertEqual(result[0].value, expected_value) + + def test_func_to_sql(self): + # No args + self.assertEqual(F('func').to_sql(), 'func()') + # String args + self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')") + # Numeric args + self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)") + # Date args + self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func(toDate('2018-12-31'))") + # Datetime args + self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func(toDateTime('1546214400'))") + # Boolean args + self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)") + # Timezone args + self.assertEqual(F('func', pytz.utc).to_sql(), "func('UTC')") + self.assertEqual(F('func', pytz.timezone('Europe/Athens')).to_sql(), "func('Europe/Athens')") + # Null args + self.assertEqual(F('func', None).to_sql(), "func(NULL)") + # Fields as args + self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)") + # Funcs as args + self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))') + # Iterables as args + x = [1, 'z', F('foo', 17)] + for y in [x, tuple(x), iter(x)]: + self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)") + self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([[1, 2], [3, 4]])") + # Binary operator functions + self.assertEqual(F.plus(1, 2).to_sql(), "(1 + 2)") + self.assertEqual(F.lessOrEquals(1, 2).to_sql(), "(1 <= 2)") + + def test_filter_float_field(self): + qs = Person.objects_in(self.database) + # Height > 2 + self._test_qs(qs.filter(F.greater(Person.height, 2)), 0) + self._test_qs(qs.filter(Person.height > 2), 0) + # Height > 1.61 + self._test_qs(qs.filter(F.greater(Person.height, 1.61)), 96) + self._test_qs(qs.filter(Person.height > 1.61), 96) + # Height < 1.61 + self._test_qs(qs.filter(F.less(Person.height, 1.61)), 4) + self._test_qs(qs.filter(Person.height < 1.61), 4) + + def test_filter_date_field(self): + qs = Person.objects_in(self.database) + # People born on the 30th + self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3) + self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3) + self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3) + # People born on Sunday + self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18) + self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18) + self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18) + # People born on 1976-10-01 + self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1) + self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 01))), 1) + self._test_qs(qs.filter(Person.birthday == date(1976, 10, 01)), 1) + + def test_func_as_field_value(self): + qs = Person.objects_in(self.database) + self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96) + self._test_qs(qs.exclude(birthday=F.today()), 100) + self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100) + + def test_comparison_operators(self): + one = F.plus(1, 0) + two = F.plus(1, 1) + self._test_func(one > one, 0) + self._test_func(two > one, 1) + self._test_func(one >= two, 0) + self._test_func(one >= one, 1) + self._test_func(one < one, 0) + self._test_func(one < two, 1) + self._test_func(two <= one, 0) + self._test_func(one <= one, 1) + self._test_func(one == two, 0) + self._test_func(one == one, 1) + self._test_func(one != one, 0) + self._test_func(one != two, 1) + + def test_arithmetic_operators(self): + one = F.plus(1, 0) + two = F.plus(1, 1) + # + + self._test_func(one + two, 3) + self._test_func(one + 2, 3) + self._test_func(2 + one, 3) + # - + self._test_func(one - two, -1) + self._test_func(one - 2, -1) + self._test_func(1 - two, -1) + # * + self._test_func(one * two, 2) + self._test_func(one * 2, 2) + self._test_func(1 * two, 2) + # / + self._test_func(one / two, 0.5) + self._test_func(one / 2, 0.5) + self._test_func(1 / two, 0.5) + # % + self._test_func(one % two, 1) + self._test_func(one % 2, 1) + self._test_func(1 % two, 1) + # sign + self._test_func(-one, -1) + self._test_func(--one, 1) + self._test_func(+one, 1) + + def test_logical_operators(self): + one = F.plus(1, 0) + two = F.plus(1, 1) + # & + self._test_func(one & two, 1) + self._test_func(one & two, 1) + self._test_func(one & 0, 0) + self._test_func(0 & one, 0) + # | + self._test_func(one | two, 1) + self._test_func(one | 0, 1) + self._test_func(0 | one, 1) + # ^ + self._test_func(one ^ one, 0) + self._test_func(one ^ 0, 1) + self._test_func(0 ^ one, 1) + # ~ + self._test_func(~one, 0) + self._test_func(~~one, 1) + # compound + self._test_func(one & 0 | two, 1) + self._test_func(one & 0 & two, 0) + self._test_func(one & 0 | 0, 0) + self._test_func((one | 0) & two, 1) + + def test_date_functions(self): + d = date(2018, 12, 31) + dt = datetime(2018, 12, 31, 11, 22, 33) + self._test_func(F.toYear(d), 2018) + self._test_func(F.toYear(dt), 2018) + self._test_func(F.toMonth(d), 12) + self._test_func(F.toMonth(dt), 12) + self._test_func(F.toDayOfMonth(d), 31) + self._test_func(F.toDayOfMonth(dt), 31) + self._test_func(F.toDayOfWeek(d), 1) + self._test_func(F.toDayOfWeek(dt), 1) + self._test_func(F.toHour(dt), 11) + self._test_func(F.toMinute(dt), 22) + self._test_func(F.toSecond(dt), 33) + self._test_func(F.toMonday(d), d) + self._test_func(F.toMonday(dt), d) + self._test_func(F.toStartOfMonth(d), date(2018, 12, 1)) + self._test_func(F.toStartOfMonth(dt), date(2018, 12, 1)) + self._test_func(F.toStartOfQuarter(d), date(2018, 10, 1)) + self._test_func(F.toStartOfQuarter(dt), date(2018, 10, 1)) + self._test_func(F.toStartOfYear(d), date(2018, 1, 1)) + self._test_func(F.toStartOfYear(dt), date(2018, 1, 1)) + self._test_func(F.toStartOfMinute(dt), datetime(2018, 12, 31, 11, 22, 0, tzinfo=pytz.utc)) + self._test_func(F.toStartOfFiveMinute(dt), datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc)) + self._test_func(F.toStartOfFifteenMinutes(dt), datetime(2018, 12, 31, 11, 15, 0, tzinfo=pytz.utc)) + self._test_func(F.toStartOfHour(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)) + self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc)) + self._test_func(F.toTime(dt), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc)) + self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc)) + self._test_func(F.toTime(dt, 'Europe/Athens'), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc)) + self._test_func(F.toTime(dt, pytz.timezone('Europe/Athens')), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc)) + self._test_func(F.toRelativeYearNum(dt), 2018) + self._test_func(F.toRelativeYearNum(dt, 'Europe/Athens'), 2018) + self._test_func(F.toRelativeMonthNum(dt), 2018 * 12 + 12) + self._test_func(F.toRelativeMonthNum(dt, 'Europe/Athens'), 2018 * 12 + 12) + self._test_func(F.toRelativeWeekNum(dt), 2557) + self._test_func(F.toRelativeWeekNum(dt, 'Europe/Athens'), 2557) + self._test_func(F.toRelativeDayNum(dt), 17896) + self._test_func(F.toRelativeDayNum(dt, 'Europe/Athens'), 17896) + self._test_func(F.toRelativeHourNum(dt), 429515) + self._test_func(F.toRelativeHourNum(dt, 'Europe/Athens'), 429515) + self._test_func(F.toRelativeMinuteNum(dt), 25770922) + self._test_func(F.toRelativeMinuteNum(dt, 'Europe/Athens'), 25770922) + self._test_func(F.toRelativeSecondNum(dt), 1546255353) + self._test_func(F.toRelativeSecondNum(dt, 'Europe/Athens'), 1546255353) + self._test_func(F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0)) + self._test_func(F.today(), date.today()) + self._test_func(F.yesterday(), date.today() - timedelta(days=1)) + self._test_func(F.timeSlot(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)) + self._test_func(F.timeSlots(dt, 300), [datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)]) + self._test_func(F.formatDateTime(dt, '%D %T'), '12/31/18 11:22:33') + self._test_func(F.formatDateTime(dt, '%D %T', 'Europe/Athens'), '12/31/18 13:22:33') + + def test_type_conversion_functions(self): + for f in (F.toUInt8, F.toUInt16, F.toUInt32, F.toUInt64, F.toInt8, F.toInt16, F.toInt32, F.toInt64, F.toFloat32, F.toFloat64): + self._test_func(f(17), 17) + self._test_func(f('17'), 17) + for f in (F.toUInt8OrZero, F.toUInt16OrZero, F.toUInt32OrZero, F.toUInt64OrZero, F.toInt8OrZero, F.toInt16OrZero, F.toInt32OrZero, F.toInt64OrZero, F.toFloat32OrZero, F.toFloat64OrZero): + self._test_func(f('17'), 17) + self._test_func(f('a'), 0) + for f in (F.toDecimal32, F.toDecimal64, F.toDecimal128): + self._test_func(f(17.17, 2), Decimal('17.17')) + self._test_func(f('17.17', 2), Decimal('17.17')) + self._test_func(F.toDate('2018-12-31'), date(2018, 12, 31)) + self._test_func(F.toDateTime('2018-12-31 11:22:33'), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc)) + self._test_func(F.toString(123), '123') + self._test_func(F.toFixedString('123', 5), '123') + self._test_func(F.toStringCutToZero('123\0'), '123') + self._test_func(F.CAST(17, 'String'), '17') + + def test_string_functions(self): + self._test_func(F.empty(''), 1) + self._test_func(F.empty('x'), 0) + self._test_func(F.notEmpty(''), 0) + self._test_func(F.notEmpty('x'), 1) + self._test_func(F.length('x'), 1) + self._test_func(F.lengthUTF8('x'), 1) + self._test_func(F.lower('Ab'), 'ab') + self._test_func(F.upper('Ab'), 'AB') + self._test_func(F.lowerUTF8('Ab'), 'ab') + self._test_func(F.upperUTF8('Ab'), 'AB') + self._test_func(F.reverse('Ab'), 'bA') + self._test_func(F.reverseUTF8('Ab'), 'bA') + self._test_func(F.concat('Ab', 'Cd', 'Ef'), 'AbCdEf') + self._test_func(F.substring('123456', 3, 2), '34') + self._test_func(F.substringUTF8('123456', 3, 2), '34') + self._test_func(F.appendTrailingCharIfAbsent('Hello', '!'), 'Hello!') + self._test_func(F.appendTrailingCharIfAbsent('Hello!', '!'), 'Hello!') + self._test_func(F.convertCharset(F.convertCharset('Hello', 'latin1', 'utf16'), 'utf16', 'latin1'), 'Hello') + + def test_base64_functions(self): + try: + self._test_func(F.base64Decode(F.base64Encode('Hello')), 'Hello') + self._test_func(F.tryBase64Decode(F.base64Encode('Hello')), 'Hello') + self._test_func(F.tryBase64Decode('zzz'), '') + except ServerError as e: + # ClickHouse version that doesn't support these functions + raise unittest.SkipTest(e.message) diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 4858e03..de31c0f 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -3,9 +3,10 @@ from __future__ import unicode_literals, print_function import unittest from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.query import Q, F +from infi.clickhouse_orm.query import Q +from infi.clickhouse_orm.funcs import F from .base_test_with_data import * -from datetime import date, datetime +from datetime import date, datetime, timedelta from logging import getLogger logger = getLogger('tests') @@ -432,136 +433,6 @@ class AggregateTestCase(TestCaseWithData): self.assertEqual(qs.conditions_as_sql(), 'the__next__number > 1') -class FuncsTestCase(TestCaseWithData): - - def setUp(self): - super(FuncsTestCase, self).setUp() - self.database.insert(self._sample_data()) - - def _test_qs(self, qs, expected_count): - logger.info(qs.as_sql()) - count = 0 - for instance in qs: - count += 1 - logger.info('\t[%d]\t%s' % (count, instance.to_dict())) - self.assertEqual(count, expected_count) - self.assertEqual(qs.count(), expected_count) - - def _test_func(self, func, expected_value=None): - sql = 'SELECT %s AS value' % func.to_sql() - logger.info(sql) - result = list(self.database.select(sql)) - logger.info('\t==> %s', result[0].value) - if expected_value is not None: - self.assertEqual(result[0].value, expected_value) - - def test_func_to_sql(self): - # No args - self.assertEqual(F('func').to_sql(), 'func()') - # String args - self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')") - # Numeric args - self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)") - # Date args - self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func('2018-12-31')") - # Datetime args - self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func('1546214400')") - # Boolean args - self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)") - # Null args - self.assertEqual(F('func', None).to_sql(), "func(NULL)") - # Fields as args - self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)") - # Funcs as args - self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))') - # Iterables as args - x = [1, 'z', F('foo', 17)] - for y in [x, tuple(x), iter(x)]: - self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)") - self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([[1, 2], [3, 4]])") - - def test_filter_float_field(self): - qs = Person.objects_in(self.database) - # Height > 2 - self._test_qs(qs.filter(F.greater(Person.height, 2)), 0) - self._test_qs(qs.filter(Person.height > 2), 0) - # Height > 1.61 - self._test_qs(qs.filter(F.greater(Person.height, 1.61)), 96) - self._test_qs(qs.filter(Person.height > 1.61), 96) - # Height < 1.61 - self._test_qs(qs.filter(F.less(Person.height, 1.61)), 4) - self._test_qs(qs.filter(Person.height < 1.61), 4) - - def test_filter_date_field(self): - qs = Person.objects_in(self.database) - # People born on the 30th - self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3) - self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3) - self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3) - # People born on Sunday - self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18) - self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18) - self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18) - # People born on 1976-10-01 - self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1) - self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 01))), 1) - self._test_qs(qs.filter(Person.birthday == date(1976, 10, 01)), 1) - - def test_func_as_field_value(self): - qs = Person.objects_in(self.database) - self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96) - self._test_qs(qs.exclude(birthday=F.today()), 100) - self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100) - - def test_comparison_operators(self): - one = F.plus(1, 0) - two = F.plus(1, 1) - self._test_func(one > one, 0) - self._test_func(two > one, 1) - self._test_func(one >= two, 0) - self._test_func(one >= one, 1) - self._test_func(one < one, 0) - self._test_func(one < two, 1) - self._test_func(two <= one, 0) - self._test_func(one <= one, 1) - self._test_func(one == two, 0) - self._test_func(one == one, 1) - self._test_func(one != one, 0) - self._test_func(one != two, 1) - - def test_arithmetic_operators(self): - one = F.plus(1, 0) - two = F.plus(1, 1) - # + - self._test_func(one + two, 3) - self._test_func(one + 2, 3) - self._test_func(2 + one, 3) - # - - self._test_func(one - two, -1) - self._test_func(one - 2, -1) - self._test_func(1 - two, -1) - # * - self._test_func(one * two, 2) - self._test_func(one * 2, 2) - self._test_func(1 * two, 2) - # / - self._test_func(one / two, 0.5) - self._test_func(one / 2, 0.5) - self._test_func(1 / two, 0.5) - # % - self._test_func(one % two, 1) - self._test_func(one % 2, 1) - self._test_func(1 % two, 1) - # sign - self._test_func(-one, -1) - self._test_func(--one, 1) - self._test_func(+one, 1) - - - - - - Color = Enum('Color', u'red blue green yellow brown white black')