Infinidat/infi.clickhouse_orm
1
1
Fork 0
mirror of https://github.com/Infinidat/infi.clickhouse_orm.git synced 2024-11-22 00:56:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Functions WIP

Browse Source
This commit is contained in:
Itai Shirav 2019-01-28 09:51:53 +02:00
parent 602d0671f1
commit f96bd22c38
6 changed files with 858 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/infi/clickhouse_orm/fields.py
View File

@ -8,10 +8,10 @@ from calendar import timegm
from decimal import Decimal, localcontext
from .utils import escape, parse_array, comma_join
from .query import F
from .funcs import F, FunctionOperatorsMixin
class Field(object):
class Field(FunctionOperatorsMixin):
'''
Abstract base class for all field types.
'''
@ -99,26 +99,6 @@ class Field(object):
inner_field = getattr(inner_field, 'inner_field', None)
return False
# Support comparison operators (for use in querysets)
def __lt__(self, other):
return F.less(self, other)
def __le__(self, other):
return F.lessOrEquals(self, other)
def __eq__(self, other):
return F.equals(self, other)
def __ne__(self, other):
return F.notEquals(self, other)
def __gt__(self, other):
return F.greater(self, other)
def __ge__(self, other):
return F.greaterOrEquals(self, other)
class StringField(Field):

578
src/infi/clickhouse_orm/funcs.py Normal file
View File

@ -0,0 +1,578 @@
import six
from datetime import date, datetime, tzinfo
import functools
from .utils import is_iterable, comma_join
from .query import Cond
def binary_operator(func):
"""
Decorates a function to mark it as a binary operator.
"""
@functools.wraps(func)
def wrapper(*args, **kwargs):
ret = func(*args, **kwargs)
ret.is_binary_operator = True
return ret
return wrapper
class FunctionOperatorsMixin(object):
"""
A mixin for implementing Python operators using F objects.
"""
# Comparison operators
def __lt__(self, other):
return F.less(self, other)
def __le__(self, other):
return F.lessOrEquals(self, other)
def __eq__(self, other):
return F.equals(self, other)
def __ne__(self, other):
return F.notEquals(self, other)
def __gt__(self, other):
return F.greater(self, other)
def __ge__(self, other):
return F.greaterOrEquals(self, other)
# Arithmetic operators
def __add__(self, other):
return F.plus(self, other)
def __radd__(self, other):
return F.plus(other, self)
def __sub__(self, other):
return F.minus(self, other)
def __rsub__(self, other):
return F.minus(other, self)
def __mul__(self, other):
return F.multiply(self, other)
def __rmul__(self, other):
return F.multiply(other, self)
def __div__(self, other):
return F.divide(self, other)
def __rdiv__(self, other):
return F.divide(other, self)
def __mod__(self, other):
return F.modulo(self, other)
def __rmod__(self, other):
return F.modulo(other, self)
def __neg__(self):
return F.negate(self)
def __pos__(self):
return self
# Logical operators
def __and__(self, other):
return F._and(self, other)
def __rand__(self, other):
return F._and(other, self)
def __or__(self, other):
return F._or(self, other)
def __ror__(self, other):
return F._or(other, self)
def __xor__(self, other):
return F._xor(self, other)
def __rxor__(self, other):
return F._xor(other, self)
def __invert__(self):
return F._not(self)
class F(Cond, FunctionOperatorsMixin):
"""
Represents a database function call and its arguments.
It doubles as a query condition when the function returns a boolean result.
"""
def __init__(self, name, *args):
self.name = name
self.args = args
self.is_binary_operator = False
def to_sql(self, *args):
"""
Generates an SQL string for this function and its arguments.
For example if the function name is a symbol of a binary operator:
(2.54 * `height`)
For other functions:
gcd(12, 300)
"""
if self.is_binary_operator:
prefix = ''
sep = ' ' + self.name + ' '
else:
prefix = self.name
sep = ', '
arg_strs = (self.arg_to_sql(arg) for arg in self.args)
return prefix + '(' + sep.join(arg_strs) + ')'
def arg_to_sql(self, arg):
"""
Converts a function argument to SQL string according to its type.
Supports functions, model fields, strings, dates, datetimes, booleans,
None, numbers, timezones, arrays/iterables.
"""
from .fields import Field, StringField, DateTimeField, DateField
if isinstance(arg, F):
return arg.to_sql()
if isinstance(arg, Field):
return "`%s`" % arg.name
if isinstance(arg, six.string_types):
return StringField().to_db_string(arg)
if isinstance(arg, datetime):
return "toDateTime(%s)" % DateTimeField().to_db_string(arg)
if isinstance(arg, date):
return "toDate('%s')" % arg.isoformat()
if isinstance(arg, bool):
return six.text_type(int(arg))
if isinstance(arg, tzinfo):
return StringField().to_db_string(arg.tzname(None))
if arg is None:
return 'NULL'
if is_iterable(arg):
return '[' + comma_join(self.arg_to_sql(x) for x in arg) + ']'
return six.text_type(arg)
# Arithmetic functions
@staticmethod
@binary_operator
def plus(a, b):
return F('+', a, b)
@staticmethod
@binary_operator
def minus(a, b):
return F('-', a, b)
@staticmethod
@binary_operator
def multiply(a, b):
return F('*', a, b)
@staticmethod
@binary_operator
def divide(a, b):
return F('/', a, b)
@staticmethod
def intDiv(a, b):
return F('intDiv', a, b)
@staticmethod
def intDivOrZero(a, b):
return F('intDivOrZero', a, b)
@staticmethod
@binary_operator
def modulo(a, b):
return F('%', a, b)
@staticmethod
def negate(a):
return F('negate', a)
@staticmethod
def abs(a):
return F('abs', a)
@staticmethod
def gcd(a, b):
return F('gcd',a, b)
@staticmethod
def lcm(a, b):
return F('lcm', a, b)
# Comparison functions
@staticmethod
@binary_operator
def equals(a, b):
return F('=', a, b)
@staticmethod
@binary_operator
def notEquals(a, b):
return F('!=', a, b)
@staticmethod
@binary_operator
def less(a, b):
return F('<', a, b)
@staticmethod
@binary_operator
def greater(a, b):
return F('>', a, b)
@staticmethod
@binary_operator
def lessOrEquals(a, b):
return F('<=', a, b)
@staticmethod
@binary_operator
def greaterOrEquals(a, b):
return F('>=', a, b)
# Logical functions (should be used as python operators: & | ^ ~)
@staticmethod
@binary_operator
def _and(a, b):
return F('AND', a, b)
@staticmethod
@binary_operator
def _or(a, b):
return F('OR', a, b)
@staticmethod
def _xor(a, b):
return F('xor', a, b)
@staticmethod
def _not(a):
return F('not', a)
# Functions for working with dates and times
@staticmethod
def toYear(d):
return F('toYear', d)
@staticmethod
def toMonth(d):
return F('toMonth', d)
@staticmethod
def toDayOfMonth(d):
return F('toDayOfMonth', d)
@staticmethod
def toDayOfWeek(d):
return F('toDayOfWeek', d)
@staticmethod
def toHour(d):
return F('toHour', d)
@staticmethod
def toMinute(d):
return F('toMinute', d)
@staticmethod
def toSecond(d):
return F('toSecond', d)
@staticmethod
def toMonday(d):
return F('toMonday', d)
@staticmethod
def toStartOfMonth(d):
return F('toStartOfMonth', d)
@staticmethod
def toStartOfQuarter(d):
return F('toStartOfQuarter', d)
@staticmethod
def toStartOfYear(d):
return F('toStartOfYear', d)
@staticmethod
def toStartOfMinute(d):
return F('toStartOfMinute', d)
@staticmethod
def toStartOfFiveMinute(d):
return F('toStartOfFiveMinute', d)
@staticmethod
def toStartOfFifteenMinutes(d):
return F('toStartOfFifteenMinutes', d)
@staticmethod
def toStartOfHour(d):
return F('toStartOfHour', d)
@staticmethod
def toStartOfDay(d):
return F('toStartOfDay', d)
@staticmethod
def toTime(d, timezone=''):
return F('toTime', d, timezone)
@staticmethod
def toRelativeYearNum(d, timezone=''):
return F('toRelativeYearNum', d, timezone)
@staticmethod
def toRelativeMonthNum(d, timezone=''):
return F('toRelativeMonthNum', d, timezone)
@staticmethod
def toRelativeWeekNum(d, timezone=''):
return F('toRelativeWeekNum', d, timezone)
@staticmethod
def toRelativeDayNum(d, timezone=''):
return F('toRelativeDayNum', d, timezone)
@staticmethod
def toRelativeHourNum(d, timezone=''):
return F('toRelativeHourNum', d, timezone)
@staticmethod
def toRelativeMinuteNum(d, timezone=''):
return F('toRelativeMinuteNum', d, timezone)
@staticmethod
def toRelativeSecondNum(d, timezone=''):
return F('toRelativeSecondNum', d, timezone)
@staticmethod
def now():
return F('now')
@staticmethod
def today():
return F('today')
@staticmethod
def yesterday():
return F('yesterday')
@staticmethod
def timeSlot(d):
return F('timeSlot', d)
@staticmethod
def timeSlots(start_time, duration):
return F('timeSlots', start_time, F.toUInt32(duration))
@staticmethod
def formatDateTime(d, format, timezone=''):
return F('formatDateTime', d, format, timezone)
# Type conversion functions
@staticmethod
def toUInt8(x):
return F('toUInt8', x)
@staticmethod
def toUInt16(x):
return F('toUInt16', x)
@staticmethod
def toUInt32(x):
return F('toUInt32', x)
@staticmethod
def toUInt64(x):
return F('toUInt64', x)
@staticmethod
def toInt8(x):
return F('toInt8', x)
@staticmethod
def toInt16(x):
return F('toInt16', x)
@staticmethod
def toInt32(x):
return F('toInt32', x)
@staticmethod
def toInt64(x):
return F('toInt64', x)
@staticmethod
def toFloat32(x):
return F('toFloat32', x)
@staticmethod
def toFloat64(x):
return F('toFloat64', x)
@staticmethod
def toUInt8OrZero(x):
return F('toUInt8OrZero', x)
@staticmethod
def toUInt16OrZero(x):
return F('toUInt16OrZero', x)
@staticmethod
def toUInt32OrZero(x):
return F('toUInt32OrZero', x)
@staticmethod
def toUInt64OrZero(x):
return F('toUInt64OrZero', x)
@staticmethod
def toInt8OrZero(x):
return F('toInt8OrZero', x)
@staticmethod
def toInt16OrZero(x):
return F('toInt16OrZero', x)
@staticmethod
def toInt32OrZero(x):
return F('toInt32OrZero', x)
@staticmethod
def toInt64OrZero(x):
return F('toInt64OrZero', x)
@staticmethod
def toFloat32OrZero(x):
return F('toFloat32OrZero', x)
@staticmethod
def toFloat64OrZero(x):
return F('toFloat64OrZero', x)
@staticmethod
def toDecimal32(x, scale):
return F('toDecimal32', x, scale)
@staticmethod
def toDecimal64(x, scale):
return F('toDecimal64', x, scale)
@staticmethod
def toDecimal128(x, scale):
return F('toDecimal128', x, scale)
@staticmethod
def toDate(x):
return F('toDate', x)
@staticmethod
def toDateTime(x):
return F('toDateTime', x)
@staticmethod
def toString(x):
return F('toString', x)
@staticmethod
def toFixedString(s, length):
return F('toFixedString', s, length)
@staticmethod
def toStringCutToZero(s):
return F('toStringCutToZero', s)
@staticmethod
def CAST(x, type):
return F('CAST', x, type)
# Functions for working with strings
@staticmethod
def empty(s):
return F('empty', s)
@staticmethod
def notEmpty(s):
return F('notEmpty', s)
@staticmethod
def length(s):
return F('length', s)
@staticmethod
def lengthUTF8(s):
return F('lengthUTF8', s)
@staticmethod
def lower(s):
return F('lower', s)
@staticmethod
def upper(s):
return F('upper', s)
@staticmethod
def lowerUTF8(s):
return F('lowerUTF8', s)
@staticmethod
def upperUTF8(s):
return F('upperUTF8', s)
@staticmethod
def reverse(s):
return F('reverse', s)
@staticmethod
def reverseUTF8(s):
return F('reverseUTF8', s)
@staticmethod
def concat(*args):
return F('concat', *args)
@staticmethod
def substring(s, offset, length):
return F('substring', s, offset, length)
@staticmethod
def substringUTF8(s, offset, length):
return F('substringUTF8', s, offset, length)
@staticmethod
def appendTrailingCharIfAbsent(s, c):
return F('appendTrailingCharIfAbsent', s, c)
@staticmethod
def convertCharset(s, from_charset, to_charset):
return F('convertCharset', s, from_charset, to_charset)
@staticmethod
def base64Encode(s):
return F('base64Encode', s)
@staticmethod
def base64Decode(s):
return F('base64Decode', s)
@staticmethod
def tryBase64Decode(s):
return F('tryBase64Decode', s)

301
src/infi/clickhouse_orm/query.py
View File

@ -5,8 +5,8 @@ import pytz
from copy import copy
from math import ceil
from .engines import CollapsingMergeTree
from datetime import date, datetime
from .utils import comma_join, is_iterable
from datetime import date, datetime, tzinfo
from .utils import comma_join
# TODO
@ -26,6 +26,7 @@ class Operator(object):
raise NotImplementedError # pragma: no cover
def _value_to_sql(self, field, value, quote=True):
from infi.clickhouse_orm.funcs import F
if isinstance(value, F):
return value.to_sql()
return field.to_db_string(field.to_python(value, pytz.utc), quote)
@ -185,293 +186,6 @@ class FieldCond(Cond):
return self._operator.to_sql(model_cls, self._field_name, self._value)
class F(Cond):
"""
Represents a database function call and its arguments.
It doubles as a query condition when the function returns a boolean result.
"""
def __init__(self, name, *args):
self.name = name
self.args = args
def to_sql(self, *args):
args_sql = comma_join(self.arg_to_sql(arg) for arg in self.args)
return self.name + '(' + args_sql + ')'
def arg_to_sql(self, arg):
from .fields import Field, StringField, DateTimeField, DateField
if isinstance(arg, F):
return arg.to_sql()
if isinstance(arg, Field):
return "`%s`" % arg.name
if isinstance(arg, six.string_types):
return StringField().to_db_string(arg)
if isinstance(arg, datetime):
return DateTimeField().to_db_string(arg)
if isinstance(arg, date):
return DateField().to_db_string(arg)
if isinstance(arg, bool):
return six.text_type(int(arg))
if arg is None:
return 'NULL'
if is_iterable(arg):
return '[' + comma_join(self.arg_to_sql(x) for x in arg) + ']'
return six.text_type(arg)
# Support comparison operators with F objects
def __lt__(self, other):
return F.less(self, other)
def __le__(self, other):
return F.lessOrEquals(self, other)
def __eq__(self, other):
return F.equals(self, other)
def __ne__(self, other):
return F.notEquals(self, other)
def __gt__(self, other):
return F.greater(self, other)
def __ge__(self, other):
return F.greaterOrEquals(self, other)
# Support arithmetic operations on F objects
def __add__(self, other):
return F.plus(self, other)
def __radd__(self, other):
return F.plus(other, self)
def __sub__(self, other):
return F.minus(self, other)
def __rsub__(self, other):
return F.minus(other, self)
def __mul__(self, other):
return F.multiply(self, other)
def __rmul__(self, other):
return F.multiply(other, self)
def __div__(self, other):
return F.divide(self, other)
def __rdiv__(self, other):
return F.divide(other, self)
def __mod__(self, other):
return F.modulo(self, other)
def __rmod__(self, other):
return F.modulo(other, self)
def __neg__(self):
return F.negate(self)
def __pos__(self):
return self
# Arithmetic functions
@staticmethod
def plus(a, b):
return F('plus', a, b)
@staticmethod
def minus(a, b):
return F('minus', a, b)
@staticmethod
def multiply(a, b):
return F('multiply', a, b)
@staticmethod
def divide(a, b):
return F('divide', a, b)
@staticmethod
def intDiv(a, b):
return F('intDiv', a, b)
@staticmethod
def intDivOrZero(a, b):
return F('intDivOrZero', a, b)
@staticmethod
def modulo(a, b):
return F('modulo', a, b)
@staticmethod
def negate(a):
return F('negate', a)
@staticmethod
def abs(a):
return F('abs', a)
@staticmethod
def gcd(a, b):
return F('gcd',a, b)
@staticmethod
def lcm(a, b):
return F('lcm', a, b)
# Comparison functions
@staticmethod
def equals(a, b):
return F('equals', a, b)
@staticmethod
def notEquals(a, b):
return F('notEquals', a, b)
@staticmethod
def less(a, b):
return F('less', a, b)
@staticmethod
def greater(a, b):
return F('greater', a, b)
@staticmethod
def lessOrEquals(a, b):
return F('lessOrEquals', a, b)
@staticmethod
def greaterOrEquals(a, b):
return F('greaterOrEquals', a, b)
# Functions for working with dates and times
@staticmethod
def toYear(d):
return F('toYear', d)
@staticmethod
def toMonth(d):
return F('toMonth', d)
@staticmethod
def toDayOfMonth(d):
return F('toDayOfMonth', d)
@staticmethod
def toDayOfWeek(d):
return F('toDayOfWeek', d)
@staticmethod
def toHour(d):
return F('toHour', d)
@staticmethod
def toMinute(d):
return F('toMinute', d)
@staticmethod
def toSecond(d):
return F('toSecond', d)
@staticmethod
def toMonday(d):
return F('toMonday', d)
@staticmethod
def toStartOfMonth(d):
return F('toStartOfMonth', d)
@staticmethod
def toStartOfQuarter(d):
return F('toStartOfQuarter', d)
@staticmethod
def toStartOfYear(d):
return F('toStartOfYear', d)
@staticmethod
def toStartOfMinute(d):
return F('toStartOfMinute', d)
@staticmethod
def toStartOfFiveMinute(d):
return F('toStartOfFiveMinute', d)
@staticmethod
def toStartOfFifteenMinutes(d):
return F('toStartOfFifteenMinutes', d)
@staticmethod
def toStartOfHour(d):
return F('toStartOfHour', d)
@staticmethod
def toStartOfDay(d):
return F('toStartOfDay', d)
@staticmethod
def toTime(d):
return F('toTime', d)
@staticmethod
def toRelativeYearNum(d, timezone=''):
return F('toRelativeYearNum', d, timezone)
@staticmethod
def toRelativeMonthNum(d, timezone=''):
return F('toRelativeMonthNum', d, timezone)
@staticmethod
def toRelativeWeekNum(d, timezone=''):
return F('toRelativeWeekNum', d, timezone)
@staticmethod
def toRelativeDayNum(d, timezone=''):
return F('toRelativeDayNum', d, timezone)
@staticmethod
def toRelativeHourNum(d, timezone=''):
return F('toRelativeHourNum', d, timezone)
@staticmethod
def toRelativeMinuteNum(d, timezone=''):
return F('toRelativeMinuteNum', d, timezone)
@staticmethod
def toRelativeSecondNum(d, timezone=''):
return F('toRelativeSecondNum', d, timezone)
@staticmethod
def now():
return F('now')
@staticmethod
def today():
return F('today')
@staticmethod
def yesterday(d):
return F('yesterday')
@staticmethod
def timeSlot(d):
return F('timeSlot', d)
@staticmethod
def timeSlots(start_time, duration):
return F('timeSlots', start_time, duration)
@staticmethod
def formatDateTime(d, format, timezone=''):
return F('formatDateTime', d, format, timezone)
class Q(object):
AND_MODE = ' AND '
@ -542,6 +256,7 @@ class QuerySet(object):
self._order_by = []
self._q = []
self._fields = model_cls.fields().keys()
self._extra = {}
self._limits = None
self._distinct = False
self._final = False
@ -590,6 +305,8 @@ class QuerySet(object):
fields = '*'
if self._fields:
fields = comma_join('`%s`' % field for field in self._fields)
for name, func in self._extra.items():
fields += ', %s AS %s' % (func.to_sql(), name)
ordering = '\nORDER BY ' + self.order_by_as_sql() if self._order_by else ''
limit = '\nLIMIT %d, %d' % self._limits if self._limits else ''
final = ' FINAL' if self._final else ''
@ -645,11 +362,17 @@ class QuerySet(object):
qs._fields = field_names
return qs
def extra(self, **kwargs):
qs = copy(self)
qs._extra = kwargs
return qs
def filter(self, *q, **filter_fields):
"""
Returns a copy of this queryset that includes only rows matching the conditions.
Add q object to query if it specified.
"""
from infi.clickhouse_orm.funcs import F
qs = copy(self)
qs._q = list(self._q)
for arg in q:

4
tests/test_decimal_fields.py
View File

@ -13,15 +13,11 @@ class DecimalFieldsTest(unittest.TestCase):
def setUp(self):
self.database = Database('test-db')
self.database.add_setting('allow_experimental_decimal_type', 1)
try:
self.database.create_table(DecimalModel)
except ServerError as e:
if 'Unknown setting' in e.message:
# This ClickHouse version does not support decimals yet
raise unittest.SkipTest(e.message)
else:
raise
def tearDown(self):
self.database.drop_database()

261
tests/test_funcs.py Normal file
View File

@ -0,0 +1,261 @@
import unittest
from .base_test_with_data import *
from .test_querysets import SampleModel
from datetime import date, datetime, tzinfo, timedelta
from infi.clickhouse_orm.database import ServerError
class FuncsTestCase(TestCaseWithData):
def setUp(self):
super(FuncsTestCase, self).setUp()
self.database.insert(self._sample_data())
def _test_qs(self, qs, expected_count):
logger.info(qs.as_sql())
count = 0
for instance in qs:
count += 1
logger.info('\t[%d]\t%s' % (count, instance.to_dict()))
self.assertEqual(count, expected_count)
self.assertEqual(qs.count(), expected_count)
def _test_func(self, func, expected_value=None):
sql = 'SELECT %s AS value' % func.to_sql()
logger.info(sql)
result = list(self.database.select(sql))
logger.info('\t==> %s', result[0].value)
if expected_value is not None:
self.assertEqual(result[0].value, expected_value)
def test_func_to_sql(self):
# No args
self.assertEqual(F('func').to_sql(), 'func()')
# String args
self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')")
# Numeric args
self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)")
# Date args
self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func(toDate('2018-12-31'))")
# Datetime args
self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func(toDateTime('1546214400'))")
# Boolean args
self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)")
# Timezone args
self.assertEqual(F('func', pytz.utc).to_sql(), "func('UTC')")
self.assertEqual(F('func', pytz.timezone('Europe/Athens')).to_sql(), "func('Europe/Athens')")
# Null args
self.assertEqual(F('func', None).to_sql(), "func(NULL)")
# Fields as args
self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)")
# Funcs as args
self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))')
# Iterables as args
x = [1, 'z', F('foo', 17)]
for y in [x, tuple(x), iter(x)]:
self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)")
self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([[1, 2], [3, 4]])")
# Binary operator functions
self.assertEqual(F.plus(1, 2).to_sql(), "(1 + 2)")
self.assertEqual(F.lessOrEquals(1, 2).to_sql(), "(1 <= 2)")
def test_filter_float_field(self):
qs = Person.objects_in(self.database)
# Height > 2
self._test_qs(qs.filter(F.greater(Person.height, 2)), 0)
self._test_qs(qs.filter(Person.height > 2), 0)
# Height > 1.61
self._test_qs(qs.filter(F.greater(Person.height, 1.61)), 96)
self._test_qs(qs.filter(Person.height > 1.61), 96)
# Height < 1.61
self._test_qs(qs.filter(F.less(Person.height, 1.61)), 4)
self._test_qs(qs.filter(Person.height < 1.61), 4)
def test_filter_date_field(self):
qs = Person.objects_in(self.database)
# People born on the 30th
self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3)
self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3)
self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3)
# People born on Sunday
self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18)
self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18)
self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18)
# People born on 1976-10-01
self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1)
self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 01))), 1)
self._test_qs(qs.filter(Person.birthday == date(1976, 10, 01)), 1)
def test_func_as_field_value(self):
qs = Person.objects_in(self.database)
self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96)
self._test_qs(qs.exclude(birthday=F.today()), 100)
self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100)
def test_comparison_operators(self):
one = F.plus(1, 0)
two = F.plus(1, 1)
self._test_func(one > one, 0)
self._test_func(two > one, 1)
self._test_func(one >= two, 0)
self._test_func(one >= one, 1)
self._test_func(one < one, 0)
self._test_func(one < two, 1)
self._test_func(two <= one, 0)
self._test_func(one <= one, 1)
self._test_func(one == two, 0)
self._test_func(one == one, 1)
self._test_func(one != one, 0)
self._test_func(one != two, 1)
def test_arithmetic_operators(self):
one = F.plus(1, 0)
two = F.plus(1, 1)
# +
self._test_func(one + two, 3)
self._test_func(one + 2, 3)
self._test_func(2 + one, 3)
# -
self._test_func(one - two, -1)
self._test_func(one - 2, -1)
self._test_func(1 - two, -1)
# *
self._test_func(one * two, 2)
self._test_func(one * 2, 2)
self._test_func(1 * two, 2)
# /
self._test_func(one / two, 0.5)
self._test_func(one / 2, 0.5)
self._test_func(1 / two, 0.5)
# %
self._test_func(one % two, 1)
self._test_func(one % 2, 1)
self._test_func(1 % two, 1)
# sign
self._test_func(-one, -1)
self._test_func(--one, 1)
self._test_func(+one, 1)
def test_logical_operators(self):
one = F.plus(1, 0)
two = F.plus(1, 1)
# &
self._test_func(one & two, 1)
self._test_func(one & two, 1)
self._test_func(one & 0, 0)
self._test_func(0 & one, 0)
# |
self._test_func(one | two, 1)
self._test_func(one | 0, 1)
self._test_func(0 | one, 1)
# ^
self._test_func(one ^ one, 0)
self._test_func(one ^ 0, 1)
self._test_func(0 ^ one, 1)
# ~
self._test_func(~one, 0)
self._test_func(~~one, 1)
# compound
self._test_func(one & 0 | two, 1)
self._test_func(one & 0 & two, 0)
self._test_func(one & 0 | 0, 0)
self._test_func((one | 0) & two, 1)
def test_date_functions(self):
d = date(2018, 12, 31)
dt = datetime(2018, 12, 31, 11, 22, 33)
self._test_func(F.toYear(d), 2018)
self._test_func(F.toYear(dt), 2018)
self._test_func(F.toMonth(d), 12)
self._test_func(F.toMonth(dt), 12)
self._test_func(F.toDayOfMonth(d), 31)
self._test_func(F.toDayOfMonth(dt), 31)
self._test_func(F.toDayOfWeek(d), 1)
self._test_func(F.toDayOfWeek(dt), 1)
self._test_func(F.toHour(dt), 11)
self._test_func(F.toMinute(dt), 22)
self._test_func(F.toSecond(dt), 33)
self._test_func(F.toMonday(d), d)
self._test_func(F.toMonday(dt), d)
self._test_func(F.toStartOfMonth(d), date(2018, 12, 1))
self._test_func(F.toStartOfMonth(dt), date(2018, 12, 1))
self._test_func(F.toStartOfQuarter(d), date(2018, 10, 1))
self._test_func(F.toStartOfQuarter(dt), date(2018, 10, 1))
self._test_func(F.toStartOfYear(d), date(2018, 1, 1))
self._test_func(F.toStartOfYear(dt), date(2018, 1, 1))
self._test_func(F.toStartOfMinute(dt), datetime(2018, 12, 31, 11, 22, 0, tzinfo=pytz.utc))
self._test_func(F.toStartOfFiveMinute(dt), datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc))
self._test_func(F.toStartOfFifteenMinutes(dt), datetime(2018, 12, 31, 11, 15, 0, tzinfo=pytz.utc))
self._test_func(F.toStartOfHour(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc))
self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc))
self._test_func(F.toTime(dt), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, 'Europe/Athens'), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toTime(dt, pytz.timezone('Europe/Athens')), datetime(1970, 1, 2, 13, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toRelativeYearNum(dt), 2018)
self._test_func(F.toRelativeYearNum(dt, 'Europe/Athens'), 2018)
self._test_func(F.toRelativeMonthNum(dt), 2018 * 12 + 12)
self._test_func(F.toRelativeMonthNum(dt, 'Europe/Athens'), 2018 * 12 + 12)
self._test_func(F.toRelativeWeekNum(dt), 2557)
self._test_func(F.toRelativeWeekNum(dt, 'Europe/Athens'), 2557)
self._test_func(F.toRelativeDayNum(dt), 17896)
self._test_func(F.toRelativeDayNum(dt, 'Europe/Athens'), 17896)
self._test_func(F.toRelativeHourNum(dt), 429515)
self._test_func(F.toRelativeHourNum(dt, 'Europe/Athens'), 429515)
self._test_func(F.toRelativeMinuteNum(dt), 25770922)
self._test_func(F.toRelativeMinuteNum(dt, 'Europe/Athens'), 25770922)
self._test_func(F.toRelativeSecondNum(dt), 1546255353)
self._test_func(F.toRelativeSecondNum(dt, 'Europe/Athens'), 1546255353)
self._test_func(F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0))
self._test_func(F.today(), date.today())
self._test_func(F.yesterday(), date.today() - timedelta(days=1))
self._test_func(F.timeSlot(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc))
self._test_func(F.timeSlots(dt, 300), [datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)])
self._test_func(F.formatDateTime(dt, '%D %T'), '12/31/18 11:22:33')
self._test_func(F.formatDateTime(dt, '%D %T', 'Europe/Athens'), '12/31/18 13:22:33')
def test_type_conversion_functions(self):
for f in (F.toUInt8, F.toUInt16, F.toUInt32, F.toUInt64, F.toInt8, F.toInt16, F.toInt32, F.toInt64, F.toFloat32, F.toFloat64):
self._test_func(f(17), 17)
self._test_func(f('17'), 17)
for f in (F.toUInt8OrZero, F.toUInt16OrZero, F.toUInt32OrZero, F.toUInt64OrZero, F.toInt8OrZero, F.toInt16OrZero, F.toInt32OrZero, F.toInt64OrZero, F.toFloat32OrZero, F.toFloat64OrZero):
self._test_func(f('17'), 17)
self._test_func(f('a'), 0)
for f in (F.toDecimal32, F.toDecimal64, F.toDecimal128):
self._test_func(f(17.17, 2), Decimal('17.17'))
self._test_func(f('17.17', 2), Decimal('17.17'))
self._test_func(F.toDate('2018-12-31'), date(2018, 12, 31))
self._test_func(F.toDateTime('2018-12-31 11:22:33'), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc))
self._test_func(F.toString(123), '123')
self._test_func(F.toFixedString('123', 5), '123')
self._test_func(F.toStringCutToZero('123\0'), '123')
self._test_func(F.CAST(17, 'String'), '17')
def test_string_functions(self):
self._test_func(F.empty(''), 1)
self._test_func(F.empty('x'), 0)
self._test_func(F.notEmpty(''), 0)
self._test_func(F.notEmpty('x'), 1)
self._test_func(F.length('x'), 1)
self._test_func(F.lengthUTF8('x'), 1)
self._test_func(F.lower('Ab'), 'ab')
self._test_func(F.upper('Ab'), 'AB')
self._test_func(F.lowerUTF8('Ab'), 'ab')
self._test_func(F.upperUTF8('Ab'), 'AB')
self._test_func(F.reverse('Ab'), 'bA')
self._test_func(F.reverseUTF8('Ab'), 'bA')
self._test_func(F.concat('Ab', 'Cd', 'Ef'), 'AbCdEf')
self._test_func(F.substring('123456', 3, 2), '34')
self._test_func(F.substringUTF8('123456', 3, 2), '34')
self._test_func(F.appendTrailingCharIfAbsent('Hello', '!'), 'Hello!')
self._test_func(F.appendTrailingCharIfAbsent('Hello!', '!'), 'Hello!')
self._test_func(F.convertCharset(F.convertCharset('Hello', 'latin1', 'utf16'), 'utf16', 'latin1'), 'Hello')
def test_base64_functions(self):
try:
self._test_func(F.base64Decode(F.base64Encode('Hello')), 'Hello')
self._test_func(F.tryBase64Decode(F.base64Encode('Hello')), 'Hello')
self._test_func(F.tryBase64Decode('zzz'), '')
except ServerError as e:
# ClickHouse version that doesn't support these functions
raise unittest.SkipTest(e.message)

135
tests/test_querysets.py
View File

@ -3,9 +3,10 @@ from __future__ import unicode_literals, print_function
import unittest
from infi.clickhouse_orm.database import Database
from infi.clickhouse_orm.query import Q, F
from infi.clickhouse_orm.query import Q
from infi.clickhouse_orm.funcs import F
from .base_test_with_data import *
from datetime import date, datetime
from datetime import date, datetime, timedelta
from logging import getLogger
logger = getLogger('tests')
@ -432,136 +433,6 @@ class AggregateTestCase(TestCaseWithData):
self.assertEqual(qs.conditions_as_sql(), 'the__next__number > 1')
class FuncsTestCase(TestCaseWithData):
def setUp(self):
super(FuncsTestCase, self).setUp()
self.database.insert(self._sample_data())
def _test_qs(self, qs, expected_count):
logger.info(qs.as_sql())
count = 0
for instance in qs:
count += 1
logger.info('\t[%d]\t%s' % (count, instance.to_dict()))
self.assertEqual(count, expected_count)
self.assertEqual(qs.count(), expected_count)
def _test_func(self, func, expected_value=None):
sql = 'SELECT %s AS value' % func.to_sql()
logger.info(sql)
result = list(self.database.select(sql))
logger.info('\t==> %s', result[0].value)
if expected_value is not None:
self.assertEqual(result[0].value, expected_value)
def test_func_to_sql(self):
# No args
self.assertEqual(F('func').to_sql(), 'func()')
# String args
self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')")
# Numeric args
self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)")
# Date args
self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func('2018-12-31')")
# Datetime args
self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func('1546214400')")
# Boolean args
self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)")
# Null args
self.assertEqual(F('func', None).to_sql(), "func(NULL)")
# Fields as args
self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)")
# Funcs as args
self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))')
# Iterables as args
x = [1, 'z', F('foo', 17)]
for y in [x, tuple(x), iter(x)]:
self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)")
self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([[1, 2], [3, 4]])")
def test_filter_float_field(self):
qs = Person.objects_in(self.database)
# Height > 2
self._test_qs(qs.filter(F.greater(Person.height, 2)), 0)
self._test_qs(qs.filter(Person.height > 2), 0)
# Height > 1.61
self._test_qs(qs.filter(F.greater(Person.height, 1.61)), 96)
self._test_qs(qs.filter(Person.height > 1.61), 96)
# Height < 1.61
self._test_qs(qs.filter(F.less(Person.height, 1.61)), 4)
self._test_qs(qs.filter(Person.height < 1.61), 4)
def test_filter_date_field(self):
qs = Person.objects_in(self.database)
# People born on the 30th
self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3)
self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3)
self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3)
# People born on Sunday
self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18)
self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18)
self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18)
# People born on 1976-10-01
self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1)
self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 01))), 1)
self._test_qs(qs.filter(Person.birthday == date(1976, 10, 01)), 1)
def test_func_as_field_value(self):
qs = Person.objects_in(self.database)
self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96)
self._test_qs(qs.exclude(birthday=F.today()), 100)
self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100)
def test_comparison_operators(self):
one = F.plus(1, 0)
two = F.plus(1, 1)
self._test_func(one > one, 0)
self._test_func(two > one, 1)
self._test_func(one >= two, 0)
self._test_func(one >= one, 1)
self._test_func(one < one, 0)
self._test_func(one < two, 1)
self._test_func(two <= one, 0)
self._test_func(one <= one, 1)
self._test_func(one == two, 0)
self._test_func(one == one, 1)
self._test_func(one != one, 0)
self._test_func(one != two, 1)
def test_arithmetic_operators(self):
one = F.plus(1, 0)
two = F.plus(1, 1)
# +
self._test_func(one + two, 3)
self._test_func(one + 2, 3)
self._test_func(2 + one, 3)
# -
self._test_func(one - two, -1)
self._test_func(one - 2, -1)
self._test_func(1 - two, -1)
# *
self._test_func(one * two, 2)
self._test_func(one * 2, 2)
self._test_func(1 * two, 2)
# /
self._test_func(one / two, 0.5)
self._test_func(one / 2, 0.5)
self._test_func(1 / two, 0.5)
# %
self._test_func(one % two, 1)
self._test_func(one % 2, 1)
self._test_func(1 % two, 1)
# sign
self._test_func(-one, -1)
self._test_func(--one, 1)
self._test_func(+one, 1)
Color = Enum('Color', u'red blue green yellow brown white black')