From 2f99d58a6184cb85dcbd117dceb8d2cd4a1de120 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 14:05:07 +0300 Subject: [PATCH 01/14] added all project files --- .gitignore | 59 +++++++++++++++++++++++++++ README.md | 15 +++++++ buildout.cfg | 54 ++++++++++++++++++++++++ setup.in | 48 ++++++++++++++++++++++ src/infi/__init__.py | 1 + src/infi/clickhouse_utils/__init__.py | 1 + 6 files changed, 178 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 buildout.cfg create mode 100644 setup.in create mode 100644 src/infi/__init__.py create mode 100644 src/infi/clickhouse_utils/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..529b8e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,59 @@ + +# generic +*.tmp +*.swp +*.pyc +*.pyo +*.bak +*.egg-info +*.EGG-INFO +.coverage +.settings + +# eclipse +.project +.pydevproject + +# setuptools +bin/ +build/ +dist/ +setup.py + +# buildout +buildout.in.cfg +.installed*.cfg +.cache +.cache/ +eggs +eggs/ +develop-eggs/ +parts +parts/ +build/ +dist/ +src/infi/hello/__version__.py +.achievements + +# scm +.bzr/ +.svn/ +.hg/ + +# msi-related +parts/wix +parts/product* +parts/*.wixpdb +parts/*.msi + + +src/infi/projector/__version__.py +.codeintel +*sublime* + +tmp* +!tmp*py +buildout.in + +src/infi/clickhouse_utils/__version__.py +bootstrap.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..9e79596 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +Overview +======== +This is an Infinidat project. + +Usage +----- +Nothing to use here. + +Checking out the code +===================== +Run the following commands: + + easy_install -U infi.projector + projector devenv build + diff --git a/buildout.cfg b/buildout.cfg new file mode 100644 index 0000000..404dbdb --- /dev/null +++ b/buildout.cfg @@ -0,0 +1,54 @@ +[buildout] +prefer-final = false +newest = false +download-cache = .cache +develop = . +parts = + +[project] +name = infi.clickhouse_utils +company = Infinidat +namespace_packages = ['infi'] +install_requires = ['setuptools'] +version_file = src/infi/clickhouse_utils/__version__.py +description = A Python library for working with the ClickHouse database +long_description = A Python library for working with the ClickHouse database +console_scripts = [] +gui_scripts = [] +package_data = [] +upgrade_code = {58530fba-3932-11e6-a20e-7071bc32067f} +product_name = infi.clickhouse_utils +post_install_script_name = None +pre_uninstall_script_name = None + +[isolated-python] +recipe = infi.recipe.python +version = v2.7.8.4 + +[setup.py] +recipe = infi.recipe.template.version +input = setup.in +output = setup.py + +[__version__.py] +recipe = infi.recipe.template.version +output = ${project:version_file} + +[development-scripts] +dependent-scripts = true +recipe = infi.recipe.console_scripts +eggs = ${project:name} + ipython + nose + infi.unittest + infi.traceback + zc.buildout +interpreter = python + +[pack] +recipe = infi.recipe.application_packager + +[sublime] +recipe = corneti.recipes.codeintel +eggs = ${development-scripts:eggs} + diff --git a/setup.in b/setup.in new file mode 100644 index 0000000..2f3cef0 --- /dev/null +++ b/setup.in @@ -0,0 +1,48 @@ + +SETUP_INFO = dict( + name = '${project:name}', + version = '${infi.recipe.template.version:version}', + author = '${infi.recipe.template.version:author}', + author_email = '${infi.recipe.template.version:author_email}', + + url = ${infi.recipe.template.version:homepage}, + license = 'PSF', + description = """${project:description}""", + long_description = """${project:long_description}""", + + # http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Python Software Foundation License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + + install_requires = ${project:install_requires}, + namespace_packages = ${project:namespace_packages}, + + package_dir = {'': 'src'}, + package_data = {'': ${project:package_data}}, + include_package_data = True, + zip_safe = False, + + entry_points = dict( + console_scripts = ${project:console_scripts}, + gui_scripts = ${project:gui_scripts}, + ), +) + +if SETUP_INFO['url'] is None: + _ = SETUP_INFO.pop('url') + +def setup(): + from setuptools import setup as _setup + from setuptools import find_packages + SETUP_INFO['packages'] = find_packages('src') + _setup(**SETUP_INFO) + +if __name__ == '__main__': + setup() + diff --git a/src/infi/__init__.py b/src/infi/__init__.py new file mode 100644 index 0000000..5284146 --- /dev/null +++ b/src/infi/__init__.py @@ -0,0 +1 @@ +__import__("pkg_resources").declare_namespace(__name__) diff --git a/src/infi/clickhouse_utils/__init__.py b/src/infi/clickhouse_utils/__init__.py new file mode 100644 index 0000000..5284146 --- /dev/null +++ b/src/infi/clickhouse_utils/__init__.py @@ -0,0 +1 @@ +__import__("pkg_resources").declare_namespace(__name__) From 4da45b0be5953579684dc381d904b3a130f666aa Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 14:11:20 +0300 Subject: [PATCH 02/14] Getting started --- buildout.cfg | 8 +- src/infi/clickhouse_utils/engines.py | 64 +++++++++++ src/infi/clickhouse_utils/fields.py | 155 +++++++++++++++++++++++++++ src/infi/clickhouse_utils/models.py | 62 +++++++++++ src/infi/clickhouse_utils/utils.py | 28 +++++ 5 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 src/infi/clickhouse_utils/engines.py create mode 100644 src/infi/clickhouse_utils/fields.py create mode 100644 src/infi/clickhouse_utils/models.py create mode 100644 src/infi/clickhouse_utils/utils.py diff --git a/buildout.cfg b/buildout.cfg index 404dbdb..f5bd512 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -3,13 +3,17 @@ prefer-final = false newest = false download-cache = .cache develop = . -parts = +parts = [project] name = infi.clickhouse_utils company = Infinidat namespace_packages = ['infi'] -install_requires = ['setuptools'] +install_requires = [ + 'pytz', + 'requests', + 'setuptools' +] version_file = src/infi/clickhouse_utils/__version__.py description = A Python library for working with the ClickHouse database long_description = A Python library for working with the ClickHouse database diff --git a/src/infi/clickhouse_utils/engines.py b/src/infi/clickhouse_utils/engines.py new file mode 100644 index 0000000..3f4870b --- /dev/null +++ b/src/infi/clickhouse_utils/engines.py @@ -0,0 +1,64 @@ + +class Engine(object): + + def create_table_sql(self): + raise NotImplementedError() + + +class MergeTree(Engine): + + def __init__(self, date_col, key_cols, sampling_expr=None, + index_granularity=8192, replica_table_path=None, replica_name=None): + self.date_col = date_col + self.key_cols = key_cols + self.sampling_expr = sampling_expr + self.index_granularity = index_granularity + self.replica_table_path = replica_table_path + self.replica_name = replica_name + # TODO verify that both replica fields are either present or missing + + def create_table_sql(self): + name = self.__class__.__name__ + if self.replica_name: + name = 'Replicated' + name + params = self._build_sql_params() + return '%s(%s)' % (name, ', '.join(params)) + + def _build_sql_params(self): + params = [] + if self.replica_name: + params += ["'%s'" % self.replica_table_path, "'%s'" % self.replica_name] + params.append(self.date_col) + if self.sampling_expr: + params.append(self.sampling_expr) + params.append('(%s)' % ', '.join(self.key_cols)) + params.append(str(self.index_granularity)) + return params + + +class CollapsingMergeTree(MergeTree): + + def __init__(self, date_col, key_cols, sign_col, sampling_expr=None, + index_granularity=8192, replica_table_path=None, replica_name=None): + super(CollapsingMergeTree, self).__init__(date_col, key_cols, sampling_expr, index_granularity, replica_table_path, replica_name) + self.sign_col = sign_col + + def _build_sql_params(self): + params = super(CollapsingMergeTree, self)._build_sql_params() + params.append(self.sign_col) + return params + + +class SummingMergeTree(MergeTree): + + def __init__(self, date_col, key_cols, summing_cols=None, sampling_expr=None, + index_granularity=8192, replica_table_path=None, replica_name=None): + super(SummingMergeTree, self).__init__(date_col, key_cols, sampling_expr, index_granularity, replica_table_path, replica_name) + self.summing_cols = summing_cols + + def _build_sql_params(self): + params = super(SummingMergeTree, self)._build_sql_params() + if self.summing_cols: + params.append('(%s)' % ', '.join(self.summing_cols)) + return params + diff --git a/src/infi/clickhouse_utils/fields.py b/src/infi/clickhouse_utils/fields.py new file mode 100644 index 0000000..f357ac5 --- /dev/null +++ b/src/infi/clickhouse_utils/fields.py @@ -0,0 +1,155 @@ +import datetime +import pytz +import time + + +class Field(object): + + creation_counter = 0 + class_default = 0 + db_type = None + + def __init__(self, default=None): + self.creation_counter = Field.creation_counter + Field.creation_counter += 1 + self.default = default or self.class_default + + def to_python(self, value): + """ + Converts the input value into the expected Python data type, raising ValueError if the + data can't be converted. Returns the converted value. Subclasses should override this. + """ + return value + + def get_db_prep_value(self, value): + """ + Returns the field's value prepared for interacting with the database. + """ + return value + + +class StringField(Field): + + class_default = '' + db_type = 'String' + + def to_python(self, value): + if isinstance(value, unicode): + return value + if isinstance(value, str): + return value.decode('UTF-8') + raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + + def get_db_prep_value(self, value): + if isinstance(value, unicode): + return value.encode('UTF-8') + return value + + +class DateField(Field): + + class_default = datetime.date(1970, 1, 1) + db_type = 'Date' + + def to_python(self, value): + if isinstance(value, datetime.date): + return value + if isinstance(value, int): + return DateField.class_default + datetime.timedelta(days=value) + if isinstance(value, basestring): + return datetime.datetime.strptime(value, '%Y-%m-%d').date() + raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + + def get_db_prep_value(self, value): + return value.isoformat() + + +class DateTimeField(Field): + + class_default = datetime.datetime.fromtimestamp(0, pytz.utc) + db_type = 'DateTime' + + def to_python(self, value): + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime(value.year, value.month, value.day) + if isinstance(value, int): + return datetime.datetime.fromtimestamp(value, pytz.utc) + if isinstance(value, basestring): + return datetime.datetime.strptime(value, '%Y-%m-%d %H-%M-%S') + raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + + def get_db_prep_value(self, value): + return int(time.mktime(value.timetuple())) + + +class BaseIntField(Field): + + def to_python(self, value): + if isinstance(value, int): + return value + if isinstance(value, basestring): + return int(value) + raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + + +class UInt8Field(BaseIntField): + + db_type = 'UInt8' + + +class UInt16Field(BaseIntField): + + db_type = 'UInt16' + + +class UInt32Field(BaseIntField): + + db_type = 'UInt32' + + +class UInt64Field(BaseIntField): + + db_type = 'UInt64' + + +class Int8Field(BaseIntField): + + db_type = 'Int8' + + +class Int16Field(BaseIntField): + + db_type = 'Int16' + + +class Int32Field(BaseIntField): + + db_type = 'Int32' + + +class Int64Field(BaseIntField): + + db_type = 'Int64' + + +class BaseFloatField(Field): + + def to_python(self, value): + if isinstance(value, float): + return value + if isinstance(value, basestring): + return float(value) + raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + + +class Float32Field(BaseFloatField): + + db_type = 'Float32' + + +class Float64Field(BaseFloatField): + + db_type = 'Float64' + diff --git a/src/infi/clickhouse_utils/models.py b/src/infi/clickhouse_utils/models.py new file mode 100644 index 0000000..c4074f7 --- /dev/null +++ b/src/infi/clickhouse_utils/models.py @@ -0,0 +1,62 @@ +from fields import * +from utils import escape, parse_tsv +from engines import * + + +class ModelBase(type): + + def __new__(cls, name, bases, attrs): + new_cls = super(ModelBase, cls).__new__(cls, name, bases, attrs) + #print name, bases, attrs + # Build a list of fields, in the order they were listed in the class + fields = [item for item in attrs.items() if isinstance(item[1], Field)] + fields.sort(key=lambda item: item[1].creation_counter) + setattr(new_cls, '_fields', fields) + return new_cls + + +class Model(object): + + __metaclass__ = ModelBase + engine = None + + def __init__(self, *args, **kwargs): + super(Model, self).__init__() + for name, field in self._fields: + val = kwargs.get(name, field.default) + setattr(self, name, val) + + @classmethod + def table_name(cls): + return cls.__name__.lower() + + @classmethod + def create_table_sql(cls, db): + parts = ['CREATE TABLE IF NOT EXISTS %s.%s (' % (db, cls.table_name())] + for name, field in cls._fields: + default = field.get_db_prep_value(field.default) + parts.append(' %s %s DEFAULT %s,' % (name, field.db_type, escape(default))) + parts.append(')') + parts.append('ENGINE = ' + cls.engine.create_table_sql()) + return '\n'.join(parts) + + @classmethod + def from_tsv(cls, line): + ''' + Create a model instance from a tab-separated line. The line may or may not include a newline. + ''' + values = iter(parse_tsv(line)) + kwargs = {} + for name, field in cls._fields: + kwargs[name] = field.to_python(values.next()) + return cls(**kwargs) + + def to_tsv(self): + ''' + Returns the instance's column values as a tab-separated line. A newline is not included. + ''' + parts = [] + for name, field in self._fields: + value = field.get_db_prep_value(field.to_python(getattr(self, name))) + parts.append(escape(value, quote=False)) + return '\t'.join(parts) diff --git a/src/infi/clickhouse_utils/utils.py b/src/infi/clickhouse_utils/utils.py new file mode 100644 index 0000000..5f49778 --- /dev/null +++ b/src/infi/clickhouse_utils/utils.py @@ -0,0 +1,28 @@ + +SPECIAL_CHARS = { + "\b" : "\\b", + "\f" : "\\f", + "\r" : "\\r", + "\n" : "\\n", + "\t" : "\\t", + "\0" : "\\0", + "\\" : "\\\\", + "'" : "\\'" +} + + +def escape(value, quote=True): + if isinstance(value, basestring): + chars = (SPECIAL_CHARS.get(c, c) for c in value) + return "'" + "".join(chars) + "'" if quote else "".join(chars) + return str(value) + + +def unescape(value): + return value.decode('string_escape') + + +def parse_tsv(line): + if line[-1] == '\n': + line = line[:-1] + return [_unescape(value) for value in line.split('\t')] From b1174f09eab354edbb4150d7745b78cde708a112 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 15:10:49 +0300 Subject: [PATCH 03/14] Initial commit of database class --- buildout.cfg | 4 ++- src/infi/clickhouse_utils/database.py | 38 +++++++++++++++++++++++++++ src/infi/clickhouse_utils/models.py | 12 ++++++--- tests/test_orm.py | 32 ++++++++++++++++++++++ 4 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/infi/clickhouse_utils/database.py create mode 100644 tests/test_orm.py diff --git a/buildout.cfg b/buildout.cfg index f5bd512..f0c8519 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -13,7 +13,7 @@ install_requires = [ 'pytz', 'requests', 'setuptools' -] + ] version_file = src/infi/clickhouse_utils/__version__.py description = A Python library for working with the ClickHouse database long_description = A Python library for working with the ClickHouse database @@ -47,6 +47,8 @@ eggs = ${project:name} infi.unittest infi.traceback zc.buildout +scripts = ipython + nosetests interpreter = python [pack] diff --git a/src/infi/clickhouse_utils/database.py b/src/infi/clickhouse_utils/database.py new file mode 100644 index 0000000..b7c0548 --- /dev/null +++ b/src/infi/clickhouse_utils/database.py @@ -0,0 +1,38 @@ +import requests + + +class DatabaseException(Exception): + pass + + +class Database(object): + + def __init__(self, db_name, db_url='http://localhost:8123/', username=None, password=None): + self.db_name = db_name + self.db_url = db_url + self.username = username + self.password = password + self._send('CREATE DATABASE IF NOT EXISTS ' + db_name) + + def create_table(self, model_class): + self._send(model_class.create_table_sql(self.db_name)) + + def drop_table(self, model_class): + self._send(model_class.drop_table_sql(self.db_name)) + + def drop_database(self): + self._send('DROP DATABASE ' + self.db_name) + + def _send(self, sql, settings=None): + params = self._build_params(settings) + r = requests.post(self.db_url, params=params, data=sql) + if r.status_code != 200: + raise DatabaseException(r.text) + + def _build_params(self, settings): + params = dict(settings or {}) + if self.username: + params['username'] = username + if self.password: + params['password'] = password + return params diff --git a/src/infi/clickhouse_utils/models.py b/src/infi/clickhouse_utils/models.py index c4074f7..f9fad43 100644 --- a/src/infi/clickhouse_utils/models.py +++ b/src/infi/clickhouse_utils/models.py @@ -31,15 +31,21 @@ class Model(object): return cls.__name__.lower() @classmethod - def create_table_sql(cls, db): - parts = ['CREATE TABLE IF NOT EXISTS %s.%s (' % (db, cls.table_name())] + def create_table_sql(cls, db_name): + parts = ['CREATE TABLE IF NOT EXISTS %s.%s (' % (db_name, cls.table_name())] + cols = [] for name, field in cls._fields: default = field.get_db_prep_value(field.default) - parts.append(' %s %s DEFAULT %s,' % (name, field.db_type, escape(default))) + cols.append(' %s %s DEFAULT %s' % (name, field.db_type, escape(default))) + parts.append(', \n'.join(cols)) parts.append(')') parts.append('ENGINE = ' + cls.engine.create_table_sql()) return '\n'.join(parts) + @classmethod + def drop_table_sql(cls, db_name): + return 'DROP TABLE IF EXISTS %s.%s' % (db_name, cls.table_name()) + @classmethod def from_tsv(cls, line): ''' diff --git a/tests/test_orm.py b/tests/test_orm.py new file mode 100644 index 0000000..c4f8d74 --- /dev/null +++ b/tests/test_orm.py @@ -0,0 +1,32 @@ +import unittest + +from infi.clickhouse_utils.database import Database +from infi.clickhouse_utils.models import Model +from infi.clickhouse_utils.fields import * +from infi.clickhouse_utils.engines import * + + +class ORMTestCase(unittest.TestCase): + + def setUp(self): + self.database = Database('test_db') + + def tearDown(self): + self.database.drop_database() + + def test_create_table(self): + self.database.create_table(Person) + self.database.drop_table(Person) + + +class Person(Model): + + first_name = StringField() + last_name = StringField() + birthday = DateField() + height = Float32Field() + + engine = MergeTree('birthday', ('first_name', 'last_name', 'birthday')) + + + From 571ea8b6e594eaba11c58bc1bd64d055ac28f73f Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 15:46:11 +0300 Subject: [PATCH 04/14] rename project --- .gitignore | 2 +- buildout.cfg | 6 +++--- src/infi/{clickhouse_utils => clickhouse_orm}/__init__.py | 0 src/infi/{clickhouse_utils => clickhouse_orm}/database.py | 0 src/infi/{clickhouse_utils => clickhouse_orm}/engines.py | 0 src/infi/{clickhouse_utils => clickhouse_orm}/fields.py | 0 src/infi/{clickhouse_utils => clickhouse_orm}/models.py | 0 src/infi/{clickhouse_utils => clickhouse_orm}/utils.py | 0 tests/test_orm.py | 8 ++++---- 9 files changed, 8 insertions(+), 8 deletions(-) rename src/infi/{clickhouse_utils => clickhouse_orm}/__init__.py (100%) rename src/infi/{clickhouse_utils => clickhouse_orm}/database.py (100%) rename src/infi/{clickhouse_utils => clickhouse_orm}/engines.py (100%) rename src/infi/{clickhouse_utils => clickhouse_orm}/fields.py (100%) rename src/infi/{clickhouse_utils => clickhouse_orm}/models.py (100%) rename src/infi/{clickhouse_utils => clickhouse_orm}/utils.py (100%) diff --git a/.gitignore b/.gitignore index 529b8e1..c1eacdb 100644 --- a/.gitignore +++ b/.gitignore @@ -55,5 +55,5 @@ tmp* !tmp*py buildout.in -src/infi/clickhouse_utils/__version__.py +src/infi/clickhouse_orm/__version__.py bootstrap.py diff --git a/buildout.cfg b/buildout.cfg index f0c8519..8238f89 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -6,7 +6,7 @@ develop = . parts = [project] -name = infi.clickhouse_utils +name = infi.clickhouse_orm company = Infinidat namespace_packages = ['infi'] install_requires = [ @@ -14,14 +14,14 @@ install_requires = [ 'requests', 'setuptools' ] -version_file = src/infi/clickhouse_utils/__version__.py +version_file = src/infi/clickhouse_orm/__version__.py description = A Python library for working with the ClickHouse database long_description = A Python library for working with the ClickHouse database console_scripts = [] gui_scripts = [] package_data = [] upgrade_code = {58530fba-3932-11e6-a20e-7071bc32067f} -product_name = infi.clickhouse_utils +product_name = infi.clickhouse_orm post_install_script_name = None pre_uninstall_script_name = None diff --git a/src/infi/clickhouse_utils/__init__.py b/src/infi/clickhouse_orm/__init__.py similarity index 100% rename from src/infi/clickhouse_utils/__init__.py rename to src/infi/clickhouse_orm/__init__.py diff --git a/src/infi/clickhouse_utils/database.py b/src/infi/clickhouse_orm/database.py similarity index 100% rename from src/infi/clickhouse_utils/database.py rename to src/infi/clickhouse_orm/database.py diff --git a/src/infi/clickhouse_utils/engines.py b/src/infi/clickhouse_orm/engines.py similarity index 100% rename from src/infi/clickhouse_utils/engines.py rename to src/infi/clickhouse_orm/engines.py diff --git a/src/infi/clickhouse_utils/fields.py b/src/infi/clickhouse_orm/fields.py similarity index 100% rename from src/infi/clickhouse_utils/fields.py rename to src/infi/clickhouse_orm/fields.py diff --git a/src/infi/clickhouse_utils/models.py b/src/infi/clickhouse_orm/models.py similarity index 100% rename from src/infi/clickhouse_utils/models.py rename to src/infi/clickhouse_orm/models.py diff --git a/src/infi/clickhouse_utils/utils.py b/src/infi/clickhouse_orm/utils.py similarity index 100% rename from src/infi/clickhouse_utils/utils.py rename to src/infi/clickhouse_orm/utils.py diff --git a/tests/test_orm.py b/tests/test_orm.py index c4f8d74..d77da18 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -1,9 +1,9 @@ import unittest -from infi.clickhouse_utils.database import Database -from infi.clickhouse_utils.models import Model -from infi.clickhouse_utils.fields import * -from infi.clickhouse_utils.engines import * +from infi.clickhouse_orm.database import Database +from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.fields import * +from infi.clickhouse_orm.engines import * class ORMTestCase(unittest.TestCase): From 25e85adc0d8b27949cb1c14a1ff1447f41fdff61 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 17:35:17 +0300 Subject: [PATCH 05/14] add database.insert and database.count --- src/infi/clickhouse_orm/database.py | 28 +++++- src/infi/clickhouse_orm/models.py | 2 +- tests/test_orm.py | 134 +++++++++++++++++++++++++++- 3 files changed, 157 insertions(+), 7 deletions(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index b7c0548..ae37f11 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -23,11 +23,35 @@ class Database(object): def drop_database(self): self._send('DROP DATABASE ' + self.db_name) - def _send(self, sql, settings=None): + def insert(self, model_instances): + i = iter(model_instances) + try: + first_instance = i.next() + except StopIteration: + return # model_instances is empty + model_class = first_instance.__class__ + def gen(): + yield 'INSERT INTO %s.%s FORMAT TabSeparated\n' % (self.db_name, model_class.table_name()) + yield first_instance.to_tsv() + yield '\n' + for instance in i: + yield instance.to_tsv() + yield '\n' + self._send(gen()) + + def count(self, model_class, conditions=None): + query = 'SELECT uniq(height) FROM %s.%s' % (self.db_name, model_class.table_name()) + if conditions: + query += ' WHERE ' + conditions + r = self._send(query) + return int(r.text) if r.text else 0 + + def _send(self, data, settings=None): params = self._build_params(settings) - r = requests.post(self.db_url, params=params, data=sql) + r = requests.post(self.db_url, params=params, data=data) if r.status_code != 200: raise DatabaseException(r.text) + return r def _build_params(self, settings): params = dict(settings or {}) diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index f9fad43..7b898ac 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -37,7 +37,7 @@ class Model(object): for name, field in cls._fields: default = field.get_db_prep_value(field.default) cols.append(' %s %s DEFAULT %s' % (name, field.db_type, escape(default))) - parts.append(', \n'.join(cols)) + parts.append(',\n'.join(cols)) parts.append(')') parts.append('ENGINE = ' + cls.engine.create_table_sql()) return '\n'.join(parts) diff --git a/tests/test_orm.py b/tests/test_orm.py index d77da18..77aea01 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -10,13 +10,38 @@ class ORMTestCase(unittest.TestCase): def setUp(self): self.database = Database('test_db') + self.database.create_table(Person) def tearDown(self): + self.database.drop_table(Person) self.database.drop_database() - def test_create_table(self): - self.database.create_table(Person) - self.database.drop_table(Person) + def _insert_and_check(self, data, count): + self.database.insert(data) + self.assertEquals(count, self.database.count(Person)) + + def test_insert__generator(self): + self._insert_and_check(self._sample_data(), len(data)) + + def test_insert__list(self): + self._insert_and_check(list(self._sample_data()), len(data)) + + def test_insert__iterator(self): + self._insert_and_check(iter(self._sample_data()), len(data)) + + def test_insert__empty(self): + self._insert_and_check([], 0) + + def test_count(self): + self.database.insert(self._sample_data()) + self.assertEquals(self.database.count(Person), 100) + self.assertEquals(self.database.count(Person, "first_name = 'Courtney'"), 2) + self.assertEquals(self.database.count(Person, "birthday > '2000-01-01'"), 22) + self.assertEquals(self.database.count(Person, "birthday < '1900-01-01'"), 0) + + def _sample_data(self): + for entry in data: + yield Person(**entry) class Person(Model): @@ -29,4 +54,105 @@ class Person(Model): engine = MergeTree('birthday', ('first_name', 'last_name', 'birthday')) - +data = [ + {"first_name": "Courtney", "last_name": "Cannon", "birthday": "1967-10-26", "height": "1.76"}, + {"first_name": "Kieran", "last_name": "Solomon", "birthday": "1963-10-30", "height": "1.69"}, + {"first_name": "Wynter", "last_name": "Garcia", "birthday": "1945-01-10", "height": "1.69"}, + {"first_name": "Buffy", "last_name": "Webb", "birthday": "1960-03-06", "height": "1.68"}, + {"first_name": "Jarrod", "last_name": "Gibbs", "birthday": "1957-06-13", "height": "1.62"}, + {"first_name": "Cara", "last_name": "Fox", "birthday": "2004-05-15", "height": "1.71"}, + {"first_name": "Shoshana", "last_name": "Solis", "birthday": "1968-07-18", "height": "1.65"}, + {"first_name": "Adam", "last_name": "Goodman", "birthday": "1956-01-07", "height": "1.74"}, + {"first_name": "Ciaran", "last_name": "Hurley", "birthday": "1965-10-25", "height": "1.65"}, + {"first_name": "Emma", "last_name": "Clements", "birthday": "1996-08-07", "height": "1.75"}, + {"first_name": "Warren", "last_name": "Dudley", "birthday": "1965-10-23", "height": "1.59"}, + {"first_name": "Courtney", "last_name": "Hoffman", "birthday": "1994-11-07", "height": "1.65"}, + {"first_name": "Karleigh", "last_name": "Bartlett", "birthday": "1991-10-24", "height": "1.69"}, + {"first_name": "Aline", "last_name": "Crane", "birthday": "1958-05-01", "height": "1.62"}, + {"first_name": "Sawyer", "last_name": "Fischer", "birthday": "1995-04-01", "height": "1.78"}, + {"first_name": "Ella", "last_name": "Castillo", "birthday": "1943-03-28", "height": "1.73"}, + {"first_name": "Jesse", "last_name": "Gomez", "birthday": "2011-01-28", "height": "1.71"}, + {"first_name": "Mary", "last_name": "Kirkland", "birthday": "1987-10-09", "height": "1.73"}, + {"first_name": "Quon", "last_name": "Wiggins", "birthday": "1962-05-06", "height": "1.74"}, + {"first_name": "Dominique", "last_name": "Sandoval", "birthday": "1942-02-01", "height": "1.72"}, + {"first_name": "Elton", "last_name": "Smith", "birthday": "1952-06-20", "height": "1.66"}, + {"first_name": "Whitney", "last_name": "Scott", "birthday": "1941-07-04", "height": "1.70"}, + {"first_name": "Sydney", "last_name": "Stevens", "birthday": "1959-07-11", "height": "1.70"}, + {"first_name": "Lesley", "last_name": "Stephenson", "birthday": "2010-04-10", "height": "1.64"}, + {"first_name": "Georgia", "last_name": "Kennedy", "birthday": "1944-12-29", "height": "1.66"}, + {"first_name": "Norman", "last_name": "Santos", "birthday": "1959-01-10", "height": "1.68"}, + {"first_name": "Vaughan", "last_name": "Schmidt", "birthday": "1955-06-19", "height": "1.61"}, + {"first_name": "Edan", "last_name": "Dennis", "birthday": "1959-09-18", "height": "1.73"}, + {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1967-02-25", "height": "1.74"}, + {"first_name": "Seth", "last_name": "Serrano", "birthday": "2017-06-02", "height": "1.71"}, + {"first_name": "Callie", "last_name": "Wiley", "birthday": "1957-11-24", "height": "1.69"}, + {"first_name": "Shana", "last_name": "Jarvis", "birthday": "1967-05-21", "height": "1.72"}, + {"first_name": "Madeline", "last_name": "Kidd", "birthday": "1954-12-09", "height": "1.69"}, + {"first_name": "Macaulay", "last_name": "Rowe", "birthday": "1952-03-02", "height": "1.68"}, + {"first_name": "Rafael", "last_name": "Parker", "birthday": "2016-01-24", "height": "1.76"}, + {"first_name": "Ava", "last_name": "Sanders", "birthday": "1967-08-10", "height": "1.60"}, + {"first_name": "Ciaran", "last_name": "Carver", "birthday": "2016-12-25", "height": "1.76"}, + {"first_name": "Hyacinth", "last_name": "Kent", "birthday": "1971-07-18", "height": "1.72"}, + {"first_name": "Velma", "last_name": "English", "birthday": "1999-01-18", "height": "1.65"}, + {"first_name": "Adena", "last_name": "Norman", "birthday": "1949-05-14", "height": "1.66"}, + {"first_name": "Hedy", "last_name": "Strong", "birthday": "2001-10-04", "height": "1.60"}, + {"first_name": "Griffith", "last_name": "Henry", "birthday": "1955-04-02", "height": "1.66"}, + {"first_name": "Barrett", "last_name": "Clemons", "birthday": "1985-07-03", "height": "1.71"}, + {"first_name": "Roary", "last_name": "Simmons", "birthday": "1956-07-23", "height": "1.63"}, + {"first_name": "Connor", "last_name": "Jenkins", "birthday": "1969-07-23", "height": "1.67"}, + {"first_name": "Cassady", "last_name": "Knapp", "birthday": "1947-12-15", "height": "1.72"}, + {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63"}, + {"first_name": "Evangeline", "last_name": "Weber", "birthday": "1984-06-03", "height": "1.70"}, + {"first_name": "Clementine", "last_name": "Moon", "birthday": "1964-03-29", "height": "1.73"}, + {"first_name": "Montana", "last_name": "Bruce", "birthday": "1952-06-28", "height": "1.66"}, + {"first_name": "Whitney", "last_name": "Durham", "birthday": "1947-09-15", "height": "1.72"}, + {"first_name": "Naomi", "last_name": "Hays", "birthday": "2004-11-27", "height": "1.70"}, + {"first_name": "Macey", "last_name": "Griffin", "birthday": "1971-09-18", "height": "1.63"}, + {"first_name": "Dora", "last_name": "Cabrera", "birthday": "2016-04-26", "height": "1.68"}, + {"first_name": "Chelsea", "last_name": "Castro", "birthday": "2001-08-10", "height": "1.71"}, + {"first_name": "Reese", "last_name": "Noel", "birthday": "1966-11-04", "height": "1.77"}, + {"first_name": "Althea", "last_name": "Barrett", "birthday": "2004-07-28", "height": "1.71"}, + {"first_name": "Odette", "last_name": "Mcneil", "birthday": "1948-05-21", "height": "1.76"}, + {"first_name": "Oliver", "last_name": "Ashley", "birthday": "2004-08-13", "height": "1.68"}, + {"first_name": "Ashton", "last_name": "Fuller", "birthday": "1995-11-17", "height": "1.75"}, + {"first_name": "Denton", "last_name": "Sanchez", "birthday": "1971-10-16", "height": "1.72"}, + {"first_name": "Molly", "last_name": "Salas", "birthday": "1994-04-23", "height": "1.70"}, + {"first_name": "Josiah", "last_name": "Hodges", "birthday": "2011-09-04", "height": "1.68"}, + {"first_name": "Victoria", "last_name": "Slater", "birthday": "2009-07-19", "height": "1.72"}, + {"first_name": "Catherine", "last_name": "Hicks", "birthday": "1959-05-23", "height": "1.80"}, + {"first_name": "Sharon", "last_name": "Shelton", "birthday": "1970-05-02", "height": "1.65"}, + {"first_name": "Rigel", "last_name": "Oneal", "birthday": "1993-11-05", "height": "1.63"}, + {"first_name": "Victor", "last_name": "Woods", "birthday": "1959-06-23", "height": "1.67"}, + {"first_name": "Cathleen", "last_name": "Frank", "birthday": "1947-09-04", "height": "1.61"}, + {"first_name": "Russell", "last_name": "Pruitt", "birthday": "1979-05-04", "height": "1.63"}, + {"first_name": "Warren", "last_name": "Bowen", "birthday": "2000-07-20", "height": "1.76"}, + {"first_name": "Hu", "last_name": "May", "birthday": "1976-10-01", "height": "1.76"}, + {"first_name": "Amanda", "last_name": "Vang", "birthday": "1943-02-23", "height": "1.68"}, + {"first_name": "Fritz", "last_name": "Atkinson", "birthday": "2011-06-15", "height": "1.73"}, + {"first_name": "Scarlett", "last_name": "Durham", "birthday": "2005-09-29", "height": "1.65"}, + {"first_name": "Leroy", "last_name": "Pacheco", "birthday": "1968-12-30", "height": "1.70"}, + {"first_name": "Eagan", "last_name": "Dodson", "birthday": "2015-10-22", "height": "1.67"}, + {"first_name": "Whilemina", "last_name": "Blankenship", "birthday": "1970-07-14", "height": "1.66"}, + {"first_name": "Anne", "last_name": "Rasmussen", "birthday": "1995-04-03", "height": "1.77"}, + {"first_name": "Caryn", "last_name": "Sears", "birthday": "1999-02-17", "height": "1.71"}, + {"first_name": "Elton", "last_name": "Ayers", "birthday": "1994-06-20", "height": "1.68"}, + {"first_name": "Venus", "last_name": "Hurst", "birthday": "1963-10-22", "height": "1.72"}, + {"first_name": "Octavius", "last_name": "Floyd", "birthday": "1955-02-22", "height": "1.68"}, + {"first_name": "Keelie", "last_name": "Mathis", "birthday": "1963-10-26", "height": "1.69"}, + {"first_name": "Wang", "last_name": "Goodwin", "birthday": "1983-05-15", "height": "1.66"}, + {"first_name": "Rhona", "last_name": "Camacho", "birthday": "1946-12-17", "height": "1.59"}, + {"first_name": "Ulla", "last_name": "Arnold", "birthday": "1990-06-04", "height": "1.63"}, + {"first_name": "Idola", "last_name": "Fulton", "birthday": "1974-11-27", "height": "1.66"}, + {"first_name": "Galvin", "last_name": "Phillips", "birthday": "2004-01-17", "height": "1.74"}, + {"first_name": "Faith", "last_name": "Emerson", "birthday": "1959-12-30", "height": "1.62"}, + {"first_name": "Shad", "last_name": "Bradshaw", "birthday": "1968-08-25", "height": "1.72"}, + {"first_name": "Stephen", "last_name": "Baxter", "birthday": "2004-09-24", "height": "1.74"}, + {"first_name": "Tasha", "last_name": "Campos", "birthday": "1984-02-11", "height": "1.72"}, + {"first_name": "Celeste", "last_name": "James", "birthday": "1990-03-08", "height": "1.67"}, + {"first_name": "Ariana", "last_name": "Cole", "birthday": "1947-12-20", "height": "1.72"}, + {"first_name": "Beatrice", "last_name": "Gregory", "birthday": "1992-01-19", "height": "1.80"}, + {"first_name": "Laith", "last_name": "Howell", "birthday": "1961-07-07", "height": "1.70"}, + {"first_name": "Angela", "last_name": "Sanders", "birthday": "2016-01-08", "height": "1.66"}, + {"first_name": "Cassady", "last_name": "Rogers", "birthday": "2013-11-04", "height": "1.71"}, + {"first_name": "Maia", "last_name": "Hyde", "birthday": "1972-06-09", "height": "1.74"} +]; From b08f1b3688ac5374930ec7e34c5488d97681cf24 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 18:24:20 +0300 Subject: [PATCH 06/14] model field conversion on assignment --- src/infi/clickhouse_orm/database.py | 2 +- src/infi/clickhouse_orm/fields.py | 12 +++--- src/infi/clickhouse_orm/models.py | 27 +++++++++++-- tests/__init__.py | 0 tests/{test_orm.py => test_database.py} | 2 +- tests/test_models.py | 54 +++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 tests/__init__.py rename tests/{test_orm.py => test_database.py} (99%) create mode 100644 tests/test_models.py diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index ae37f11..b7ae30a 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -40,7 +40,7 @@ class Database(object): self._send(gen()) def count(self, model_class, conditions=None): - query = 'SELECT uniq(height) FROM %s.%s' % (self.db_name, model_class.table_name()) + query = 'SELECT count() FROM %s.%s' % (self.db_name, model_class.table_name()) if conditions: query += ' WHERE ' + conditions r = self._send(query) diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index f357ac5..91b0e50 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -38,7 +38,7 @@ class StringField(Field): return value if isinstance(value, str): return value.decode('UTF-8') - raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) def get_db_prep_value(self, value): if isinstance(value, unicode): @@ -58,7 +58,7 @@ class DateField(Field): return DateField.class_default + datetime.timedelta(days=value) if isinstance(value, basestring): return datetime.datetime.strptime(value, '%Y-%m-%d').date() - raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) def get_db_prep_value(self, value): return value.isoformat() @@ -78,7 +78,7 @@ class DateTimeField(Field): return datetime.datetime.fromtimestamp(value, pytz.utc) if isinstance(value, basestring): return datetime.datetime.strptime(value, '%Y-%m-%d %H-%M-%S') - raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) def get_db_prep_value(self, value): return int(time.mktime(value.timetuple())) @@ -91,7 +91,7 @@ class BaseIntField(Field): return value if isinstance(value, basestring): return int(value) - raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) class UInt8Field(BaseIntField): @@ -139,9 +139,9 @@ class BaseFloatField(Field): def to_python(self, value): if isinstance(value, float): return value - if isinstance(value, basestring): + if isinstance(value, basestring) or isinstance(value, int): return float(value) - raise ValueError('Invalid value for %s: %r', self.__class__.__name__, value) + raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) class Float32Field(BaseFloatField): diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index 7b898ac..eb566ff 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -4,10 +4,12 @@ from engines import * class ModelBase(type): + ''' + A metaclass for ORM models. It adds the _fields list to model classes. + ''' def __new__(cls, name, bases, attrs): new_cls = super(ModelBase, cls).__new__(cls, name, bases, attrs) - #print name, bases, attrs # Build a list of fields, in the order they were listed in the class fields = [item for item in attrs.items() if isinstance(item[1], Field)] fields.sort(key=lambda item: item[1].creation_counter) @@ -16,16 +18,34 @@ class ModelBase(type): class Model(object): + ''' + A base class for ORM models. + ''' __metaclass__ = ModelBase engine = None - def __init__(self, *args, **kwargs): + def __init__(self, **kwargs): + ''' + Creates a model instance, using keyword arguments as field values. + Since values are immediately converted to their Pythonic type, + invalid values will cause a ValueError to be raised. + ''' super(Model, self).__init__() for name, field in self._fields: val = kwargs.get(name, field.default) setattr(self, name, val) + def __setattr__(self, name, value): + field = self.get_field(name) + if field: + value = field.to_python(value) + super(Model, self).__setattr__(name, value) + + def get_field(self, name): + field = getattr(self.__class__, name, None) + return field if isinstance(field, Field) else None + @classmethod def table_name(cls): return cls.__name__.lower() @@ -54,8 +74,9 @@ class Model(object): values = iter(parse_tsv(line)) kwargs = {} for name, field in cls._fields: - kwargs[name] = field.to_python(values.next()) + kwargs[name] = values.next() return cls(**kwargs) + # TODO verify that the number of values matches the number of fields def to_tsv(self): ''' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_orm.py b/tests/test_database.py similarity index 99% rename from tests/test_orm.py rename to tests/test_database.py index 77aea01..c368e7a 100644 --- a/tests/test_orm.py +++ b/tests/test_database.py @@ -6,7 +6,7 @@ from infi.clickhouse_orm.fields import * from infi.clickhouse_orm.engines import * -class ORMTestCase(unittest.TestCase): +class DatabaseTestCase(unittest.TestCase): def setUp(self): self.database = Database('test_db') diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..7725172 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,54 @@ +import unittest +import datetime +import pytz + +from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.fields import * +from infi.clickhouse_orm.engines import * + + +class ModelTestCase(unittest.TestCase): + + def test_defaults(self): + # Check that all fields have their defaults + instance = SimpleModel() + self.assertEquals(instance.date_field, datetime.date(1970, 1, 1)) + self.assertEquals(instance.datetime_field, datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)) + self.assertEquals(instance.str_field, 'dozo') + self.assertEquals(instance.int_field, 17) + self.assertEquals(instance.float_field, 0) + + def test_assignment(self): + # Check that all fields are assigned during construction + kwargs = dict( + date_field=datetime.date(1973, 12, 6), + datetime_field=datetime.datetime(2000, 5, 24, 10, 22, tzinfo=pytz.utc), + str_field='aloha', + int_field=-50, + float_field=3.14 + ) + instance = SimpleModel(**kwargs) + for name, value in kwargs.items(): + self.assertEquals(kwargs[name], getattr(instance, name)) + + def test_string_conversion(self): + # Check field conversion from string during construction + instance = SimpleModel(date_field='1973-12-06', int_field='100', float_field='7') + self.assertEquals(instance.date_field, datetime.date(1973, 12, 6)) + self.assertEquals(instance.int_field, 100) + self.assertEquals(instance.float_field, 7) + # Check field conversion from string during assignment + instance.int_field = '99' + self.assertEquals(instance.int_field, 99) + + +class SimpleModel(Model): + + date_field = DateField() + datetime_field = DateTimeField() + str_field = StringField(default='dozo') + int_field = Int32Field(default=17) + float_field = Float32Field() + + engine = MergeTree('date_field', ('int_field', 'date_field')) + From 92e8b8259f144dced8bccc41edc51d3e5244df7d Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 18:34:52 +0300 Subject: [PATCH 07/14] better validation of keyword arguments in model constructor --- src/infi/clickhouse_orm/models.py | 13 +++++++++++-- tests/test_models.py | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index eb566ff..e5d41fd 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -30,11 +30,20 @@ class Model(object): Creates a model instance, using keyword arguments as field values. Since values are immediately converted to their Pythonic type, invalid values will cause a ValueError to be raised. + Unrecognized field names will cause an AttributeError. ''' super(Model, self).__init__() + # Assign field values from keyword arguments + for name, value in kwargs.iteritems(): + field = self.get_field(name) + if field: + setattr(self, name, value) + else: + raise AttributeError('%s does not have a field called %s' % (self.__class__.__name__, name)) + # Assign default values for fields not included in the keyword arguments for name, field in self._fields: - val = kwargs.get(name, field.default) - setattr(self, name, val) + if name not in kwargs: + setattr(self, name, field.default) def __setattr__(self, name, value): field = self.get_field(name) diff --git a/tests/test_models.py b/tests/test_models.py index 7725172..4b259a9 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -10,7 +10,7 @@ from infi.clickhouse_orm.engines import * class ModelTestCase(unittest.TestCase): def test_defaults(self): - # Check that all fields have their defaults + # Check that all fields have their explicit or implicit defaults instance = SimpleModel() self.assertEquals(instance.date_field, datetime.date(1970, 1, 1)) self.assertEquals(instance.datetime_field, datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)) @@ -31,6 +31,20 @@ class ModelTestCase(unittest.TestCase): for name, value in kwargs.items(): self.assertEquals(kwargs[name], getattr(instance, name)) + def test_assignment_error(self): + # Check non-existing field during construction + with self.assertRaises(AttributeError): + instance = SimpleModel(int_field=7450, pineapple='tasty') + # Check invalid field values during construction + with self.assertRaises(ValueError): + instance = SimpleModel(int_field='nope') + with self.assertRaises(ValueError): + instance = SimpleModel(date_field='nope') + # Check invalid field values during assignment + instance = SimpleModel() + with self.assertRaises(ValueError): + instance.datetime_field = datetime.timedelta(days=1) + def test_string_conversion(self): # Check field conversion from string during construction instance = SimpleModel(date_field='1973-12-06', int_field='100', float_field='7') From 9262f0eae6a05bfe65e5ef9d861b37af6646f0b4 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Thu, 23 Jun 2016 19:05:44 +0300 Subject: [PATCH 08/14] add database.select and field range checks --- src/infi/clickhouse_orm/database.py | 8 +- src/infi/clickhouse_orm/fields.py | 42 +++++- src/infi/clickhouse_orm/models.py | 1 + src/infi/clickhouse_orm/utils.py | 2 +- tests/test_database.py | 200 +++++++++++++++------------- 5 files changed, 150 insertions(+), 103 deletions(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index b7ae30a..0151c50 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -46,9 +46,15 @@ class Database(object): r = self._send(query) return int(r.text) if r.text else 0 + def select(self, query, model_class=None, settings=None): + query += ' FORMAT TabSeparated' + r = self._send(query, settings) + for line in r.iter_lines(): + yield model_class.from_tsv(line) + def _send(self, data, settings=None): params = self._build_params(settings) - r = requests.post(self.db_url, params=params, data=data) + r = requests.post(self.db_url, params=params, data=data, stream=True) if r.status_code != 200: raise DatabaseException(r.text) return r diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index 91b0e50..c90363e 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -21,6 +21,13 @@ class Field(object): """ return value + def validate(self, value): + pass + + def _range_check(self, value, min_value, max_value): + if value < min_value or value > max_value: + raise ValueError('%s out of range - %s is not between %s and %s' % (self.__class__.__name__, value, min_value, max_value)) + def get_db_prep_value(self, value): """ Returns the field's value prepared for interacting with the database. @@ -48,7 +55,9 @@ class StringField(Field): class DateField(Field): - class_default = datetime.date(1970, 1, 1) + min_value = datetime.date(1970, 1, 1) + max_value = datetime.date(2038, 1, 19) + class_default = min_value db_type = 'Date' def to_python(self, value): @@ -57,8 +66,12 @@ class DateField(Field): if isinstance(value, int): return DateField.class_default + datetime.timedelta(days=value) if isinstance(value, basestring): + # TODO parse '0000-00-00' return datetime.datetime.strptime(value, '%Y-%m-%d').date() - raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) + raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + + def validate(self, value): + self._range_check(value, DateField.min_value, DateField.max_value) def get_db_prep_value(self, value): return value.isoformat() @@ -78,7 +91,7 @@ class DateTimeField(Field): return datetime.datetime.fromtimestamp(value, pytz.utc) if isinstance(value, basestring): return datetime.datetime.strptime(value, '%Y-%m-%d %H-%M-%S') - raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) + raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) def get_db_prep_value(self, value): return int(time.mktime(value.timetuple())) @@ -91,46 +104,65 @@ class BaseIntField(Field): return value if isinstance(value, basestring): return int(value) - raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) + raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + + def validate(self, value): + self._range_check(value, self.min_value, self.max_value) class UInt8Field(BaseIntField): + min_value = 0 + max_value = 2**8 - 1 db_type = 'UInt8' class UInt16Field(BaseIntField): + min_value = 0 + max_value = 2**16 - 1 db_type = 'UInt16' class UInt32Field(BaseIntField): + min_value = 0 + max_value = 2**32 - 1 db_type = 'UInt32' class UInt64Field(BaseIntField): + min_value = 0 + max_value = 2**64 - 1 db_type = 'UInt64' class Int8Field(BaseIntField): + min_value = -2**7 + max_value = 2**7 - 1 db_type = 'Int8' class Int16Field(BaseIntField): + min_value = -2**16 + max_value = 2**16 - 1 db_type = 'Int16' class Int32Field(BaseIntField): + min_value = -2**32 + max_value = 2**32 - 1 db_type = 'Int32' class Int64Field(BaseIntField): + min_value = -2**64 + max_value = 2**64 - 1 db_type = 'Int64' @@ -141,7 +173,7 @@ class BaseFloatField(Field): return value if isinstance(value, basestring) or isinstance(value, int): return float(value) - raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) + raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) class Float32Field(BaseFloatField): diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index e5d41fd..8f58818 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -49,6 +49,7 @@ class Model(object): field = self.get_field(name) if field: value = field.to_python(value) + field.validate(value) super(Model, self).__setattr__(name, value) def get_field(self, name): diff --git a/src/infi/clickhouse_orm/utils.py b/src/infi/clickhouse_orm/utils.py index 5f49778..62c61e2 100644 --- a/src/infi/clickhouse_orm/utils.py +++ b/src/infi/clickhouse_orm/utils.py @@ -25,4 +25,4 @@ def unescape(value): def parse_tsv(line): if line[-1] == '\n': line = line[:-1] - return [_unescape(value) for value in line.split('\t')] + return [unescape(value) for value in line.split('\t')] diff --git a/tests/test_database.py b/tests/test_database.py index c368e7a..1d89e9a 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -39,6 +39,14 @@ class DatabaseTestCase(unittest.TestCase): self.assertEquals(self.database.count(Person, "birthday > '2000-01-01'"), 22) self.assertEquals(self.database.count(Person, "birthday < '1900-01-01'"), 0) + def test_select(self): + self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT * FROM test_db.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = list(self.database.select(query, Person)) + self.assertEquals(len(results), 2) + self.assertEquals(results[0].last_name, 'Durham') + self.assertEquals(results[1].last_name, 'Scott') + def _sample_data(self): for entry in data: yield Person(**entry) @@ -55,104 +63,104 @@ class Person(Model): data = [ - {"first_name": "Courtney", "last_name": "Cannon", "birthday": "1967-10-26", "height": "1.76"}, - {"first_name": "Kieran", "last_name": "Solomon", "birthday": "1963-10-30", "height": "1.69"}, - {"first_name": "Wynter", "last_name": "Garcia", "birthday": "1945-01-10", "height": "1.69"}, - {"first_name": "Buffy", "last_name": "Webb", "birthday": "1960-03-06", "height": "1.68"}, - {"first_name": "Jarrod", "last_name": "Gibbs", "birthday": "1957-06-13", "height": "1.62"}, - {"first_name": "Cara", "last_name": "Fox", "birthday": "2004-05-15", "height": "1.71"}, - {"first_name": "Shoshana", "last_name": "Solis", "birthday": "1968-07-18", "height": "1.65"}, - {"first_name": "Adam", "last_name": "Goodman", "birthday": "1956-01-07", "height": "1.74"}, - {"first_name": "Ciaran", "last_name": "Hurley", "birthday": "1965-10-25", "height": "1.65"}, - {"first_name": "Emma", "last_name": "Clements", "birthday": "1996-08-07", "height": "1.75"}, - {"first_name": "Warren", "last_name": "Dudley", "birthday": "1965-10-23", "height": "1.59"}, - {"first_name": "Courtney", "last_name": "Hoffman", "birthday": "1994-11-07", "height": "1.65"}, - {"first_name": "Karleigh", "last_name": "Bartlett", "birthday": "1991-10-24", "height": "1.69"}, - {"first_name": "Aline", "last_name": "Crane", "birthday": "1958-05-01", "height": "1.62"}, - {"first_name": "Sawyer", "last_name": "Fischer", "birthday": "1995-04-01", "height": "1.78"}, - {"first_name": "Ella", "last_name": "Castillo", "birthday": "1943-03-28", "height": "1.73"}, - {"first_name": "Jesse", "last_name": "Gomez", "birthday": "2011-01-28", "height": "1.71"}, - {"first_name": "Mary", "last_name": "Kirkland", "birthday": "1987-10-09", "height": "1.73"}, - {"first_name": "Quon", "last_name": "Wiggins", "birthday": "1962-05-06", "height": "1.74"}, - {"first_name": "Dominique", "last_name": "Sandoval", "birthday": "1942-02-01", "height": "1.72"}, - {"first_name": "Elton", "last_name": "Smith", "birthday": "1952-06-20", "height": "1.66"}, - {"first_name": "Whitney", "last_name": "Scott", "birthday": "1941-07-04", "height": "1.70"}, - {"first_name": "Sydney", "last_name": "Stevens", "birthday": "1959-07-11", "height": "1.70"}, - {"first_name": "Lesley", "last_name": "Stephenson", "birthday": "2010-04-10", "height": "1.64"}, - {"first_name": "Georgia", "last_name": "Kennedy", "birthday": "1944-12-29", "height": "1.66"}, - {"first_name": "Norman", "last_name": "Santos", "birthday": "1959-01-10", "height": "1.68"}, - {"first_name": "Vaughan", "last_name": "Schmidt", "birthday": "1955-06-19", "height": "1.61"}, - {"first_name": "Edan", "last_name": "Dennis", "birthday": "1959-09-18", "height": "1.73"}, - {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1967-02-25", "height": "1.74"}, - {"first_name": "Seth", "last_name": "Serrano", "birthday": "2017-06-02", "height": "1.71"}, - {"first_name": "Callie", "last_name": "Wiley", "birthday": "1957-11-24", "height": "1.69"}, - {"first_name": "Shana", "last_name": "Jarvis", "birthday": "1967-05-21", "height": "1.72"}, - {"first_name": "Madeline", "last_name": "Kidd", "birthday": "1954-12-09", "height": "1.69"}, - {"first_name": "Macaulay", "last_name": "Rowe", "birthday": "1952-03-02", "height": "1.68"}, - {"first_name": "Rafael", "last_name": "Parker", "birthday": "2016-01-24", "height": "1.76"}, - {"first_name": "Ava", "last_name": "Sanders", "birthday": "1967-08-10", "height": "1.60"}, - {"first_name": "Ciaran", "last_name": "Carver", "birthday": "2016-12-25", "height": "1.76"}, - {"first_name": "Hyacinth", "last_name": "Kent", "birthday": "1971-07-18", "height": "1.72"}, - {"first_name": "Velma", "last_name": "English", "birthday": "1999-01-18", "height": "1.65"}, - {"first_name": "Adena", "last_name": "Norman", "birthday": "1949-05-14", "height": "1.66"}, - {"first_name": "Hedy", "last_name": "Strong", "birthday": "2001-10-04", "height": "1.60"}, - {"first_name": "Griffith", "last_name": "Henry", "birthday": "1955-04-02", "height": "1.66"}, - {"first_name": "Barrett", "last_name": "Clemons", "birthday": "1985-07-03", "height": "1.71"}, - {"first_name": "Roary", "last_name": "Simmons", "birthday": "1956-07-23", "height": "1.63"}, - {"first_name": "Connor", "last_name": "Jenkins", "birthday": "1969-07-23", "height": "1.67"}, - {"first_name": "Cassady", "last_name": "Knapp", "birthday": "1947-12-15", "height": "1.72"}, {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63"}, - {"first_name": "Evangeline", "last_name": "Weber", "birthday": "1984-06-03", "height": "1.70"}, - {"first_name": "Clementine", "last_name": "Moon", "birthday": "1964-03-29", "height": "1.73"}, - {"first_name": "Montana", "last_name": "Bruce", "birthday": "1952-06-28", "height": "1.66"}, - {"first_name": "Whitney", "last_name": "Durham", "birthday": "1947-09-15", "height": "1.72"}, - {"first_name": "Naomi", "last_name": "Hays", "birthday": "2004-11-27", "height": "1.70"}, - {"first_name": "Macey", "last_name": "Griffin", "birthday": "1971-09-18", "height": "1.63"}, - {"first_name": "Dora", "last_name": "Cabrera", "birthday": "2016-04-26", "height": "1.68"}, - {"first_name": "Chelsea", "last_name": "Castro", "birthday": "2001-08-10", "height": "1.71"}, - {"first_name": "Reese", "last_name": "Noel", "birthday": "1966-11-04", "height": "1.77"}, + {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74"}, + {"first_name": "Adena", "last_name": "Norman", "birthday": "1979-05-14", "height": "1.66"}, + {"first_name": "Aline", "last_name": "Crane", "birthday": "1988-05-01", "height": "1.62"}, {"first_name": "Althea", "last_name": "Barrett", "birthday": "2004-07-28", "height": "1.71"}, - {"first_name": "Odette", "last_name": "Mcneil", "birthday": "1948-05-21", "height": "1.76"}, - {"first_name": "Oliver", "last_name": "Ashley", "birthday": "2004-08-13", "height": "1.68"}, - {"first_name": "Ashton", "last_name": "Fuller", "birthday": "1995-11-17", "height": "1.75"}, - {"first_name": "Denton", "last_name": "Sanchez", "birthday": "1971-10-16", "height": "1.72"}, - {"first_name": "Molly", "last_name": "Salas", "birthday": "1994-04-23", "height": "1.70"}, - {"first_name": "Josiah", "last_name": "Hodges", "birthday": "2011-09-04", "height": "1.68"}, - {"first_name": "Victoria", "last_name": "Slater", "birthday": "2009-07-19", "height": "1.72"}, - {"first_name": "Catherine", "last_name": "Hicks", "birthday": "1959-05-23", "height": "1.80"}, - {"first_name": "Sharon", "last_name": "Shelton", "birthday": "1970-05-02", "height": "1.65"}, - {"first_name": "Rigel", "last_name": "Oneal", "birthday": "1993-11-05", "height": "1.63"}, - {"first_name": "Victor", "last_name": "Woods", "birthday": "1959-06-23", "height": "1.67"}, - {"first_name": "Cathleen", "last_name": "Frank", "birthday": "1947-09-04", "height": "1.61"}, - {"first_name": "Russell", "last_name": "Pruitt", "birthday": "1979-05-04", "height": "1.63"}, - {"first_name": "Warren", "last_name": "Bowen", "birthday": "2000-07-20", "height": "1.76"}, - {"first_name": "Hu", "last_name": "May", "birthday": "1976-10-01", "height": "1.76"}, - {"first_name": "Amanda", "last_name": "Vang", "birthday": "1943-02-23", "height": "1.68"}, - {"first_name": "Fritz", "last_name": "Atkinson", "birthday": "2011-06-15", "height": "1.73"}, - {"first_name": "Scarlett", "last_name": "Durham", "birthday": "2005-09-29", "height": "1.65"}, - {"first_name": "Leroy", "last_name": "Pacheco", "birthday": "1968-12-30", "height": "1.70"}, - {"first_name": "Eagan", "last_name": "Dodson", "birthday": "2015-10-22", "height": "1.67"}, - {"first_name": "Whilemina", "last_name": "Blankenship", "birthday": "1970-07-14", "height": "1.66"}, - {"first_name": "Anne", "last_name": "Rasmussen", "birthday": "1995-04-03", "height": "1.77"}, - {"first_name": "Caryn", "last_name": "Sears", "birthday": "1999-02-17", "height": "1.71"}, - {"first_name": "Elton", "last_name": "Ayers", "birthday": "1994-06-20", "height": "1.68"}, - {"first_name": "Venus", "last_name": "Hurst", "birthday": "1963-10-22", "height": "1.72"}, - {"first_name": "Octavius", "last_name": "Floyd", "birthday": "1955-02-22", "height": "1.68"}, - {"first_name": "Keelie", "last_name": "Mathis", "birthday": "1963-10-26", "height": "1.69"}, - {"first_name": "Wang", "last_name": "Goodwin", "birthday": "1983-05-15", "height": "1.66"}, - {"first_name": "Rhona", "last_name": "Camacho", "birthday": "1946-12-17", "height": "1.59"}, - {"first_name": "Ulla", "last_name": "Arnold", "birthday": "1990-06-04", "height": "1.63"}, - {"first_name": "Idola", "last_name": "Fulton", "birthday": "1974-11-27", "height": "1.66"}, - {"first_name": "Galvin", "last_name": "Phillips", "birthday": "2004-01-17", "height": "1.74"}, - {"first_name": "Faith", "last_name": "Emerson", "birthday": "1959-12-30", "height": "1.62"}, - {"first_name": "Shad", "last_name": "Bradshaw", "birthday": "1968-08-25", "height": "1.72"}, - {"first_name": "Stephen", "last_name": "Baxter", "birthday": "2004-09-24", "height": "1.74"}, - {"first_name": "Tasha", "last_name": "Campos", "birthday": "1984-02-11", "height": "1.72"}, - {"first_name": "Celeste", "last_name": "James", "birthday": "1990-03-08", "height": "1.67"}, - {"first_name": "Ariana", "last_name": "Cole", "birthday": "1947-12-20", "height": "1.72"}, - {"first_name": "Beatrice", "last_name": "Gregory", "birthday": "1992-01-19", "height": "1.80"}, - {"first_name": "Laith", "last_name": "Howell", "birthday": "1961-07-07", "height": "1.70"}, + {"first_name": "Amanda", "last_name": "Vang", "birthday": "1973-02-23", "height": "1.68"}, {"first_name": "Angela", "last_name": "Sanders", "birthday": "2016-01-08", "height": "1.66"}, + {"first_name": "Anne", "last_name": "Rasmussen", "birthday": "1995-04-03", "height": "1.77"}, + {"first_name": "Ariana", "last_name": "Cole", "birthday": "1977-12-20", "height": "1.72"}, + {"first_name": "Ashton", "last_name": "Fuller", "birthday": "1995-11-17", "height": "1.75"}, + {"first_name": "Ava", "last_name": "Sanders", "birthday": "1997-08-10", "height": "1.60"}, + {"first_name": "Barrett", "last_name": "Clemons", "birthday": "1985-07-03", "height": "1.71"}, + {"first_name": "Beatrice", "last_name": "Gregory", "birthday": "1992-01-19", "height": "1.80"}, + {"first_name": "Buffy", "last_name": "Webb", "birthday": "1990-03-06", "height": "1.68"}, + {"first_name": "Callie", "last_name": "Wiley", "birthday": "1987-11-24", "height": "1.69"}, + {"first_name": "Cara", "last_name": "Fox", "birthday": "2004-05-15", "height": "1.71"}, + {"first_name": "Caryn", "last_name": "Sears", "birthday": "1999-02-17", "height": "1.71"}, + {"first_name": "Cassady", "last_name": "Knapp", "birthday": "1977-12-15", "height": "1.72"}, {"first_name": "Cassady", "last_name": "Rogers", "birthday": "2013-11-04", "height": "1.71"}, - {"first_name": "Maia", "last_name": "Hyde", "birthday": "1972-06-09", "height": "1.74"} + {"first_name": "Catherine", "last_name": "Hicks", "birthday": "1989-05-23", "height": "1.80"}, + {"first_name": "Cathleen", "last_name": "Frank", "birthday": "1977-09-04", "height": "1.61"}, + {"first_name": "Celeste", "last_name": "James", "birthday": "1990-03-08", "height": "1.67"}, + {"first_name": "Chelsea", "last_name": "Castro", "birthday": "2001-08-10", "height": "1.71"}, + {"first_name": "Ciaran", "last_name": "Carver", "birthday": "2016-12-25", "height": "1.76"}, + {"first_name": "Ciaran", "last_name": "Hurley", "birthday": "1995-10-25", "height": "1.65"}, + {"first_name": "Clementine", "last_name": "Moon", "birthday": "1994-03-29", "height": "1.73"}, + {"first_name": "Connor", "last_name": "Jenkins", "birthday": "1999-07-23", "height": "1.67"}, + {"first_name": "Courtney", "last_name": "Cannon", "birthday": "1997-10-26", "height": "1.76"}, + {"first_name": "Courtney", "last_name": "Hoffman", "birthday": "1994-11-07", "height": "1.65"}, + {"first_name": "Denton", "last_name": "Sanchez", "birthday": "1971-10-16", "height": "1.72"}, + {"first_name": "Dominique", "last_name": "Sandoval", "birthday": "1972-02-01", "height": "1.72"}, + {"first_name": "Dora", "last_name": "Cabrera", "birthday": "2016-04-26", "height": "1.68"}, + {"first_name": "Eagan", "last_name": "Dodson", "birthday": "2015-10-22", "height": "1.67"}, + {"first_name": "Edan", "last_name": "Dennis", "birthday": "1989-09-18", "height": "1.73"}, + {"first_name": "Ella", "last_name": "Castillo", "birthday": "1973-03-28", "height": "1.73"}, + {"first_name": "Elton", "last_name": "Ayers", "birthday": "1994-06-20", "height": "1.68"}, + {"first_name": "Elton", "last_name": "Smith", "birthday": "1982-06-20", "height": "1.66"}, + {"first_name": "Emma", "last_name": "Clements", "birthday": "1996-08-07", "height": "1.75"}, + {"first_name": "Evangeline", "last_name": "Weber", "birthday": "1984-06-03", "height": "1.70"}, + {"first_name": "Faith", "last_name": "Emerson", "birthday": "1989-12-30", "height": "1.62"}, + {"first_name": "Fritz", "last_name": "Atkinson", "birthday": "2011-06-15", "height": "1.73"}, + {"first_name": "Galvin", "last_name": "Phillips", "birthday": "2004-01-17", "height": "1.74"}, + {"first_name": "Georgia", "last_name": "Kennedy", "birthday": "1974-12-29", "height": "1.66"}, + {"first_name": "Griffith", "last_name": "Henry", "birthday": "1985-04-02", "height": "1.66"}, + {"first_name": "Hedy", "last_name": "Strong", "birthday": "2001-10-04", "height": "1.60"}, + {"first_name": "Hu", "last_name": "May", "birthday": "1976-10-01", "height": "1.76"}, + {"first_name": "Hyacinth", "last_name": "Kent", "birthday": "1971-07-18", "height": "1.72"}, + {"first_name": "Idola", "last_name": "Fulton", "birthday": "1974-11-27", "height": "1.66"}, + {"first_name": "Jarrod", "last_name": "Gibbs", "birthday": "1987-06-13", "height": "1.62"}, + {"first_name": "Jesse", "last_name": "Gomez", "birthday": "2011-01-28", "height": "1.71"}, + {"first_name": "Josiah", "last_name": "Hodges", "birthday": "2011-09-04", "height": "1.68"}, + {"first_name": "Karleigh", "last_name": "Bartlett", "birthday": "1991-10-24", "height": "1.69"}, + {"first_name": "Keelie", "last_name": "Mathis", "birthday": "1993-10-26", "height": "1.69"}, + {"first_name": "Kieran", "last_name": "Solomon", "birthday": "1993-10-30", "height": "1.69"}, + {"first_name": "Laith", "last_name": "Howell", "birthday": "1991-07-07", "height": "1.70"}, + {"first_name": "Leroy", "last_name": "Pacheco", "birthday": "1998-12-30", "height": "1.70"}, + {"first_name": "Lesley", "last_name": "Stephenson", "birthday": "2010-04-10", "height": "1.64"}, + {"first_name": "Macaulay", "last_name": "Rowe", "birthday": "1982-03-02", "height": "1.68"}, + {"first_name": "Macey", "last_name": "Griffin", "birthday": "1971-09-18", "height": "1.63"}, + {"first_name": "Madeline", "last_name": "Kidd", "birthday": "1984-12-09", "height": "1.69"}, + {"first_name": "Maia", "last_name": "Hyde", "birthday": "1972-06-09", "height": "1.74"}, + {"first_name": "Mary", "last_name": "Kirkland", "birthday": "1987-10-09", "height": "1.73"}, + {"first_name": "Molly", "last_name": "Salas", "birthday": "1994-04-23", "height": "1.70"}, + {"first_name": "Montana", "last_name": "Bruce", "birthday": "1982-06-28", "height": "1.66"}, + {"first_name": "Naomi", "last_name": "Hays", "birthday": "2004-11-27", "height": "1.70"}, + {"first_name": "Norman", "last_name": "Santos", "birthday": "1989-01-10", "height": "1.68"}, + {"first_name": "Octavius", "last_name": "Floyd", "birthday": "1985-02-22", "height": "1.68"}, + {"first_name": "Odette", "last_name": "Mcneil", "birthday": "1978-05-21", "height": "1.76"}, + {"first_name": "Oliver", "last_name": "Ashley", "birthday": "2004-08-13", "height": "1.68"}, + {"first_name": "Quon", "last_name": "Wiggins", "birthday": "1992-05-06", "height": "1.74"}, + {"first_name": "Rafael", "last_name": "Parker", "birthday": "2016-01-24", "height": "1.76"}, + {"first_name": "Reese", "last_name": "Noel", "birthday": "1996-11-04", "height": "1.77"}, + {"first_name": "Rhona", "last_name": "Camacho", "birthday": "1976-12-17", "height": "1.59"}, + {"first_name": "Rigel", "last_name": "Oneal", "birthday": "1993-11-05", "height": "1.63"}, + {"first_name": "Roary", "last_name": "Simmons", "birthday": "1986-07-23", "height": "1.63"}, + {"first_name": "Russell", "last_name": "Pruitt", "birthday": "1979-05-04", "height": "1.63"}, + {"first_name": "Sawyer", "last_name": "Fischer", "birthday": "1995-04-01", "height": "1.78"}, + {"first_name": "Scarlett", "last_name": "Durham", "birthday": "2005-09-29", "height": "1.65"}, + {"first_name": "Seth", "last_name": "Serrano", "birthday": "2017-06-02", "height": "1.71"}, + {"first_name": "Shad", "last_name": "Bradshaw", "birthday": "1998-08-25", "height": "1.72"}, + {"first_name": "Shana", "last_name": "Jarvis", "birthday": "1997-05-21", "height": "1.72"}, + {"first_name": "Sharon", "last_name": "Shelton", "birthday": "1970-05-02", "height": "1.65"}, + {"first_name": "Shoshana", "last_name": "Solis", "birthday": "1998-07-18", "height": "1.65"}, + {"first_name": "Stephen", "last_name": "Baxter", "birthday": "2004-09-24", "height": "1.74"}, + {"first_name": "Sydney", "last_name": "Stevens", "birthday": "1989-07-11", "height": "1.70"}, + {"first_name": "Tasha", "last_name": "Campos", "birthday": "1984-02-11", "height": "1.72"}, + {"first_name": "Ulla", "last_name": "Arnold", "birthday": "1990-06-04", "height": "1.63"}, + {"first_name": "Vaughan", "last_name": "Schmidt", "birthday": "1985-06-19", "height": "1.61"}, + {"first_name": "Velma", "last_name": "English", "birthday": "1999-01-18", "height": "1.65"}, + {"first_name": "Venus", "last_name": "Hurst", "birthday": "1993-10-22", "height": "1.72"}, + {"first_name": "Victor", "last_name": "Woods", "birthday": "1989-06-23", "height": "1.67"}, + {"first_name": "Victoria", "last_name": "Slater", "birthday": "2009-07-19", "height": "1.72"}, + {"first_name": "Wang", "last_name": "Goodwin", "birthday": "1983-05-15", "height": "1.66"}, + {"first_name": "Warren", "last_name": "Bowen", "birthday": "2000-07-20", "height": "1.76"}, + {"first_name": "Warren", "last_name": "Dudley", "birthday": "1995-10-23", "height": "1.59"}, + {"first_name": "Whilemina", "last_name": "Blankenship", "birthday": "1970-07-14", "height": "1.66"}, + {"first_name": "Whitney", "last_name": "Durham", "birthday": "1977-09-15", "height": "1.72"}, + {"first_name": "Whitney", "last_name": "Scott", "birthday": "1971-07-04", "height": "1.70"}, + {"first_name": "Wynter", "last_name": "Garcia", "birthday": "1975-01-10", "height": "1.69"}, + {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1997-02-25", "height": "1.74"} ]; From 92ea9d413e177bba8eacf780728087f496eadf33 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 26 Jun 2016 15:11:16 +0300 Subject: [PATCH 09/14] support ad-hoc models --- src/infi/clickhouse_orm/database.py | 12 +++++++++--- src/infi/clickhouse_orm/models.py | 22 ++++++++++++++++++---- tests/test_database.py | 25 ++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 0151c50..2472a58 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -1,4 +1,6 @@ import requests +from models import ModelBase +from utils import escape, parse_tsv class DatabaseException(Exception): @@ -47,10 +49,14 @@ class Database(object): return int(r.text) if r.text else 0 def select(self, query, model_class=None, settings=None): - query += ' FORMAT TabSeparated' + query += ' FORMAT TabSeparatedWithNamesAndTypes' r = self._send(query, settings) - for line in r.iter_lines(): - yield model_class.from_tsv(line) + lines = r.iter_lines() + field_names = parse_tsv(next(lines)) + field_types = parse_tsv(next(lines)) + model_class = model_class or ModelBase.create_ad_hoc_model(zip(field_names, field_types)) + for line in lines: + yield model_class.from_tsv(line, field_names) def _send(self, data, settings=None): params = self._build_params(settings) diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index 8f58818..e4a4209 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -1,6 +1,6 @@ -from fields import * from utils import escape, parse_tsv from engines import * +from fields import Field class ModelBase(type): @@ -16,6 +16,18 @@ class ModelBase(type): setattr(new_cls, '_fields', fields) return new_cls + @classmethod + def create_ad_hoc_model(cls, fields): + # fields is a list of tuples (name, db_type) + import fields as orm_fields + attrs = {} + for name, db_type in fields: + field_class = db_type + 'Field' + if not hasattr(orm_fields, field_class): + raise NotImplementedError('No field class for %s' % db_type) + attrs[name] = getattr(orm_fields, field_class)() + return cls.__new__(cls, 'AdHocModel', (Model,), attrs) + class Model(object): ''' @@ -77,16 +89,18 @@ class Model(object): return 'DROP TABLE IF EXISTS %s.%s' % (db_name, cls.table_name()) @classmethod - def from_tsv(cls, line): + def from_tsv(cls, line, field_names=None): ''' Create a model instance from a tab-separated line. The line may or may not include a newline. + The field_names list must match the fields defined in the model, but does not have to include all of them. + If omitted, it is assumed to be the names of all fields in the model, in order of definition. ''' + field_names = field_names or [name for name, field in cls._fields] values = iter(parse_tsv(line)) kwargs = {} - for name, field in cls._fields: + for name in field_names: kwargs[name] = values.next() return cls(**kwargs) - # TODO verify that the number of values matches the number of fields def to_tsv(self): ''' diff --git a/tests/test_database.py b/tests/test_database.py index 1d89e9a..fa6b7cc 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -37,7 +37,7 @@ class DatabaseTestCase(unittest.TestCase): self.assertEquals(self.database.count(Person), 100) self.assertEquals(self.database.count(Person, "first_name = 'Courtney'"), 2) self.assertEquals(self.database.count(Person, "birthday > '2000-01-01'"), 22) - self.assertEquals(self.database.count(Person, "birthday < '1900-01-01'"), 0) + self.assertEquals(self.database.count(Person, "birthday < '1970-03-01'"), 0) def test_select(self): self._insert_and_check(self._sample_data(), len(data)) @@ -45,7 +45,30 @@ class DatabaseTestCase(unittest.TestCase): results = list(self.database.select(query, Person)) self.assertEquals(len(results), 2) self.assertEquals(results[0].last_name, 'Durham') + self.assertEquals(results[0].height, 1.72) self.assertEquals(results[1].last_name, 'Scott') + self.assertEquals(results[1].height, 1.70) + + def test_select_partial_fields(self): + self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT first_name, last_name FROM test_db.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = list(self.database.select(query, Person)) + self.assertEquals(len(results), 2) + self.assertEquals(results[0].last_name, 'Durham') + self.assertEquals(results[0].height, 0) # default value + self.assertEquals(results[1].last_name, 'Scott') + self.assertEquals(results[1].height, 0) # default value + + def test_select_ad_hoc_model(self): + self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT * FROM test_db.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = list(self.database.select(query)) + self.assertEquals(len(results), 2) + self.assertEquals(results[0].__class__.__name__, 'AdHocModel') + self.assertEquals(results[0].last_name, 'Durham') + self.assertEquals(results[0].height, 1.72) + self.assertEquals(results[1].last_name, 'Scott') + self.assertEquals(results[1].height, 1.70) def _sample_data(self): for entry in data: From 27ee24843a55c81827ad4652e9ba7ad1ca5d2c0d Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 26 Jun 2016 15:11:23 +0300 Subject: [PATCH 10/14] documentation --- README.md | 15 ------- README.rst | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 15 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index 9e79596..0000000 --- a/README.md +++ /dev/null @@ -1,15 +0,0 @@ -Overview -======== -This is an Infinidat project. - -Usage ------ -Nothing to use here. - -Checking out the code -===================== -Run the following commands: - - easy_install -U infi.projector - projector devenv build - diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..d161f2a --- /dev/null +++ b/README.rst @@ -0,0 +1,122 @@ +Overview +======== + +This project is simple ORM for working with the `ClickHouse database `_. +It allows you to define model classes whose instances can be written to the database and read from it. + +Installation +============ + +To install infi.clickhouse_orm:: + + pip install infi.clickhouse_orm + +Usage +===== + +Defining Models +--------------- + +Models are defined in a way reminiscent of Django's ORM: + +.. code:: python + + from infi.clickhouse_orm import models, fields, engines + + class Person(models.Model): + + first_name = fields.StringField() + last_name = fields.StringField() + birthday = fields.DateField() + height = fields.Float32Field() + + engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday')) + +It is possible to provide a default value for a field, instead of it's "natural" default (empty string for string fields, zero for numeric fields etc.). + +See below for the supported model field types. + +Using Models +------------ + +Once you have a model, you can create model instances: + +.. code:: python + + >>> dan = Person(first_name='Dan', last_name='Schwartz') + >>> suzy = Person(first_name='Suzy', last_name='Jones') + >>> dan.first_name + u'Dan' + +When values are assigned to a model fields, they are immediately converted to their Pythonic data type. +In case the value is invalid, a ValueError is raised: + +.. code:: python + + >>> suzy.birthday = '1980-01-17' + >>> suzy.birthday + datetime.date(1980, 1, 17) + >>> suzy.birthday = 0.5 + ValueError: Invalid value for DateField - 0.5 + >>> suzy.birthday = '1922-05-31' + ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2038-01-19 + +To write your instances to ClickHouse, you need a Database instance: + +.. code:: python + + from infi.clickhouse_orm.database import Database + + db = Database('my_test_db') + +This automatically connects to http://localhost:8123 and creates a database called my_test_db, unless it already exists. +If necessary, you can specify a different database URL and optional credentials: + +.. code:: python + + db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') + +Using the Database instance you can create a table for your model, and insert instances to it: + +.. code:: python + + db.create_table(Person) + db.insert([dan, suzy]) + +The insert method can take any iterable of model instances, but they all must belong to the same model class. + + + +Field Types +----------- + +Currently the following field types are supported: + +- UInt8Field +- UInt16Field +- UInt32Field +- UInt64Field +- Int8Field +- Int16Field +- Int32Field +- Int64Field +- Float32Field +- Float64Field +- StringField +- DateField +- DateTimeField + + + +Development +=========== + +After cloning the project, run the following commands:: + + easy_install -U infi.projector + cd infi.clickhouse_orm + projector devenv build + +To run the tests, ensure that the ClickHouse server is running on http://localhost:8123/ (this is the default), and run:: + + bin/nosetests From 64f8cde1c0323cb2c6a2d94b42432efb83c41615 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 26 Jun 2016 16:52:25 +0300 Subject: [PATCH 11/14] documentation --- README.rst | 60 +++++++++++++++++++++++++---- src/infi/clickhouse_orm/database.py | 1 + src/infi/clickhouse_orm/fields.py | 15 ++++++-- src/infi/clickhouse_orm/models.py | 16 ++++++++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index d161f2a..ac48d5c 100644 --- a/README.rst +++ b/README.rst @@ -32,9 +32,9 @@ Models are defined in a way reminiscent of Django's ORM: engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday')) -It is possible to provide a default value for a field, instead of it's "natural" default (empty string for string fields, zero for numeric fields etc.). +It is possible to provide a default value for a field, instead of its "natural" default (empty string for string fields, zero for numeric fields etc.). -See below for the supported model field types. +See below for the supported field types and table engines. Using Models ------------ @@ -48,8 +48,8 @@ Once you have a model, you can create model instances: >>> dan.first_name u'Dan' -When values are assigned to a model fields, they are immediately converted to their Pythonic data type. -In case the value is invalid, a ValueError is raised: +When values are assigned to model fields, they are immediately converted to their Pythonic data type. +In case the value is invalid, a ``ValueError`` is raised: .. code:: python @@ -61,7 +61,10 @@ In case the value is invalid, a ValueError is raised: >>> suzy.birthday = '1922-05-31' ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2038-01-19 -To write your instances to ClickHouse, you need a Database instance: +Inserting to the Database +------------------------- + +To write your instances to ClickHouse, you need a ``Database`` instance: .. code:: python @@ -76,16 +79,53 @@ If necessary, you can specify a different database URL and optional credentials: db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') -Using the Database instance you can create a table for your model, and insert instances to it: +Using the ``Database`` instance you can create a table for your model, and insert instances to it: .. code:: python db.create_table(Person) db.insert([dan, suzy]) -The insert method can take any iterable of model instances, but they all must belong to the same model class. +The ``insert`` method can take any iterable of model instances, but they all must belong to the same model class. +Reading from the Database +------------------------- +Loading model instances from the database is simple: + +.. code:: python + + for person in db.select("SELECT * FROM my_test_db.person", model_class=Person): + print person.first_name, person.last_name + +Do not include a ``FORMAT`` clause in the query, since the ORM automatically sets the format to ``TabSeparatedWithNamesAndTypes``. + +It is possible to select only a subset of the columns, and the rest will receive their default values: + +.. code:: python + + for person in db.select("SELECT first_name FROM my_test_db.person WHERE last_name='Smith'", model_class=Person): + print person.first_name + +Specifying a model class is not required. In case you do not provide a model class, an ad-hoc class will +be defined based on the column names and types returned by the query: + +.. code:: python + + for row in db.select("SELECT max(height) as max_height FROM my_test_db.person"): + print row.max_height + +Counting +-------- + +The ``Database`` class also supports counting records easily: + +.. code:: python + + >>> db.count(Person) + 117 + >>> db.count(Person, conditions="height > 1.90") + 6 Field Types ----------- @@ -106,6 +146,10 @@ Currently the following field types are supported: - DateField - DateTimeField +Table Engines +------------- + +TBD Development @@ -119,4 +163,4 @@ After cloning the project, run the following commands:: To run the tests, ensure that the ClickHouse server is running on http://localhost:8123/ (this is the default), and run:: - bin/nosetests + bin/nosetests \ No newline at end of file diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 2472a58..b8c2e9f 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -17,6 +17,7 @@ class Database(object): self._send('CREATE DATABASE IF NOT EXISTS ' + db_name) def create_table(self, model_class): + # TODO check that model has an engine self._send(model_class.create_table_sql(self.db_name)) def drop_table(self, model_class): diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index c90363e..feebc96 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -15,23 +15,30 @@ class Field(object): self.default = default or self.class_default def to_python(self, value): - """ + ''' Converts the input value into the expected Python data type, raising ValueError if the data can't be converted. Returns the converted value. Subclasses should override this. - """ + ''' return value def validate(self, value): + ''' + Called after to_python to validate that the value is suitable for the field's database type. + Subclasses should override this. + ''' pass def _range_check(self, value, min_value, max_value): + ''' + Utility method to check that the given value is between min_value and max_value. + ''' if value < min_value or value > max_value: raise ValueError('%s out of range - %s is not between %s and %s' % (self.__class__.__name__, value, min_value, max_value)) def get_db_prep_value(self, value): - """ + ''' Returns the field's value prepared for interacting with the database. - """ + ''' return value diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index e4a4209..13dffa4 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -58,6 +58,10 @@ class Model(object): setattr(self, name, field.default) def __setattr__(self, name, value): + ''' + When setting a field value, converts the value to its Pythonic type and validates it. + This may raise a ValueError. + ''' field = self.get_field(name) if field: value = field.to_python(value) @@ -65,15 +69,24 @@ class Model(object): super(Model, self).__setattr__(name, value) def get_field(self, name): + ''' + Get a Field instance given its name, or None if not found. + ''' field = getattr(self.__class__, name, None) return field if isinstance(field, Field) else None @classmethod def table_name(cls): + ''' + Returns the model's database table name. + ''' return cls.__name__.lower() @classmethod def create_table_sql(cls, db_name): + ''' + Returns the SQL command for creating a table for this model. + ''' parts = ['CREATE TABLE IF NOT EXISTS %s.%s (' % (db_name, cls.table_name())] cols = [] for name, field in cls._fields: @@ -86,6 +99,9 @@ class Model(object): @classmethod def drop_table_sql(cls, db_name): + ''' + Returns the SQL command for deleting this model's table. + ''' return 'DROP TABLE IF EXISTS %s.%s' % (db_name, cls.table_name()) @classmethod From 5b033a1e1b6f65824a9e4c0e342190b2148798e8 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 26 Jun 2016 17:41:17 +0300 Subject: [PATCH 12/14] fix int ranges --- src/infi/clickhouse_orm/fields.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index feebc96..1dfe3ee 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -154,22 +154,22 @@ class Int8Field(BaseIntField): class Int16Field(BaseIntField): - min_value = -2**16 - max_value = 2**16 - 1 + min_value = -2**15 + max_value = 2**15 - 1 db_type = 'Int16' class Int32Field(BaseIntField): - min_value = -2**32 - max_value = 2**32 - 1 + min_value = -2**31 + max_value = 2**31 - 1 db_type = 'Int32' class Int64Field(BaseIntField): - min_value = -2**64 - max_value = 2**64 - 1 + min_value = -2**63 + max_value = 2**63 - 1 db_type = 'Int64' From 5510de597dd21b7ea03cfac689a95b11e62957da Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 26 Jun 2016 17:56:45 +0300 Subject: [PATCH 13/14] documentation --- README.rst | 64 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index ac48d5c..2af69de 100644 --- a/README.rst +++ b/README.rst @@ -132,25 +132,61 @@ Field Types Currently the following field types are supported: -- UInt8Field -- UInt16Field -- UInt32Field -- UInt64Field -- Int8Field -- Int16Field -- Int32Field -- Int64Field -- Float32Field -- Float64Field -- StringField -- DateField -- DateTimeField +============= ======== ================= =================================================== +Class DB Type Pythonic Type Comments +============= ======== ================= =================================================== +StringField String unicode Encoded as UTF-8 when written to ClickHouse +DateField Date datetime.date Range 1970-01-01 to 2038-01-19 +DateTimeField DateTime datetime.datetime Minimal value is 1970-01-01 00:00:00; Always in UTC +Int8Field Int8 int Range -128 to 127 +Int16Field Int16 int Range -32,768 to 32,767 +Int32Field Int32 int Range -2147483648 to 2147483647 +Int64Field Int64 int/long Range -9223372036854775808 to 9223372036854775807 +UInt8Field UInt8 int Range 0 to 255 +UInt16Field UInt16 int Range 0 to 65535 +UInt32Field UInt32 int Range 0 to 4294967295 +UInt64Field UInt64 int/long Range 0 to 18446744073709551615 +Float32Field Float32 float +Float64Field Float64 float +============= ======== ================= =================================================== Table Engines ------------- -TBD +Each model must have an engine instance, used when creating the table in ClickHouse. +To define a ``MergeTree`` engine, supply the date column name and the names (or expressions) for the key columns: + +.. code:: python + + engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate')) + +You may also provide a sampling expression: + +.. code:: python + + engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'), sampling_expr='intHash32(UserID)') + +A ``CollapsingMergeTree`` engine is defined in a similar manner, but requires also a sign column: + +.. code:: python + + engine = engines.CollapsingMergeTree('EventDate', ('CounterID', 'EventDate'), 'Sign') + +For a ``SummingMergeTree`` you can optionally specify the summing columns: + +.. code:: python + + engine = engines.SummingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), + summing_cols=('Shows', 'Clicks', 'Cost')) + +Any of the above engines can be converted to a replicated engine (e.g. ``ReplicatedMergeTree``) by adding two parameters, ``replica_table_path`` and ``replica_name``: + +.. code:: python + + engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'), + replica_table_path='/clickhouse/tables/{layer}-{shard}/hits', + replica_name='{replica}') Development =========== From 3e6677f93950cbfbfb0fec65eaffcd3e06e292a6 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sun, 26 Jun 2016 18:04:46 +0300 Subject: [PATCH 14/14] documentation --- README.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2af69de..8c8c107 100644 --- a/README.rst +++ b/README.rst @@ -107,6 +107,9 @@ It is possible to select only a subset of the columns, and the rest will receive for person in db.select("SELECT first_name FROM my_test_db.person WHERE last_name='Smith'", model_class=Person): print person.first_name +Ad-Hoc Models +************* + Specifying a model class is not required. In case you do not provide a model class, an ad-hoc class will be defined based on the column names and types returned by the query: @@ -115,6 +118,9 @@ be defined based on the column names and types returned by the query: for row in db.select("SELECT max(height) as max_height FROM my_test_db.person"): print row.max_height +This is a very convenient feature that saves you the need to define a model for each query, while still letting +you work with Pythonic column values and an elegant syntax. + Counting -------- @@ -139,7 +145,7 @@ StringField String unicode Encoded as UTF-8 when written to C DateField Date datetime.date Range 1970-01-01 to 2038-01-19 DateTimeField DateTime datetime.datetime Minimal value is 1970-01-01 00:00:00; Always in UTC Int8Field Int8 int Range -128 to 127 -Int16Field Int16 int Range -32,768 to 32,767 +Int16Field Int16 int Range -32768 to 32767 Int32Field Int32 int Range -2147483648 to 2147483647 Int64Field Int64 int/long Range -9223372036854775808 to 9223372036854775807 UInt8Field UInt8 int Range 0 to 255 @@ -180,6 +186,9 @@ For a ``SummingMergeTree`` you can optionally specify the summing columns: engine = engines.SummingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), summing_cols=('Shows', 'Clicks', 'Cost')) +Data Replication +**************** + Any of the above engines can be converted to a replicated engine (e.g. ``ReplicatedMergeTree``) by adding two parameters, ``replica_table_path`` and ``replica_name``: .. code:: python @@ -199,4 +208,4 @@ After cloning the project, run the following commands:: To run the tests, ensure that the ClickHouse server is running on http://localhost:8123/ (this is the default), and run:: - bin/nosetests \ No newline at end of file + bin/nosetests