diff --git a/.gitignore b/.gitignore index c1eacdb..ae23e11 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,5 @@ buildout.in src/infi/clickhouse_orm/__version__.py bootstrap.py + +htmldocs/ \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.md similarity index 81% rename from CHANGELOG.rst rename to CHANGELOG.md index e933d06..8da6ee6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ Change Log ========== +v0.9.0 +------ +- Major new feature: building model queries using QuerySets +- Refactor and expand the documentation +- Add support for FixedString fields +- Add support for more engine types: TinyLog, Log, Memory +- Bug fix: Do not send readonly=1 when connection is already in readonly mode + v0.8.2 ------ - Fix broken Python 3 support (M1hacka) diff --git a/README b/README deleted file mode 100644 index a1320b1..0000000 --- a/README +++ /dev/null @@ -1 +0,0 @@ -README.rst diff --git a/README.md b/README.md new file mode 100644 index 0000000..4021f28 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +Introduction +============ + +This project is simple ORM for working with the [ClickHouse database](https://clickhouse.yandex/). +It allows you to define model classes whose instances can be written to the database and read from it. + +Let's jump right in with a simple example of monitoring CPU usage. First we need to define the model class, +connect to the database and create a table for the model: + +```python +from infi.clickhouse_orm.database import Database +from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.fields import * +from infi.clickhouse_orm.engines import Memory + +class CPUStats(Model): + + timestamp = DateTimeField() + cpu_id = UInt16Field() + cpu_percent = Float32Field() + + engine = Memory() + +db = Database('demo') +db.create_table(CPUStats) +``` + +Now we can collect usage statistics per CPU, and write them to the database: + +```python +import psutil, time, datetime + +psutil.cpu_percent(percpu=True) # first sample should be discarded +while True: + time.sleep(1) + stats = psutil.cpu_percent(percpu=True) + timestamp = datetime.datetime.now() + db.insert([ + CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent) + for cpu_id, cpu_percent in enumerate(stats) + ]) +``` + +Querying the table is easy, using either the query builder or raw SQL: + +```python +# Calculate what percentage of the time CPU 1 was over 95% busy +total = CPUStats.objects_in(db).filter(cpu_id=1).count() +busy = CPUStats.objects_in(db).filter(cpu_id=1, cpu_percent__gt=95).count() +print 'CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total) + +# Calculate the average usage per CPU +for row in db.select('SELECT cpu_id, avg(cpu_percent) AS average FROM demo.cpustats GROUP BY cpu_id'): + print 'CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row) +``` + +To learn more please visit the [documentation](docs/toc.md). \ No newline at end of file diff --git a/README.rst b/README.rst deleted file mode 100644 index 6c2c6d9..0000000 --- a/README.rst +++ /dev/null @@ -1,435 +0,0 @@ -Overview -======== - -This project is simple ORM for working with the `ClickHouse database `_. -It allows you to define model classes whose instances can be written to the database and read from it. - -Installation -============ - -To install infi.clickhouse_orm:: - - pip install infi.clickhouse_orm - -Usage -===== - -Defining Models ---------------- - -Models are defined in a way reminiscent of Django's ORM:: - - from infi.clickhouse_orm import models, fields, engines - - class Person(models.Model): - - first_name = fields.StringField() - last_name = fields.StringField() - birthday = fields.DateField() - height = fields.Float32Field() - - engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday')) - -It is possible to provide a default value for a field, instead of its "natural" default (empty string for string fields, zero for numeric fields etc.). -Alternatively it is possible to pass alias or materialized parameters (see below for usage examples). -Only one of ``default``, ``alias`` and ``materialized`` parameters can be provided. - -See below for the supported field types and table engines. - -Table Names -*********** - -The table name used for the model is its class name, converted to lowercase. To override the default name, -implement the ``table_name`` method:: - - class Person(models.Model): - - ... - - @classmethod - def table_name(cls): - return 'people' - -Using Models ------------- - -Once you have a model, you can create model instances:: - - >>> dan = Person(first_name='Dan', last_name='Schwartz') - >>> suzy = Person(first_name='Suzy', last_name='Jones') - >>> dan.first_name - u'Dan' - -When values are assigned to model fields, they are immediately converted to their Pythonic data type. -In case the value is invalid, a ``ValueError`` is raised:: - - >>> suzy.birthday = '1980-01-17' - >>> suzy.birthday - datetime.date(1980, 1, 17) - >>> suzy.birthday = 0.5 - ValueError: Invalid value for DateField - 0.5 - >>> suzy.birthday = '1922-05-31' - ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2038-01-19 - -Inserting to the Database -------------------------- - -To write your instances to ClickHouse, you need a ``Database`` instance:: - - from infi.clickhouse_orm.database import Database - - db = Database('my_test_db') - -This automatically connects to http://localhost:8123 and creates a database called my_test_db, unless it already exists. -If necessary, you can specify a different database URL and optional credentials:: - - db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') - -Using the ``Database`` instance you can create a table for your model, and insert instances to it:: - - db.create_table(Person) - db.insert([dan, suzy]) - -The ``insert`` method can take any iterable of model instances, but they all must belong to the same model class. - -Creating a read-only database is also supported. Such a ``Database`` instance can only read data, and cannot -modify data or schemas:: - - db = Database('my_test_db', readonly=True) - -Reading from the Database -------------------------- - -Loading model instances from the database is simple:: - - for person in db.select("SELECT * FROM my_test_db.person", model_class=Person): - print person.first_name, person.last_name - -Do not include a ``FORMAT`` clause in the query, since the ORM automatically sets the format to ``TabSeparatedWithNamesAndTypes``. - -It is possible to select only a subset of the columns, and the rest will receive their default values:: - - for person in db.select("SELECT first_name FROM my_test_db.person WHERE last_name='Smith'", model_class=Person): - print person.first_name - -SQL Placeholders -**************** - -There are a couple of special placeholders that you can use inside the SQL to make it easier to write: -``$db`` and ``$table``. The first one is replaced by the database name, and the second is replaced by -the database name plus table name (but is available only when the model is specified). - -So instead of this:: - - db.select("SELECT * FROM my_test_db.person", model_class=Person) - -you can use:: - - db.select("SELECT * FROM $db.person", model_class=Person) - -or even:: - - db.select("SELECT * FROM $table", model_class=Person) - -Ad-Hoc Models -************* - -Specifying a model class is not required. In case you do not provide a model class, an ad-hoc class will -be defined based on the column names and types returned by the query:: - - for row in db.select("SELECT max(height) as max_height FROM my_test_db.person"): - print row.max_height - -This is a very convenient feature that saves you the need to define a model for each query, while still letting -you work with Pythonic column values and an elegant syntax. - -Counting --------- - -The ``Database`` class also supports counting records easily:: - - >>> db.count(Person) - 117 - >>> db.count(Person, conditions="height > 1.90") - 6 - -Pagination ----------- - -It is possible to paginate through model instances:: - - >>> order_by = 'first_name, last_name' - >>> page = db.paginate(Person, order_by, page_num=1, page_size=10) - >>> print page.number_of_objects - 2507 - >>> print page.pages_total - 251 - >>> for person in page.objects: - >>> # do something - -The ``paginate`` method returns a ``namedtuple`` containing the following fields: - -- ``objects`` - the list of objects in this page -- ``number_of_objects`` - total number of objects in all pages -- ``pages_total`` - total number of pages -- ``number`` - the page number, starting from 1; the special value -1 may be used to retrieve the last page -- ``page_size`` - the number of objects per page - -You can optionally pass conditions to the query:: - - >>> page = db.paginate(Person, order_by, page_num=1, page_size=100, conditions='height > 1.90') - -Note that ``order_by`` must be chosen so that the ordering is unique, otherwise there might be -inconsistencies in the pagination (such as an instance that appears on two different pages). - - -System models -------------- - -`Clickhouse docs `_. - -System models are read only models for implementing part of the system's functionality, -and for providing access to information about how the system is working. - -Currently the following system models are supported: - -=================== ============ =================================================== -Class DB Table Comments -=================== ============ =================================================== -SystemPart system.parts Gives methods to work with partitions. See below. -=================== ============ =================================================== - - -Partitions and parts --------------------- - -`ClickHouse docs `_. - -A partition in a table is data for a single calendar month. Table "system.parts" contains information about each part. - -=================== ======================= ============================================================================================= -Method Parameters Comments -=================== ======================= ============================================================================================= -get(static) database, conditions="" Gets database partitions, filtered by conditions -get_active(static) database, conditions="" Gets only active (not detached or dropped) partitions, filtered by conditions -detach settings=None Detaches the partition. Settings is a dict of params to pass to http request -drop settings=None Drops the partition. Settings is a dict of params to pass to http request -attach settings=None Attaches already detached partition. Settings is a dict of params to pass to http request -freeze settings=None Freezes (makes backup) of the partition. Settings is a dict of params to pass to http request -fetch settings=None Fetches partition. Settings is a dict of params to pass to http request -=================== ======================= ============================================================================================= - -Usage example:: - - from infi.clickhouse_orm.database import Database - from infi.clickhouse_orm.system_models import SystemPart - db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') - partitions = SystemPart.get_active(db, conditions='') # Getting all active partitions of the database - if len(partitions) > 0: - partitions = sorted(partitions, key=lambda obj: obj.name) # Partition name is YYYYMM, so we can sort so - partitions[0].freeze() # Make a backup in /opt/clickhouse/shadow directory - partitions[0].drop() # Dropped partition - -``Note``: system.parts stores information for all databases. To be correct, -SystemPart model was designed to receive only given database parts. - - -Schema Migrations ------------------ - -Over time, your models may change and the database will have to be modified accordingly. -Migrations allow you to describe these changes succinctly using Python, and to apply them -to the database. A migrations table automatically keeps track of which migrations were already applied. - -For details please refer to the MIGRATIONS.rst document. - -Field Types ------------ - -Currently the following field types are supported: - -=================== ======== ================= =================================================== -Class DB Type Pythonic Type Comments -=================== ======== ================= =================================================== -StringField String unicode Encoded as UTF-8 when written to ClickHouse -DateField Date datetime.date Range 1970-01-01 to 2038-01-19 -DateTimeField DateTime datetime.datetime Minimal value is 1970-01-01 00:00:00; Always in UTC -Int8Field Int8 int Range -128 to 127 -Int16Field Int16 int Range -32768 to 32767 -Int32Field Int32 int Range -2147483648 to 2147483647 -Int64Field Int64 int/long Range -9223372036854775808 to 9223372036854775807 -UInt8Field UInt8 int Range 0 to 255 -UInt16Field UInt16 int Range 0 to 65535 -UInt32Field UInt32 int Range 0 to 4294967295 -UInt64Field UInt64 int/long Range 0 to 18446744073709551615 -Float32Field Float32 float -Float64Field Float64 float -Enum8Field Enum8 Enum See below -Enum16Field Enum16 Enum See below -ArrayField Array list See below -=================== ======== ================= =================================================== - -DateTimeField and Time Zones -**************************** - -A ``DateTimeField`` can be assigned values from one of the following types: - -- datetime -- date -- integer - number of seconds since the Unix epoch -- string in ``YYYY-MM-DD HH:MM:SS`` format - -The assigned value always gets converted to a timezone-aware ``datetime`` in UTC. If the assigned -value is a timezone-aware ``datetime`` in another timezone, it will be converted to UTC. Otherwise, the assigned value is assumed to already be in UTC. - -DateTime values that are read from the database are also converted to UTC. ClickHouse formats them according to the -timezone of the server, and the ORM makes the necessary conversions. This requires a ClickHouse version which is new -enough to support the ``timezone()`` function, otherwise it is assumed to be using UTC. In any case, we recommend -settings the server timezone to UTC in order to prevent confusion. - -Working with enum fields -************************ - -``Enum8Field`` and ``Enum16Field`` provide support for working with ClickHouse enum columns. They accept -strings or integers as values, and convert them to the matching Pythonic Enum member. - -Python 3.4 and higher supports Enums natively. When using previous Python versions you -need to install the `enum34` library. - -Example of a model with an enum field:: - - Gender = Enum('Gender', 'male female unspecified') - - class Person(models.Model): - - first_name = fields.StringField() - last_name = fields.StringField() - birthday = fields.DateField() - gender = fields.Enum32Field(Gender) - - engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday')) - - suzy = Person(first_name='Suzy', last_name='Jones', gender=Gender.female) - -Working with array fields -************************* - -You can create array fields containing any data type, for example:: - - class SensorData(models.Model): - - date = fields.DateField() - temperatures = fields.ArrayField(fields.Float32Field()) - humidity_levels = fields.ArrayField(fields.UInt8Field()) - - engine = engines.MergeTree('date', ('date',)) - - data = SensorData(date=date.today(), temperatures=[25.5, 31.2, 28.7], humidity_levels=[41, 39, 66]) - - -Working with materialized and alias fields -****************************************** - -ClickHouse provides an opportunity to create MATERIALIZED and ALIAS Fields. - -See documentation `here `_. - -Both field types can't be inserted into the database directly, so they are ignored when using the ``Database.insert()`` method. -ClickHouse does not return the field values if you use ``"SELECT * FROM ..."`` - you have to list these field -names explicitly in the query. - -Usage:: - - class Event(models.Model): - - created = fields.DateTimeField() - created_date = fields.DateTimeField(materialized='toDate(created)') - name = fields.StringField() - username = fields.StringField(alias='name') - - engine = engines.MergeTree('created_date', ('created_date', 'created')) - - obj = Event(created=datetime.now(), name='MyEvent') - db = Database('my_test_db') - db.insert([obj]) - # All values will be retrieved from database - db.select('SELECT created, created_date, username, name FROM $db.event', model_class=Event) - # created_date and username will contain a default value - db.select('SELECT * FROM $db.event', model_class=Event) - - -Table Engines -------------- - -Each model must have an engine instance, used when creating the table in ClickHouse. - -To define a ``MergeTree`` engine, supply the date column name and the names (or expressions) for the key columns:: - - engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate')) - -You may also provide a sampling expression:: - - engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'), sampling_expr='intHash32(UserID)') - -A ``CollapsingMergeTree`` engine is defined in a similar manner, but requires also a sign column:: - - engine = engines.CollapsingMergeTree('EventDate', ('CounterID', 'EventDate'), 'Sign') - -For a ``SummingMergeTree`` you can optionally specify the summing columns:: - - engine = engines.SummingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), - summing_cols=('Shows', 'Clicks', 'Cost')) - -For a ``ReplacingMergeTree`` you can optionally specify the version column:: - - engine = engines.ReplacingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), ver_col='Version') - -A ``Buffer`` engine is available for BufferModels. (See below how to use BufferModel). You can specify following parameters:: - - engine = engines.Buffer(Person) # you need to initialize engine with main Model. Other default parameters will be used - # or: - engine = engines.Buffer(Person, num_layers=16, min_time=10, - max_time=100, min_rows=10000, max_rows=1000000, - min_bytes=10000000, max_bytes=100000000) - -Buffer Models -------------- -Here's how you can define Model for Buffer Engine. The Buffer Model should be inherited from models.BufferModel and main Model:: - - class PersonBuffer(models.BufferModel, Person): - - engine = engines.Buffer(Person) - -Then you can insert objects into Buffer model and they will be handled by Clickhouse properly:: - - db.create_table(PersonBuffer) - suzy = PersonBuffer(first_name='Suzy', last_name='Jones') - dan = PersonBuffer(first_name='Dan', last_name='Schwartz') - db.insert([dan, suzy]) - - -Data Replication -**************** - -Any of the above engines can be converted to a replicated engine (e.g. ``ReplicatedMergeTree``) by adding two parameters, ``replica_table_path`` and ``replica_name``:: - - engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'), - replica_table_path='/clickhouse/tables/{layer}-{shard}/hits', - replica_name='{replica}') - -Development -=========== - -After cloning the project, run the following commands:: - - easy_install -U infi.projector - cd infi.clickhouse_orm - projector devenv build - -To run the tests, ensure that the ClickHouse server is running on http://localhost:8123/ (this is the default), and run:: - - bin/nosetests - -To see test coverage information run:: - - bin/nosetests --with-coverage --cover-package=infi.clickhouse_orm diff --git a/buildout.cfg b/buildout.cfg index e3de92a..0078704 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -28,7 +28,7 @@ homepage = https://github.com/Infinidat/infi.clickhouse_orm [isolated-python] recipe = infi.recipe.python -version = v2.7.8.4 +version = v2.7.9.4 [setup.py] recipe = infi.recipe.template.version diff --git a/docs/class_reference.md b/docs/class_reference.md new file mode 100644 index 0000000..0f4f57f --- /dev/null +++ b/docs/class_reference.md @@ -0,0 +1,594 @@ +Class Reference +=============== + +infi.clickhouse_orm.database +---------------------------- + +### Database + + +Database instances connect to a specific ClickHouse database for running queries, +inserting data and other operations. + +#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False) + + +Initializes a database instance. Unless it's readonly, the database will be +created on the ClickHouse server if it does not already exist. + +- `db_name`: name of the database to connect to. +- `db_url`: URL of the ClickHouse server. +- `username`: optional connection credentials. +- `password`: optional connection credentials. +- `readonly`: use a read-only connection. + + +#### count(model_class, conditions=None) + + +Counts the number of records in the model's table. + +- `model_class`: the model to count. +- `conditions`: optional SQL conditions (contents of the WHERE clause). + + +#### create_database() + + +Creates the database on the ClickHouse server if it does not already exist. + + +#### create_table(model_class) + + +Creates a table for the given model class, if it does not exist already. + + +#### drop_database() + + +Deletes the database on the ClickHouse server. + + +#### drop_table(model_class) + + +Drops the database table of the given model class, if it exists. + + +#### insert(model_instances, batch_size=1000) + + +Insert records into the database. + +- `model_instances`: any iterable containing instances of a single model class. +- `batch_size`: number of records to send per chunk (use a lower number if your records are very large). + + +#### migrate(migrations_package_name, up_to=9999) + + +Executes schema migrations. + +- `migrations_package_name` - fully qualified name of the Python package + containing the migrations. +- `up_to` - number of the last migration to apply. + + +#### paginate(model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None) + + +Selects records and returns a single page of model instances. + +- `model_class`: the model class matching the query's table, + or `None` for getting back instances of an ad-hoc model. +- `order_by`: columns to use for sorting the query (contents of the ORDER BY clause). +- `page_num`: the page number (1-based), or -1 to get the last page. +- `page_size`: number of records to return per page. +- `conditions`: optional SQL conditions (contents of the WHERE clause). +- `settings`: query settings to send as HTTP GET parameters + +The result is a namedtuple containing `objects` (list), `number_of_objects`, +`pages_total`, `number` (of the current page), and `page_size`. + + +#### raw(query, settings=None, stream=False) + + +Performs a query and returns its output as text. + +- `query`: the SQL query to execute. +- `settings`: query settings to send as HTTP GET parameters +- `stream`: if true, the HTTP response from ClickHouse will be streamed. + + +#### select(query, model_class=None, settings=None) + + +Performs a query and returns a generator of model instances. + +- `query`: the SQL query to execute. +- `model_class`: the model class matching the query's table, + or `None` for getting back instances of an ad-hoc model. +- `settings`: query settings to send as HTTP GET parameters + + +### DatabaseException + +Extends Exception + + +Raised when a database operation fails. + +infi.clickhouse_orm.models +-------------------------- + +### Model + + +A base class for ORM models. Each model class represent a ClickHouse table. For example: + + class CPUStats(Model): + timestamp = DateTimeField() + cpu_id = UInt16Field() + cpu_percent = Float32Field() + engine = Memory() + +#### Model(**kwargs) + + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### Model.create_table_sql(db_name) + + +Returns the SQL command for creating a table for this model. + + +#### Model.drop_table_sql(db_name) + + +Returns the SQL command for deleting this model's table. + + +#### Model.from_tsv(line, field_names=None, timezone_in_use=UTC, database=None) + + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. +If omitted, it is assumed to be the names of all fields in the model, in order of definition. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. +- `database`: if given, sets the database that this instance belongs to. + + +#### get_database() + + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + + +Gets a `Field` instance given its name, or `None` if not found. + + +#### Model.objects_in(database) + + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### Model.table_name() + + +Returns the model's database table name. By default this is the +class name converted to lowercase. Override this if you want to use +a different table name. + + +#### to_dict(include_readonly=True, field_names=None) + + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tsv(include_readonly=True) + + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +### BufferModel + +Extends Model + +#### BufferModel(**kwargs) + + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### BufferModel.create_table_sql(db_name) + + +Returns the SQL command for creating a table for this model. + + +#### BufferModel.drop_table_sql(db_name) + + +Returns the SQL command for deleting this model's table. + + +#### BufferModel.from_tsv(line, field_names=None, timezone_in_use=UTC, database=None) + + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. +If omitted, it is assumed to be the names of all fields in the model, in order of definition. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. +- `database`: if given, sets the database that this instance belongs to. + + +#### get_database() + + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + + +Gets a `Field` instance given its name, or `None` if not found. + + +#### BufferModel.objects_in(database) + + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### BufferModel.table_name() + + +Returns the model's database table name. By default this is the +class name converted to lowercase. Override this if you want to use +a different table name. + + +#### to_dict(include_readonly=True, field_names=None) + + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tsv(include_readonly=True) + + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +infi.clickhouse_orm.fields +-------------------------- + +### Field + + +Abstract base class for all field types. + +#### Field(default=None, alias=None, materialized=None) + + +### StringField + +Extends Field + +#### StringField(default=None, alias=None, materialized=None) + + +### DateField + +Extends Field + +#### DateField(default=None, alias=None, materialized=None) + + +### DateTimeField + +Extends Field + +#### DateTimeField(default=None, alias=None, materialized=None) + + +### BaseIntField + +Extends Field + + +Abstract base class for all integer-type fields. + +#### BaseIntField(default=None, alias=None, materialized=None) + + +### BaseFloatField + +Extends Field + + +Abstract base class for all float-type fields. + +#### BaseFloatField(default=None, alias=None, materialized=None) + + +### BaseEnumField + +Extends Field + + +Abstract base class for all enum-type fields. + +#### BaseEnumField(enum_cls, default=None, alias=None, materialized=None) + + +### ArrayField + +Extends Field + +#### ArrayField(inner_field, default=None, alias=None, materialized=None) + + +### FixedStringField + +Extends StringField + +#### FixedStringField(length, default=None, alias=None, materialized=None) + + +### UInt8Field + +Extends BaseIntField + +#### UInt8Field(default=None, alias=None, materialized=None) + + +### UInt16Field + +Extends BaseIntField + +#### UInt16Field(default=None, alias=None, materialized=None) + + +### UInt32Field + +Extends BaseIntField + +#### UInt32Field(default=None, alias=None, materialized=None) + + +### UInt64Field + +Extends BaseIntField + +#### UInt64Field(default=None, alias=None, materialized=None) + + +### Int8Field + +Extends BaseIntField + +#### Int8Field(default=None, alias=None, materialized=None) + + +### Int16Field + +Extends BaseIntField + +#### Int16Field(default=None, alias=None, materialized=None) + + +### Int32Field + +Extends BaseIntField + +#### Int32Field(default=None, alias=None, materialized=None) + + +### Int64Field + +Extends BaseIntField + +#### Int64Field(default=None, alias=None, materialized=None) + + +### Float32Field + +Extends BaseFloatField + +#### Float32Field(default=None, alias=None, materialized=None) + + +### Float64Field + +Extends BaseFloatField + +#### Float64Field(default=None, alias=None, materialized=None) + + +### Enum8Field + +Extends BaseEnumField + +#### Enum8Field(enum_cls, default=None, alias=None, materialized=None) + + +### Enum16Field + +Extends BaseEnumField + +#### Enum16Field(enum_cls, default=None, alias=None, materialized=None) + + +infi.clickhouse_orm.engines +--------------------------- + +### Engine + +### TinyLog + +Extends Engine + +### Log + +Extends Engine + +### Memory + +Extends Engine + +### MergeTree + +Extends Engine + +#### MergeTree(date_col, key_cols, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +### Buffer + +Extends Engine + + +Buffers the data to write in RAM, periodically flushing it to another table. +Must be used in conjuction with a `BufferModel`. +Read more [here](https://clickhouse.yandex/reference_en.html#Buffer). + +#### Buffer(main_model, num_layers=16, min_time=10, max_time=100, min_rows=10000, max_rows=1000000, min_bytes=10000000, max_bytes=100000000) + + +### CollapsingMergeTree + +Extends MergeTree + +#### CollapsingMergeTree(date_col, key_cols, sign_col, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +### SummingMergeTree + +Extends MergeTree + +#### SummingMergeTree(date_col, key_cols, summing_cols=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +### ReplacingMergeTree + +Extends MergeTree + +#### ReplacingMergeTree(date_col, key_cols, ver_col=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +infi.clickhouse_orm.query +------------------------- + +### QuerySet + + +A queryset is an object that represents a database query using a specific `Model`. +It is lazy, meaning that it does not hit the database until you iterate over its +matching rows (model instances). + +#### QuerySet(model_cls, database) + + +Initializer. It is possible to create a queryset like this, but the standard +way is to use `MyModel.objects_in(database)`. + + +#### as_sql() + + +Returns the whole query as a SQL string. + + +#### conditions_as_sql() + + +Returns the contents of the query's `WHERE` clause as a string. + + +#### count() + + +Returns the number of matching model instances. + + +#### exclude(**kwargs) + + +Returns a new `QuerySet` instance that excludes all rows matching the conditions. + + +#### filter(**kwargs) + + +Returns a new `QuerySet` instance that includes only rows matching the conditions. + + +#### only(*field_names) + + +Returns a new `QuerySet` instance limited to the specified field names. +Useful when there are large fields that are not needed, +or for creating a subquery to use with an IN operator. + + +#### order_by(*field_names) + + +Returns a new `QuerySet` instance with the ordering changed. + + +#### order_by_as_sql() + + +Returns the contents of the query's `ORDER BY` clause as a string. + + diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..cb64e57 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,36 @@ +Contributing +============ + +This project is hosted on GitHub - [https://github.com/Infinidat/infi.clickhouse_orm/](https://github.com/Infinidat/infi.clickhouse_orm/). + +Please open an issue there if you encounter a bug or want to request a feature. +Pull requests are also welcome. + +Building +-------- + +After cloning the project, run the following commands: + + easy_install -U infi.projector + cd infi.clickhouse_orm + projector devenv build + +A `setup.py` file will be generated, which you can use to install the development version of the package: + + python setup.py install + +Tests +----- + +To run the tests, ensure that the ClickHouse server is running on (this is the default), and run: + + bin/nosetests + +To see test coverage information run: + + bin/nosetests --with-coverage --cover-package=infi.clickhouse_orm + + +--- + +[<< System Models](system_models.md) | [Table of Contents](toc.md) | [Class Reference >>](class_reference.md) \ No newline at end of file diff --git a/docs/field_types.md b/docs/field_types.md new file mode 100644 index 0000000..8936ef1 --- /dev/null +++ b/docs/field_types.md @@ -0,0 +1,109 @@ +Field Types +=========== + +Currently the following field types are supported: + +| Class | DB Type | Pythonic Type | Comments +| ------------------ | ---------- | ------------------- | ----------------------------------------------------- +| StringField | String | unicode | Encoded as UTF-8 when written to ClickHouse +| FixedStringField | String | unicode | Encoded as UTF-8 when written to ClickHouse +| DateField | Date | datetime.date | Range 1970-01-01 to 2038-01-19 +| DateTimeField | DateTime | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Always in UTC +| Int8Field | Int8 | int | Range -128 to 127 +| Int16Field | Int16 | int | Range -32768 to 32767 +| Int32Field | Int32 | int | Range -2147483648 to 2147483647 +| Int64Field | Int64 | int/long | Range -9223372036854775808 to 9223372036854775807 +| UInt8Field | UInt8 | int | Range 0 to 255 +| UInt16Field | UInt16 | int | Range 0 to 65535 +| UInt32Field | UInt32 | int | Range 0 to 4294967295 +| UInt64Field | UInt64 | int/long | Range 0 to 18446744073709551615 +| Float32Field | Float32 | float | +| Float64Field | Float64 | float | +| Enum8Field | Enum8 | Enum | See below +| Enum16Field | Enum16 | Enum | See below +| ArrayField | Array | list | See below + +DateTimeField and Time Zones +---------------------------- + +A `DateTimeField` can be assigned values from one of the following types: + +- datetime +- date +- integer - number of seconds since the Unix epoch +- string in `YYYY-MM-DD HH:MM:SS` format + +The assigned value always gets converted to a timezone-aware `datetime` in UTC. If the assigned value is a timezone-aware `datetime` in another timezone, it will be converted to UTC. Otherwise, the assigned value is assumed to already be in UTC. + +DateTime values that are read from the database are also converted to UTC. ClickHouse formats them according to the timezone of the server, and the ORM makes the necessary conversions. This requires a ClickHouse +version which is new enough to support the `timezone()` function, otherwise it is assumed to be using UTC. In any case, we recommend settings the server timezone to UTC in order to prevent confusion. + +Working with enum fields +------------------------ + +`Enum8Field` and `Enum16Field` provide support for working with ClickHouse enum columns. They accept strings or integers as values, and convert them to the matching Pythonic Enum member. + +Python 3.4 and higher supports Enums natively. When using previous Python versions you need to install the enum34 library. + +Example of a model with an enum field: + + Gender = Enum('Gender', 'male female unspecified') + + class Person(models.Model): + + first_name = fields.StringField() + last_name = fields.StringField() + birthday = fields.DateField() + gender = fields.Enum32Field(Gender) + + engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday')) + + suzy = Person(first_name='Suzy', last_name='Jones', gender=Gender.female) + +Working with array fields +------------------------- + +You can create array fields containing any data type, for example: + + class SensorData(models.Model): + + date = fields.DateField() + temperatures = fields.ArrayField(fields.Float32Field()) + humidity_levels = fields.ArrayField(fields.UInt8Field()) + + engine = engines.MergeTree('date', ('date',)) + + data = SensorData(date=date.today(), temperatures=[25.5, 31.2, 28.7], humidity_levels=[41, 39, 66]) + +Working with materialized and alias fields +------------------------------------------ + +ClickHouse provides an opportunity to create MATERIALIZED and ALIAS Fields. + +See documentation [here](https://clickhouse.yandex/reference_en.html#Default%20values). + +Both field types can't be inserted into the database directly, so they are ignored when using the `Database.insert()` method. ClickHouse does not return the field values if you use `"SELECT * FROM ..."` - you have to list these field names explicitly in the query. + +Usage: + + class Event(models.Model): + + created = fields.DateTimeField() + created_date = fields.DateTimeField(materialized='toDate(created)') + name = fields.StringField() + username = fields.StringField(alias='name') + + engine = engines.MergeTree('created_date', ('created_date', 'created')) + + obj = Event(created=datetime.now(), name='MyEvent') + db = Database('my_test_db') + db.insert([obj]) + # All values will be retrieved from database + db.select('SELECT created, created_date, username, name FROM $db.event', model_class=Event) + # created_date and username will contain a default value + db.select('SELECT * FROM $db.event', model_class=Event) + + +--- + +[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md) \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..d6df2c1 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,17 @@ +Overview +======== + +This project is simple ORM for working with the [ClickHouse database](https://clickhouse.yandex/). It allows you to define model classes whose instances can be written to the database and read from it. + +It was tested on Python 2.7 and 3.5. + +Installation +------------ + +To install infi.clickhouse_orm: + + pip install infi.clickhouse_orm + +--- + +[Table of Contents](toc.md) | [Models and Databases >>](models_and_databases.md) \ No newline at end of file diff --git a/docs/models_and_databases.md b/docs/models_and_databases.md new file mode 100644 index 0000000..2879c32 --- /dev/null +++ b/docs/models_and_databases.md @@ -0,0 +1,176 @@ +Models and Databases +==================== + +Models represent ClickHouse tables, allowing you to work with them using familiar pythonic syntax. + +Database instances connect to a specific ClickHouse database for running queries, inserting data and other operations. + +Defining Models +--------------- + +Models are defined in a way reminiscent of Django's ORM: + + from infi.clickhouse_orm import models, fields, engines + + class Person(models.Model): + + first_name = fields.StringField() + last_name = fields.StringField() + birthday = fields.DateField() + height = fields.Float32Field() + + engine = engines.MergeTree('birthday', ('first_name', 'last_name', 'birthday')) + +It is possible to provide a default value for a field, instead of its "natural" default (empty string for string fields, zero for numeric fields etc.). Alternatively it is possible to pass alias or materialized parameters (see below for usage examples). Only one of `default`, `alias` and `materialized` parameters can be provided. + +For more details see [Field Types](field_types.md) and [Table Engines](table_engines.md). + +### Table Names + +The table name used for the model is its class name, converted to lowercase. To override the default name, implement the `table_name` method: + + class Person(models.Model): + + ... + + @classmethod + def table_name(cls): + return 'people' + +Using Models +------------ + +Once you have a model, you can create model instances: + + >>> dan = Person(first_name='Dan', last_name='Schwartz') + >>> suzy = Person(first_name='Suzy', last_name='Jones') + >>> dan.first_name + u'Dan' + +When values are assigned to model fields, they are immediately converted to their Pythonic data type. In case the value is invalid, a `ValueError` is raised: + + >>> suzy.birthday = '1980-01-17' + >>> suzy.birthday + datetime.date(1980, 1, 17) + >>> suzy.birthday = 0.5 + ValueError: Invalid value for DateField - 0.5 + >>> suzy.birthday = '1922-05-31' + ValueError: DateField out of range - 1922-05-31 is not between 1970-01-01 and 2038-01-19 + +Inserting to the Database +------------------------- + +To write your instances to ClickHouse, you need a `Database` instance: + + from infi.clickhouse_orm.database import Database + + db = Database('my_test_db') + +This automatically connects to and creates a database called my_test_db, unless it already exists. If necessary, you can specify a different database URL and optional credentials: + + db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') + +Using the `Database` instance you can create a table for your model, and insert instances to it: + + db.create_table(Person) + db.insert([dan, suzy]) + +The `insert` method can take any iterable of model instances, but they all must belong to the same model class. + +Creating a read-only database is also supported. Such a `Database` instance can only read data, and cannot modify data or schemas: + + db = Database('my_test_db', readonly=True) + +Reading from the Database +------------------------- + +Loading model instances from the database is simple: + + for person in db.select("SELECT * FROM my_test_db.person", model_class=Person): + print person.first_name, person.last_name + +Do not include a `FORMAT` clause in the query, since the ORM automatically sets the format to `TabSeparatedWithNamesAndTypes`. + +It is possible to select only a subset of the columns, and the rest will receive their default values: + + for person in db.select("SELECT first_name FROM my_test_db.person WHERE last_name='Smith'", model_class=Person): + print person.first_name + +The ORM provides a way to build simple queries without writing SQL by hand. The previous snippet can be written like this: + + for person in Person.objects_in(db).filter(last_name='Smith').only('first_name'): + print person.first_name + +See [Querysets](querysets.md) for more information. + + +Reading without a Model +----------------------- + +When running a query, specifying a model class is not required. In case you do not provide a model class, an ad-hoc class will be defined based on the column names and types returned by the query: + + for row in db.select("SELECT max(height) as max_height FROM my_test_db.person"): + print row.max_height + +This is a very convenient feature that saves you the need to define a model for each query, while still letting you work with Pythonic column values and an elegant syntax. + +SQL Placeholders +---------------- + +There are a couple of special placeholders that you can use inside the SQL to make it easier to write: `$db` and `$table`. The first one is replaced by the database name, and the second is replaced by the database name plus table name (but is available only when the model is specified). + +So instead of this: + + db.select("SELECT * FROM my_test_db.person", model_class=Person) + +you can use: + + db.select("SELECT * FROM $db.person", model_class=Person) + +or even: + + db.select("SELECT * FROM $table", model_class=Person) + +Counting +-------- + +The `Database` class also supports counting records easily: + + >>> db.count(Person) + 117 + >>> db.count(Person, conditions="height > 1.90") + 6 + +Pagination +---------- + +It is possible to paginate through model instances: + + >>> order_by = 'first_name, last_name' + >>> page = db.paginate(Person, order_by, page_num=1, page_size=10) + >>> print page.number_of_objects + 2507 + >>> print page.pages_total + 251 + >>> for person in page.objects: + >>> # do something + +The `paginate` method returns a `namedtuple` containing the following fields: + +- `objects` - the list of objects in this page +- `number_of_objects` - total number of objects in all pages +- `pages_total` - total number of pages +- `number` - the page number, starting from 1; the special value -1 + may be used to retrieve the last page +- `page_size` - the number of objects per page + +You can optionally pass conditions to the query: + + >>> page = db.paginate(Person, order_by, page_num=1, page_size=100, conditions='height > 1.90') + +Note that `order_by` must be chosen so that the ordering is unique, otherwise there might be inconsistencies in the pagination (such as an instance that appears on two different pages). + + +--- + +[<< Overview](index.md) | [Table of Contents](toc.md) | [Querysets >>](querysets.md) \ No newline at end of file diff --git a/docs/querysets.md b/docs/querysets.md new file mode 100644 index 0000000..fd7e253 --- /dev/null +++ b/docs/querysets.md @@ -0,0 +1,101 @@ +Querysets +========= + +A queryset is an object that represents a database query using a specific Model. It is lazy, meaning that it does not hit the database until you iterate over its matching rows (model instances). To create a base queryset for a model class, use: + + qs = Person.objects_in(database) + +This queryset matches all Person instances in the database. You can get these instances using iteration: + + for person in qs: + print person.first_name, person.last_name + +Filtering +--------- + +The `filter` and `exclude` methods are used for filtering the matching instances. Calling these methods returns a new queryset instance, with the added conditions. For example: + + >>> qs = Person.objects_in(database) + >>> qs = qs.filter(first_name__startswith='V').exclude(birthday__lt='2000-01-01') + >>> qs.conditions_as_sql() + u"first_name LIKE 'V%' AND NOT (birthday < '2000-01-01')" + +It is possible to specify several fields to filter or exclude by: + + >>> qs = Person.objects_in(database).filter(last_name='Smith', height__gt=1.75) + >>> qs.conditions_as_sql() + u"last_name = 'Smith' AND height > 1.75" + +There are different operators that can be used, by passing `__=` (two underscores separate the field name from the operator). In case no operator is given, `eq` is used by default. Below are all the supported operators. + +| Operator | Equivalent SQL | Comments | +| -------- | -------------------------------------------- | ---------------------------------- | +| `eq` | `field = value` | | +| `gt` | `field > value` | | +| `gte` | `field >= value` | | +| `lt` | `field < value` | | +| `lte` | `field <= value` | | +| `in` | `field IN (values)` | See below | +| `contains` | `field LIKE '%value%'` | For string fields only | +| `startswith` | `field LIKE 'value%'` | For string fields only | +| `endswith` | `field LIKE '%value'` | For string fields only | +| `icontains` | `lowerUTF8(field) LIKE lowerUTF8('%value%')` | For string fields only | +| `istartswith` | `lowerUTF8(field) LIKE lowerUTF8('value%')` | For string fields only | +| `iendswith` | `lowerUTF8(field) LIKE lowerUTF8('%value')` | For string fields only | +| `iexact` | `lowerUTF8(field) = lowerUTF8(value)` | For string fields only | + +### Using the `in` Operator + +The `in` operator expects one of three types of values: +* A list or tuple of simple values +* A string, which is used verbatim as the contents of the parentheses +* Another queryset (subquery) + +For example if we want to select only people with Irish last names: + + # A list of simple values + qs = Person.objects_in(database).filter(last_name__in=["Murphy", "O'Sullivan"]) + + # A string + subquery = "SELECT name from $db.irishlastname" + qs = Person.objects_in(database).filter(last_name__in=subquery) + + # A queryset + subquery = IrishLastName.objects_in(database).only("name") + qs = Person.objects_in(database).filter(last_name__in=subquery) + +Counting and Checking Existence +------------------------------- + +Use the `count` method to get the number of matches: + + Person.objects_in(database).count() + +To check if there are any matches at all, you can use any of the following equivalent options: + + if qs.count(): ... + if bool(qs): ... + if qs: ... + +Ordering +-------- + +The sorting order of the results can be controlled using the `order_by` method: + + qs = Person.objects_in(database).order_by('last_name', 'first_name') + +The default order is ascending. To use descending order, add a minus sign before the field name: + + qs = Person.objects_in(database).order_by('-height') + +Omitting Fields +--------------- + +When some of the model fields aren't needed, it is more efficient to omit them from the query. This is especially true when there are large fields that may slow the query down. Use the `only` method to specify which fields to retrieve: + + qs = Person.objects_in(database).only('first_name', 'birthday') + + +--- + +[<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md) \ No newline at end of file diff --git a/docs/ref.md b/docs/ref.md new file mode 100644 index 0000000..f789578 --- /dev/null +++ b/docs/ref.md @@ -0,0 +1,526 @@ +Class Reference +=============== + +infi.clickhouse_orm.database +---------------------------- + +### Database + +#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False) + +Initializes a database instance. Unless it's readonly, the database will be +created on the ClickHouse server if it does not already exist. + +- `db_name`: name of the database to connect to. +- `db_url`: URL of the ClickHouse server. +- `username`: optional connection credentials. +- `password`: optional connection credentials. +- `readonly`: use a read-only connection. + + +#### count(model_class, conditions=None) + +Counts the number of records in the model's table. + +- `model_class`: the model to count. +- `conditions`: optional SQL conditions (contents of the WHERE clause). + + +#### create_database() + +Creates the database on the ClickHouse server if it does not already exist. + + +#### create_table(model_class) + +Creates a table for the given model class, if it does not exist already. + + +#### drop_database() + +Deletes the database on the ClickHouse server. + + +#### drop_table(model_class) + +Drops the database table of the given model class, if it exists. + + +#### insert(model_instances, batch_size=1000) + +Insert records into the database. + +- `model_instances`: any iterable containing instances of a single model class. +- `batch_size`: number of records to send per chunk (use a lower number if your records are very large). + + +#### migrate(migrations_package_name, up_to=9999) + +Executes schema migrations. + +- `migrations_package_name` - fully qualified name of the Python package +containing the migrations. +- `up_to` - number of the last migration to apply. + + +#### paginate(model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None) + +Selects records and returns a single page of model instances. + +- `model_class`: the model class matching the query's table, +or `None` for getting back instances of an ad-hoc model. +- `order_by`: columns to use for sorting the query (contents of the ORDER BY clause). +- `page_num`: the page number (1-based), or -1 to get the last page. +- `page_size`: number of records to return per page. +- `conditions`: optional SQL conditions (contents of the WHERE clause). +- `settings`: query settings to send as HTTP GET parameters + +The result is a namedtuple containing `objects` (list), `number_of_objects`, +`pages_total`, `number` (of the current page), and `page_size`. + + +#### raw(query, settings=None, stream=False) + +Performs a query and returns its output as text. + +- `query`: the SQL query to execute. +- `settings`: query settings to send as HTTP GET parameters +- `stream`: if true, the HTTP response from ClickHouse will be streamed. + + +#### select(query, model_class=None, settings=None) + +Performs a query and returns a generator of model instances. + +- `query`: the SQL query to execute. +- `model_class`: the model class matching the query's table, +or `None` for getting back instances of an ad-hoc model. +- `settings`: query settings to send as HTTP GET parameters + + +### DatabaseException + +Extends Exception + +Raised when a database operation fails. + +infi.clickhouse_orm.models +-------------------------- + +### Model + +A base class for ORM models. + +#### Model(**kwargs) + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### Model.create_table_sql(db_name) + +Returns the SQL command for creating a table for this model. + + +#### Model.drop_table_sql(db_name) + +Returns the SQL command for deleting this model's table. + + +#### Model.from_tsv(line, field_names=None, timezone_in_use=UTC, database=None) + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. +If omitted, it is assumed to be the names of all fields in the model, in order of definition. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. +- `database`: if given, sets the database that this instance belongs to. + + +#### get_database() + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + +Gets a `Field` instance given its name, or `None` if not found. + + +#### Model.objects_in(database) + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### Model.table_name() + +Returns the model's database table name. By default this is the +class name converted to lowercase. Override this if you want to use +a different table name. + + +#### to_dict(include_readonly=True, field_names=None) + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tsv(include_readonly=True) + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +### BufferModel + +Extends Model + +#### BufferModel(**kwargs) + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### BufferModel.create_table_sql(db_name) + +Returns the SQL command for creating a table for this model. + + +#### BufferModel.drop_table_sql(db_name) + +Returns the SQL command for deleting this model's table. + + +#### BufferModel.from_tsv(line, field_names=None, timezone_in_use=UTC, database=None) + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. +If omitted, it is assumed to be the names of all fields in the model, in order of definition. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. +- `database`: if given, sets the database that this instance belongs to. + + +#### get_database() + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + +Gets a `Field` instance given its name, or `None` if not found. + + +#### BufferModel.objects_in(database) + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### BufferModel.table_name() + +Returns the model's database table name. By default this is the +class name converted to lowercase. Override this if you want to use +a different table name. + + +#### to_dict(include_readonly=True, field_names=None) + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tsv(include_readonly=True) + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +infi.clickhouse_orm.fields +-------------------------- + +### Field + +Abstract base class for all field types. + +#### Field(default=None, alias=None, materialized=None) + + +### StringField + +Extends Field + +#### StringField(default=None, alias=None, materialized=None) + + +### DateField + +Extends Field + +#### DateField(default=None, alias=None, materialized=None) + + +### DateTimeField + +Extends Field + +#### DateTimeField(default=None, alias=None, materialized=None) + + +### BaseIntField + +Extends Field + +Abstract base class for all integer-type fields. + +#### BaseIntField(default=None, alias=None, materialized=None) + + +### BaseFloatField + +Extends Field + +Abstract base class for all float-type fields. + +#### BaseFloatField(default=None, alias=None, materialized=None) + + +### BaseEnumField + +Extends Field + +Abstract base class for all enum-type fields. + +#### BaseEnumField(enum_cls, default=None, alias=None, materialized=None) + + +### ArrayField + +Extends Field + +#### ArrayField(inner_field, default=None, alias=None, materialized=None) + + +### FixedStringField + +Extends StringField + +#### FixedStringField(length, default=None, alias=None, materialized=None) + + +### UInt8Field + +Extends BaseIntField + +#### UInt8Field(default=None, alias=None, materialized=None) + + +### UInt16Field + +Extends BaseIntField + +#### UInt16Field(default=None, alias=None, materialized=None) + + +### UInt32Field + +Extends BaseIntField + +#### UInt32Field(default=None, alias=None, materialized=None) + + +### UInt64Field + +Extends BaseIntField + +#### UInt64Field(default=None, alias=None, materialized=None) + + +### Int8Field + +Extends BaseIntField + +#### Int8Field(default=None, alias=None, materialized=None) + + +### Int16Field + +Extends BaseIntField + +#### Int16Field(default=None, alias=None, materialized=None) + + +### Int32Field + +Extends BaseIntField + +#### Int32Field(default=None, alias=None, materialized=None) + + +### Int64Field + +Extends BaseIntField + +#### Int64Field(default=None, alias=None, materialized=None) + + +### Float32Field + +Extends BaseFloatField + +#### Float32Field(default=None, alias=None, materialized=None) + + +### Float64Field + +Extends BaseFloatField + +#### Float64Field(default=None, alias=None, materialized=None) + + +### Enum8Field + +Extends BaseEnumField + +#### Enum8Field(enum_cls, default=None, alias=None, materialized=None) + + +### Enum16Field + +Extends BaseEnumField + +#### Enum16Field(enum_cls, default=None, alias=None, materialized=None) + + +infi.clickhouse_orm.engines +--------------------------- + +### Engine + +### TinyLog + +Extends Engine + +### Log + +Extends Engine + +### Memory + +Extends Engine + +### MergeTree + +Extends Engine + +#### MergeTree(date_col, key_cols, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +### Buffer + +Extends Engine + +Here we define Buffer engine +Read more here https://clickhouse.yandex/reference_en.html#Buffer + +#### Buffer(main_model, num_layers=16, min_time=10, max_time=100, min_rows=10000, max_rows=1000000, min_bytes=10000000, max_bytes=100000000) + + +### CollapsingMergeTree + +Extends MergeTree + +#### CollapsingMergeTree(date_col, key_cols, sign_col, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +### SummingMergeTree + +Extends MergeTree + +#### SummingMergeTree(date_col, key_cols, summing_cols=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +### ReplacingMergeTree + +Extends MergeTree + +#### ReplacingMergeTree(date_col, key_cols, ver_col=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) + + +infi.clickhouse_orm.query +------------------------- + +### QuerySet + +#### QuerySet(model_cls, database) + + +#### conditions_as_sql() + +Return the contents of the queryset's WHERE clause. + + +#### count() + +Returns the number of matching model instances. + + +#### exclude(**kwargs) + +Returns a new QuerySet instance that excludes all rows matching the conditions. + + +#### filter(**kwargs) + +Returns a new QuerySet instance that includes only rows matching the conditions. + + +#### only(*field_names) + +Limit the query to return only the specified field names. +Useful when there are large fields that are not needed, +or for creating a subquery to use with an IN operator. + + +#### order_by(*field_names) + +Returns a new QuerySet instance with the ordering changed. + + +#### order_by_as_sql() + +Return the contents of the queryset's ORDER BY clause. + + +#### query() + +Return the the queryset as SQL. + + diff --git a/MIGRATIONS.rst b/docs/schema_migrations.md similarity index 54% rename from MIGRATIONS.rst rename to docs/schema_migrations.md index 0f0d5e2..e4647bf 100644 --- a/MIGRATIONS.rst +++ b/docs/schema_migrations.md @@ -1,17 +1,14 @@ -Migrations -========== +Schema Migrations +================= -Over time, the ORM models in your application may change. Migrations provide a way to modify the database -tables according to the changes in your models, without writing raw SQL. +Over time, the ORM models in your application may change. Migrations provide a way to modify the database tables according to the changes in your models, without writing raw SQL. -The migrations that were applied to the database are recorded in the ``infi_clickhouse_orm_migrations`` table, -so migrating the database will only apply any missing migrations. +The migrations that were applied to the database are recorded in the `infi_clickhouse_orm_migrations` table, so migrating the database will only apply any missing migrations. Writing Migrations ------------------ -To write migrations, create a Python package. Then create a python file for the initial migration. The migration -files must begin with a four-digit number, and will be applied in sequence. For example:: +To write migrations, create a Python package. Then create a python file for the initial migration. The migration files must begin with a four-digit number, and will be applied in sequence. For example: analytics | @@ -23,7 +20,7 @@ files must begin with a four-digit number, and will be applied in sequence. For | +-- 0002_add_user_agents_table.py -Each migration file is expected to contain a list of ``operations``, for example:: +Each migration file is expected to contain a list of `operations`, for example: from infi.clickhouse_orm import migrations from analytics import models @@ -45,21 +42,24 @@ A migration operation that drops the table of a given model class. **AlterTable** -A migration operation that compares the table of a given model class to -the model's fields, and alters the table to match the model. The operation can: +A migration operation that compares the table of a given model class to the model’s fields, and alters the table to match the model. The operation can: -- add new columns -- drop obsolete columns -- modify column types +- add new columns +- drop obsolete columns +- modify column types Default values are not altered by this operation. Running Migrations ------------------ -To migrate a database, create a ``Database`` instance and call its ``migrate`` method with the package -name containing your migrations:: +To migrate a database, create a `Database` instance and call its `migrate` method with the package name containing your migrations: Database('analytics_db').migrate('analytics.analytics_migrations') -Note that you may have more than one migrations package. \ No newline at end of file +Note that you may have more than one migrations package. + + +--- + +[<< Table Engines](table_engines.md) | [Table of Contents](toc.md) | [System Models >>](system_models.md) \ No newline at end of file diff --git a/docs/system_models.md b/docs/system_models.md new file mode 100644 index 0000000..beed825 --- /dev/null +++ b/docs/system_models.md @@ -0,0 +1,47 @@ +System Models +============= + +[Clickhouse docs](https://clickhouse.yandex/reference_en.html#System%20tables). + +System models are read only models for implementing part of the system's functionality, and for providing access to information about how the system is working. + +Currently the following system models are supported: + +| Class | DB Table | Comments +| ------------ | -------------- | --------------------------------------------------- +| SystemPart | system.parts | Gives methods to work with partitions. See below. + +Partitions and Parts +-------------------- + +[ClickHouse docs](https://clickhouse.yandex/reference_en.html#Manipulations%20with%20partitions%20and%20parts). + +A partition in a table is data for a single calendar month. Table "system.parts" contains information about each part. + +| Method | Parameters | Comments +| --------------------- | ------------------------- | ----------------------------------------------------------------------------------------------- +| get(static) | database, conditions="" | Gets database partitions, filtered by conditions +| get_active(static) | database, conditions="" | Gets only active (not detached or dropped) partitions, filtered by conditions +| detach | settings=None | Detaches the partition. Settings is a dict of params to pass to http request +| drop | settings=None | Drops the partition. Settings is a dict of params to pass to http request +| attach | settings=None | Attaches already detached partition. Settings is a dict of params to pass to http request +| freeze | settings=None | Freezes (makes backup) of the partition. Settings is a dict of params to pass to http request +| fetch | settings=None | Fetches partition. Settings is a dict of params to pass to http request + +Usage example: + + from infi.clickhouse_orm.database import Database + from infi.clickhouse_orm.system_models import SystemPart + db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') + partitions = SystemPart.get_active(db, conditions='') # Getting all active partitions of the database + if len(partitions) > 0: + partitions = sorted(partitions, key=lambda obj: obj.name) # Partition name is YYYYMM, so we can sort so + partitions[0].freeze() # Make a backup in /opt/clickhouse/shadow directory + partitions[0].drop() # Dropped partition + +`Note`: system.parts stores information for all databases. To be correct, SystemPart model was designed to receive only parts belonging to the given database instance. + + +--- + +[<< Schema Migrations](schema_migrations.md) | [Table of Contents](toc.md) | [Contributing >>](contributing.md) \ No newline at end of file diff --git a/docs/table_engines.md b/docs/table_engines.md new file mode 100644 index 0000000..2f92183 --- /dev/null +++ b/docs/table_engines.md @@ -0,0 +1,92 @@ +Table Engines +============= + +See: [ClickHouse Documentation](https://clickhouse.yandex/reference_en.html#Table+engines) + +Each model must have an engine instance, used when creating the table in ClickHouse. + +The following engines are supported by the ORM: + +- TinyLog +- Log +- Memory +- MergeTree / ReplicatedMergeTree +- CollapsingMergeTree / ReplicatedCollapsingMergeTree +- SummingMergeTree / ReplicatedSummingMergeTree +- ReplacingMergeTree / ReplicatedReplacingMergeTree +- Buffer + + +Simple Engines +-------------- + +`TinyLog`, `Log` and `Memory` engines do not require any parameters: + + engine = engines.TinyLog() + + engine = engines.Log() + + engine = engines.Memory() + + +Engines in the MergeTree Family +------------------------------- + +To define a `MergeTree` engine, supply the date column name and the names (or expressions) for the key columns: + + engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate')) + +You may also provide a sampling expression: + + engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'), sampling_expr='intHash32(UserID)') + +A `CollapsingMergeTree` engine is defined in a similar manner, but requires also a sign column: + + engine = engines.CollapsingMergeTree('EventDate', ('CounterID', 'EventDate'), 'Sign') + +For a `SummingMergeTree` you can optionally specify the summing columns: + + engine = engines.SummingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), + summing_cols=('Shows', 'Clicks', 'Cost')) + +For a `ReplacingMergeTree` you can optionally specify the version column: + + engine = engines.ReplacingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), ver_col='Version') + +### Data Replication + +Any of the above engines can be converted to a replicated engine (e.g. `ReplicatedMergeTree`) by adding two parameters, `replica_table_path` and `replica_name`: + + engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'), + replica_table_path='/clickhouse/tables/{layer}-{shard}/hits', + replica_name='{replica}') + + +Buffer Engine +------------- + +A `Buffer` engine is only used in conjunction with a `BufferModel`. +The model should be a subclass of both `models.BufferModel` and the main model. +The main model is also passed to the engine: + + class PersonBuffer(models.BufferModel, Person): + + engine = engines.Buffer(Person) + +Additional buffer parameters can optionally be specified: + + engine = engines.Buffer(Person, num_layers=16, min_time=10, + max_time=100, min_rows=10000, max_rows=1000000, + min_bytes=10000000, max_bytes=100000000) + +Then you can insert objects into Buffer model and they will be handled by ClickHouse properly: + + db.create_table(PersonBuffer) + suzy = PersonBuffer(first_name='Suzy', last_name='Jones') + dan = PersonBuffer(first_name='Dan', last_name='Schwartz') + db.insert([dan, suzy]) + + +--- + +[<< Field Types](field_types.md) | [Table of Contents](toc.md) | [Schema Migrations >>](schema_migrations.md) \ No newline at end of file diff --git a/docs/toc.md b/docs/toc.md new file mode 100644 index 0000000..4167e64 --- /dev/null +++ b/docs/toc.md @@ -0,0 +1,88 @@ +# Table of Contents + + * [Overview](index.md#overview) + * [Installation](index.md#installation) + + * [Models and Databases](models_and_databases.md#models-and-databases) + * [Defining Models](models_and_databases.md#defining-models) + * [Table Names](models_and_databases.md#table-names) + * [Using Models](models_and_databases.md#using-models) + * [Inserting to the Database](models_and_databases.md#inserting-to-the-database) + * [Reading from the Database](models_and_databases.md#reading-from-the-database) + * [Reading without a Model](models_and_databases.md#reading-without-a-model) + * [SQL Placeholders](models_and_databases.md#sql-placeholders) + * [Counting](models_and_databases.md#counting) + * [Pagination](models_and_databases.md#pagination) + + * [Querysets](querysets.md#querysets) + * [Filtering](querysets.md#filtering) + * [Using the in Operator](querysets.md#using-the-in-operator) + * [Counting and Checking Existence](querysets.md#counting-and-checking-existence) + * [Ordering](querysets.md#ordering) + * [Omitting Fields](querysets.md#omitting-fields) + + * [Field Types](field_types.md#field-types) + * [DateTimeField and Time Zones](field_types.md#datetimefield-and-time-zones) + * [Working with enum fields](field_types.md#working-with-enum-fields) + * [Working with array fields](field_types.md#working-with-array-fields) + * [Working with materialized and alias fields](field_types.md#working-with-materialized-and-alias-fields) + + * [Table Engines](table_engines.md#table-engines) + * [Simple Engines](table_engines.md#simple-engines) + * [Engines in the MergeTree Family](table_engines.md#engines-in-the-mergetree-family) + * [Data Replication](table_engines.md#data-replication) + * [Buffer Engine](table_engines.md#buffer-engine) + + * [Schema Migrations](schema_migrations.md#schema-migrations) + * [Writing Migrations](schema_migrations.md#writing-migrations) + * [Running Migrations](schema_migrations.md#running-migrations) + + * [System Models](system_models.md#system-models) + * [Partitions and Parts](system_models.md#partitions-and-parts) + + * [Contributing](contributing.md#contributing) + * [Building](contributing.md#building) + * [Tests](contributing.md#tests) + + * [Class Reference](class_reference.md#class-reference) + * [infi.clickhouse_orm.database](class_reference.md#infi.clickhouse_orm.database) + * [Database](class_reference.md#database) + * [DatabaseException](class_reference.md#databaseexception) + * [infi.clickhouse_orm.models](class_reference.md#infi.clickhouse_orm.models) + * [Model](class_reference.md#model) + * [BufferModel](class_reference.md#buffermodel) + * [infi.clickhouse_orm.fields](class_reference.md#infi.clickhouse_orm.fields) + * [Field](class_reference.md#field) + * [StringField](class_reference.md#stringfield) + * [DateField](class_reference.md#datefield) + * [DateTimeField](class_reference.md#datetimefield) + * [BaseIntField](class_reference.md#baseintfield) + * [BaseFloatField](class_reference.md#basefloatfield) + * [BaseEnumField](class_reference.md#baseenumfield) + * [ArrayField](class_reference.md#arrayfield) + * [FixedStringField](class_reference.md#fixedstringfield) + * [UInt8Field](class_reference.md#uint8field) + * [UInt16Field](class_reference.md#uint16field) + * [UInt32Field](class_reference.md#uint32field) + * [UInt64Field](class_reference.md#uint64field) + * [Int8Field](class_reference.md#int8field) + * [Int16Field](class_reference.md#int16field) + * [Int32Field](class_reference.md#int32field) + * [Int64Field](class_reference.md#int64field) + * [Float32Field](class_reference.md#float32field) + * [Float64Field](class_reference.md#float64field) + * [Enum8Field](class_reference.md#enum8field) + * [Enum16Field](class_reference.md#enum16field) + * [infi.clickhouse_orm.engines](class_reference.md#infi.clickhouse_orm.engines) + * [Engine](class_reference.md#engine) + * [TinyLog](class_reference.md#tinylog) + * [Log](class_reference.md#log) + * [Memory](class_reference.md#memory) + * [MergeTree](class_reference.md#mergetree) + * [Buffer](class_reference.md#buffer) + * [CollapsingMergeTree](class_reference.md#collapsingmergetree) + * [SummingMergeTree](class_reference.md#summingmergetree) + * [ReplacingMergeTree](class_reference.md#replacingmergetree) + * [infi.clickhouse_orm.query](class_reference.md#infi.clickhouse_orm.query) + * [QuerySet](class_reference.md#queryset) + diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..aaf2f27 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,54 @@ +This directory contains various scripts for use while developing. + +generate_toc +------------ +Generates the table of contents (toc.md). Requires Pandoc. +Usage: + + cd docs + ../scripts/generate_toc.sh + + +html_to_markdown_toc.py +----------------------- +Used by generate_toc. + + +docs2html +--------- +Converts markdown docs to html for preview. Requires Pandoc. +Usage: + + cd docs + ../scripts/docs2html.sh + + +generate_ref +------------ +Generates a class reference. +Usage: + + cd docs + ../bin/python ../scripts/generate_ref.py > class_reference.md + + +generate_all +------------ +Does everything: + + - Generates the class reference using generate_ref + - Generates the table of contents using generate_toc + - Converts to HTML for visual inspection using docs2html + +Usage: + + cd docs + ../scripts/generate_all.sh + + +test_python3 +------------ +Creates a Python 3 virtualenv, clones the project into it, and runs the tests. +Usage: + + ./test_python3.sh diff --git a/scripts/docs2html.sh b/scripts/docs2html.sh new file mode 100755 index 0000000..3ed15a5 --- /dev/null +++ b/scripts/docs2html.sh @@ -0,0 +1,10 @@ + +mkdir -p ../htmldocs + +find ./ -iname "*.md" -type f -exec sh -c 'echo "Converting ${0}"; pandoc "${0}" -s -o "../htmldocs/${0%.md}.html"' {} \; + +echo "Converting README.md" +pandoc ../README.md -s -o "../htmldocs/README.html" + +echo "Fixing links" +sed -i 's/\.md/\.html/g' ../htmldocs/*.html diff --git a/scripts/generate_all.sh b/scripts/generate_all.sh new file mode 100755 index 0000000..eabf65c --- /dev/null +++ b/scripts/generate_all.sh @@ -0,0 +1,8 @@ +# Class reference +../bin/python ../scripts/generate_ref.py > class_reference.md + +# Table of contents +../scripts/generate_toc.sh + +# Convert to HTML for visual inspection +../scripts/docs2html.sh diff --git a/scripts/generate_ref.py b/scripts/generate_ref.py new file mode 100644 index 0000000..8d11249 --- /dev/null +++ b/scripts/generate_ref.py @@ -0,0 +1,135 @@ + +import inspect +from collections import namedtuple + +DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value') + +def _get_default_arg(args, defaults, arg_index): + """ Method that determines if an argument has default value or not, + and if yes what is the default value for the argument + + :param args: array of arguments, eg: ['first_arg', 'second_arg', 'third_arg'] + :param defaults: array of default values, eg: (42, 'something') + :param arg_index: index of the argument in the argument array for which, + this function checks if a default value exists or not. And if default value + exists it would return the default value. Example argument: 1 + :return: Tuple of whether there is a default or not, and if yes the default + value, eg: for index 2 i.e. for "second_arg" this function returns (True, 42) + """ + if not defaults: + return DefaultArgSpec(False, None) + + args_with_no_defaults = len(args) - len(defaults) + + if arg_index < args_with_no_defaults: + return DefaultArgSpec(False, None) + else: + value = defaults[arg_index - args_with_no_defaults] + if (type(value) is str): + value = '"%s"' % value + return DefaultArgSpec(True, value) + +def get_method_sig(method): + """ Given a function, it returns a string that pretty much looks how the + function signature would be written in python. + + :param method: a python method + :return: A string similar describing the pythong method signature. + eg: "my_method(first_argArg, second_arg=42, third_arg='something')" + """ + + # The return value of ArgSpec is a bit weird, as the list of arguments and + # list of defaults are returned in separate array. + # eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'], + # varargs=None, keywords=None, defaults=(42, 'something')) + argspec = inspect.getargspec(method) + arg_index=0 + args = [] + + # Use the args and defaults array returned by argspec and find out + # which arguments has default + for arg in argspec.args: + default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index) + if default_arg.has_default: + args.append("%s=%s" % (arg, default_arg.default_value)) + else: + args.append(arg) + arg_index += 1 + if argspec.varargs: + args.append('*' + argspec.varargs) + if argspec.keywords: + args.append('**' + argspec.keywords) + return "%s(%s)" % (method.__name__, ", ".join(args[1:])) + + +def docstring(obj): + doc = (obj.__doc__ or '').rstrip() + if doc: + lines = doc.split('\n') + # Find the length of the whitespace prefix common to all non-empty lines + indentation = min(len(line) - len(line.lstrip()) for line in lines if line.strip()) + # Output the lines without the indentation + for line in lines: + print line[indentation:] + print + + +def class_doc(cls, list_methods=True): + bases = ', '.join([b.__name__ for b in cls.__bases__]) + print '###', cls.__name__ + print + if bases != 'object': + print 'Extends', bases + print + docstring(cls) + for name, method in inspect.getmembers(cls, inspect.ismethod): + if name == '__init__': + # Initializer + print '####', get_method_sig(method).replace(name, cls.__name__) + elif name[0] == '_': + # Private method + continue + elif method.__self__ == cls: + # Class method + if not list_methods: + continue + print '#### %s.%s' % (cls.__name__, get_method_sig(method)) + else: + # Regular method + if not list_methods: + continue + print '####', get_method_sig(method) + print + docstring(method) + print + + +def module_doc(classes, list_methods=True): + mdl = classes[0].__module__ + print mdl + print '-' * len(mdl) + print + for cls in classes: + class_doc(cls, list_methods) + + +def all_subclasses(cls): + return cls.__subclasses__() + [g for s in cls.__subclasses__() for g in all_subclasses(s)] + + +if __name__ == '__main__': + + from infi.clickhouse_orm import database + from infi.clickhouse_orm import fields + from infi.clickhouse_orm import engines + from infi.clickhouse_orm import models + from infi.clickhouse_orm import query + + print 'Class Reference' + print '===============' + print + module_doc([database.Database, database.DatabaseException]) + module_doc([models.Model, models.BufferModel]) + module_doc([fields.Field] + all_subclasses(fields.Field), False) + module_doc([engines.Engine] + all_subclasses(engines.Engine), False) + module_doc([query.QuerySet]) diff --git a/scripts/generate_toc.sh b/scripts/generate_toc.sh new file mode 100755 index 0000000..7ed82ce --- /dev/null +++ b/scripts/generate_toc.sh @@ -0,0 +1,17 @@ + +generate_one() { + # Converts Markdown to HTML using Pandoc, and then extracts the header tags + pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md +} + +printf "# Table of Contents\n\n" > toc.md + +generate_one "index.md" +generate_one "models_and_databases.md" +generate_one "querysets.md" +generate_one "field_types.md" +generate_one "table_engines.md" +generate_one "schema_migrations.md" +generate_one "system_models.md" +generate_one "contributing.md" +generate_one "class_reference.md" diff --git a/scripts/html_to_markdown_toc.py b/scripts/html_to_markdown_toc.py new file mode 100644 index 0000000..169e698 --- /dev/null +++ b/scripts/html_to_markdown_toc.py @@ -0,0 +1,31 @@ +from HTMLParser import HTMLParser +import sys + + +HEADER_TAGS = ('h1', 'h2', 'h3') + + +class HeadersToMarkdownParser(HTMLParser): + + inside = None + text = '' + + def handle_starttag(self, tag, attrs): + if tag.lower() in HEADER_TAGS: + self.inside = tag + + def handle_endtag(self, tag): + if tag.lower() in HEADER_TAGS: + indent = ' ' * int(self.inside[1]) + fragment = self.text.lower().replace(' ', '-') + print '%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment) + self.inside = None + self.text = '' + + def handle_data(self, data): + if self.inside: + self.text += data + + +HeadersToMarkdownParser().feed(sys.stdin.read()) +print diff --git a/scripts/test_python3.sh b/scripts/test_python3.sh new file mode 100755 index 0000000..86016cd --- /dev/null +++ b/scripts/test_python3.sh @@ -0,0 +1,11 @@ +#!/bin/bash +cd /tmp +rm -rf /tmp/orm_env* +virtualenv -p python3 /tmp/orm_env +cd /tmp/orm_env +source bin/activate +pip install infi.projector +git clone https://github.com/Infinidat/infi.clickhouse_orm.git +cd infi.clickhouse_orm +projector devenv build +bin/nosetests diff --git a/src/infi/clickhouse_orm/database.py b/src/infi/clickhouse_orm/database.py index 1103a1b..6af6494 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/src/infi/clickhouse_orm/database.py @@ -16,39 +16,77 @@ Page = namedtuple('Page', 'objects number_of_objects pages_total number page_siz class DatabaseException(Exception): + ''' + Raised when a database operation fails. + ''' pass class Database(object): + ''' + Database instances connect to a specific ClickHouse database for running queries, + inserting data and other operations. + ''' def __init__(self, db_name, db_url='http://localhost:8123/', username=None, password=None, readonly=False): + ''' + Initializes a database instance. Unless it's readonly, the database will be + created on the ClickHouse server if it does not already exist. + + - `db_name`: name of the database to connect to. + - `db_url`: URL of the ClickHouse server. + - `username`: optional connection credentials. + - `password`: optional connection credentials. + - `readonly`: use a read-only connection. + ''' self.db_name = db_name self.db_url = db_url self.username = username self.password = password - self.readonly = readonly - if not self.readonly: + self.readonly = False + if readonly: + self.connection_readonly = self._is_connection_readonly() + self.readonly = True + else: self.create_database() self.server_timezone = self._get_server_timezone() def create_database(self): + ''' + Creates the database on the ClickHouse server if it does not already exist. + ''' self._send('CREATE DATABASE IF NOT EXISTS `%s`' % self.db_name) def drop_database(self): + ''' + Deletes the database on the ClickHouse server. + ''' self._send('DROP DATABASE `%s`' % self.db_name) def create_table(self, model_class): + ''' + Creates a table for the given model class, if it does not exist already. + ''' # TODO check that model has an engine if model_class.readonly: raise DatabaseException("You can't create read only table") self._send(model_class.create_table_sql(self.db_name)) def drop_table(self, model_class): + ''' + Drops the database table of the given model class, if it exists. + ''' if model_class.readonly: raise DatabaseException("You can't drop read only table") self._send(model_class.drop_table_sql(self.db_name)) def insert(self, model_instances, batch_size=1000): + ''' + Insert records into the database. + + - `model_instances`: any iterable containing instances of a single model class. + - `batch_size`: number of records to send per chunk (use a lower number if your records are very large). + ''' from six import next from io import BytesIO i = iter(model_instances) @@ -86,6 +124,12 @@ class Database(object): self._send(gen()) def count(self, model_class, conditions=None): + ''' + Counts the number of records in the model's table. + + - `model_class`: the model to count. + - `conditions`: optional SQL conditions (contents of the WHERE clause). + ''' query = 'SELECT count() FROM $table' if conditions: query += ' WHERE ' + conditions @@ -94,6 +138,14 @@ class Database(object): return int(r.text) if r.text else 0 def select(self, query, model_class=None, settings=None): + ''' + Performs a query and returns a generator of model instances. + + - `query`: the SQL query to execute. + - `model_class`: the model class matching the query's table, + or `None` for getting back instances of an ad-hoc model. + - `settings`: query settings to send as HTTP GET parameters + ''' query += ' FORMAT TabSeparatedWithNamesAndTypes' query = self._substitute(query, model_class) r = self._send(query, settings, True) @@ -107,17 +159,31 @@ class Database(object): yield model_class.from_tsv(line, field_names, self.server_timezone, self) def raw(self, query, settings=None, stream=False): - """ - Performs raw query to database. Returns its output - :param query: Query to execute - :param settings: Query settings to send as query GET parameters - :param stream: If flag is true, Http response from ClickHouse will be streamed. - :return: Query execution result - """ + ''' + Performs a query and returns its output as text. + + - `query`: the SQL query to execute. + - `settings`: query settings to send as HTTP GET parameters + - `stream`: if true, the HTTP response from ClickHouse will be streamed. + ''' query = self._substitute(query, None) return self._send(query, settings=settings, stream=stream).text def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None): + ''' + Selects records and returns a single page of model instances. + + - `model_class`: the model class matching the query's table, + or `None` for getting back instances of an ad-hoc model. + - `order_by`: columns to use for sorting the query (contents of the ORDER BY clause). + - `page_num`: the page number (1-based), or -1 to get the last page. + - `page_size`: number of records to return per page. + - `conditions`: optional SQL conditions (contents of the WHERE clause). + - `settings`: query settings to send as HTTP GET parameters + + The result is a namedtuple containing `objects` (list), `number_of_objects`, + `pages_total`, `number` (of the current page), and `page_size`. + ''' count = self.count(model_class, conditions) pages_total = int(ceil(count / float(page_size))) if page_num == -1: @@ -140,6 +206,13 @@ class Database(object): ) def migrate(self, migrations_package_name, up_to=9999): + ''' + Executes schema migrations. + + - `migrations_package_name` - fully qualified name of the Python package + containing the migrations. + - `up_to` - number of the last migration to apply. + ''' from .migrations import MigrationHistory logger = logging.getLogger('migrations') applied_migrations = self._get_applied_migrations(migrations_package_name) @@ -161,7 +234,7 @@ class Database(object): return set(obj.module_name for obj in self.select(query)) def _send(self, data, settings=None, stream=False): - if PY3 and isinstance(data, string_types): + if isinstance(data, string_types): data = data.encode('utf-8') params = self._build_params(settings) r = requests.post(self.db_url, params=params, data=data, stream=stream) @@ -175,7 +248,8 @@ class Database(object): params['user'] = self.username if self.password: params['password'] = self.password - if self.readonly: + # Send the readonly flag, unless the connection is already readonly (to prevent db error) + if self.readonly and not self.connection_readonly: params['readonly'] = '1' return params @@ -197,3 +271,7 @@ class Database(object): except DatabaseException: logger.exception('Cannot determine server timezone, assuming UTC') return pytz.utc + + def _is_connection_readonly(self): + r = self._send("SELECT value FROM system.settings WHERE name = 'readonly'") + return r.text.strip() != '0' diff --git a/src/infi/clickhouse_orm/engines.py b/src/infi/clickhouse_orm/engines.py index c26b451..9db37da 100644 --- a/src/infi/clickhouse_orm/engines.py +++ b/src/infi/clickhouse_orm/engines.py @@ -5,6 +5,24 @@ class Engine(object): raise NotImplementedError() +class TinyLog(Engine): + + def create_table_sql(self): + return 'TinyLog' + + +class Log(Engine): + + def create_table_sql(self): + return 'Log' + + +class Memory(Engine): + + def create_table_sql(self): + return 'Memory' + + class MergeTree(Engine): def __init__(self, date_col, key_cols, sampling_expr=None, @@ -80,8 +98,10 @@ class ReplacingMergeTree(MergeTree): class Buffer(Engine): - """Here we define Buffer engine - Read more here https://clickhouse.yandex/reference_en.html#Buffer + """ + Buffers the data to write in RAM, periodically flushing it to another table. + Must be used in conjuction with a `BufferModel`. + Read more [here](https://clickhouse.yandex/reference_en.html#Buffer). """ #Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index f7d3992..a136601 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -8,7 +8,9 @@ from .utils import escape, parse_array class Field(object): - + ''' + Abstract base class for all field types. + ''' creation_counter = 0 class_default = 0 db_type = None @@ -90,6 +92,24 @@ class StringField(Field): raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) +class FixedStringField(StringField): + + def __init__(self, length, default=None, alias=None, materialized=None): + self._length = length + self.db_type = 'FixedString(%d)' % length + super(FixedStringField, self).__init__(default, alias, materialized) + + def to_python(self, value, timezone_in_use): + value = super(FixedStringField, self).to_python(value, timezone_in_use) + return value.rstrip('\0') + + def validate(self, value): + if isinstance(value, text_type): + value = value.encode('UTF-8') + if len(value) > self._length: + raise ValueError('Value of %d bytes is too long for FixedStringField(%d)' % (len(value), self._length)) + + class DateField(Field): min_value = datetime.date(1970, 1, 1) @@ -147,7 +167,9 @@ class DateTimeField(Field): class BaseIntField(Field): - + ''' + Abstract base class for all integer-type fields. + ''' def to_python(self, value, timezone_in_use): try: return int(value) @@ -220,6 +242,9 @@ class Int64Field(BaseIntField): class BaseFloatField(Field): + ''' + Abstract base class for all float-type fields. + ''' def to_python(self, value, timezone_in_use): try: @@ -244,6 +269,9 @@ class Float64Field(BaseFloatField): class BaseEnumField(Field): + ''' + Abstract base class for all enum-type fields. + ''' def __init__(self, enum_cls, default=None, alias=None, materialized=None): self.enum_cls = enum_cls diff --git a/src/infi/clickhouse_orm/models.py b/src/infi/clickhouse_orm/models.py index 44437a9..1938103 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/src/infi/clickhouse_orm/models.py @@ -5,6 +5,7 @@ import pytz from .fields import Field from .utils import parse_tsv +from .query import QuerySet logger = getLogger('clickhouse_orm') @@ -57,6 +58,10 @@ class ModelBase(type): if db_type.startswith('Array'): inner_field = cls.create_ad_hoc_field(db_type[6 : -1]) return orm_fields.ArrayField(inner_field) + # FixedString + if db_type.startswith('FixedString'): + length = int(db_type[12 : -1]) + return orm_fields.FixedStringField(length) # Simple fields name = db_type + 'Field' if not hasattr(orm_fields, name): @@ -66,7 +71,13 @@ class ModelBase(type): class Model(with_metaclass(ModelBase)): ''' - A base class for ORM models. + A base class for ORM models. Each model class represent a ClickHouse table. For example: + + class CPUStats(Model): + timestamp = DateTimeField() + cpu_id = UInt16Field() + cpu_percent = Float32Field() + engine = Memory() ''' engine = None @@ -76,8 +87,8 @@ class Model(with_metaclass(ModelBase)): ''' Creates a model instance, using keyword arguments as field values. Since values are immediately converted to their Pythonic type, - invalid values will cause a ValueError to be raised. - Unrecognized field names will cause an AttributeError. + invalid values will cause a `ValueError` to be raised. + Unrecognized field names will cause an `AttributeError`. ''' super(Model, self).__init__() @@ -98,7 +109,7 @@ class Model(with_metaclass(ModelBase)): def __setattr__(self, name, value): ''' When setting a field value, converts the value to its Pythonic type and validates it. - This may raise a ValueError. + This may raise a `ValueError`. ''' field = self.get_field(name) if field: @@ -107,26 +118,25 @@ class Model(with_metaclass(ModelBase)): super(Model, self).__setattr__(name, value) def set_database(self, db): - """ - Sets _database attribute for current model instance - :param db: Database instance - :return: None - """ + ''' + Sets the `Database` that this model instance belongs to. + This is done automatically when the instance is read from the database or written to it. + ''' # This can not be imported globally due to circular import from .database import Database assert isinstance(db, Database), "database must be database.Database instance" self._database = db def get_database(self): - """ - Gets _database attribute for current model instance - :return: database.Database instance, model was inserted or selected from or None - """ + ''' + Gets the `Database` that this model instance belongs to. + Returns `None` unless the instance was read from the database or written to it. + ''' return self._database def get_field(self, name): ''' - Get a Field instance given its name, or None if not found. + Gets a `Field` instance given its name, or `None` if not found. ''' field = getattr(self.__class__, name, None) return field if isinstance(field, Field) else None @@ -134,7 +144,9 @@ class Model(with_metaclass(ModelBase)): @classmethod def table_name(cls): ''' - Returns the model's database table name. + Returns the model's database table name. By default this is the + class name converted to lowercase. Override this if you want to use + a different table name. ''' return cls.__name__.lower() @@ -163,9 +175,13 @@ class Model(with_metaclass(ModelBase)): def from_tsv(cls, line, field_names=None, timezone_in_use=pytz.utc, database=None): ''' Create a model instance from a tab-separated line. The line may or may not include a newline. - The field_names list must match the fields defined in the model, but does not have to include all of them. + The `field_names` list must match the fields defined in the model, but does not have to include all of them. If omitted, it is assumed to be the names of all fields in the model, in order of definition. - :param database: if given, model receives database + + - `line`: the TSV-formatted data. + - `field_names`: names of the model fields in the data. + - `timezone_in_use`: the timezone to use when parsing dates and datetimes. + - `database`: if given, sets the database that this instance belongs to. ''' from six import next field_names = field_names or [name for name, field in cls._fields] @@ -184,7 +200,8 @@ class Model(with_metaclass(ModelBase)): def to_tsv(self, include_readonly=True): ''' Returns the instance's column values as a tab-separated line. A newline is not included. - :param bool include_readonly: If False, returns only fields, that can be inserted into database + + - `include_readonly`: if false, returns only fields that can be inserted into database. ''' data = self.__dict__ fields = self._fields if include_readonly else self._writable_fields @@ -193,8 +210,9 @@ class Model(with_metaclass(ModelBase)): def to_dict(self, include_readonly=True, field_names=None): ''' Returns the instance's column values as a dict. - :param bool include_readonly: If False, returns only fields, that can be inserted into database - :param field_names: An iterable of field names to return + + - `include_readonly`: if false, returns only fields that can be inserted into database. + - `field_names`: an iterable of field names to return (optional) ''' fields = self._fields if include_readonly else self._writable_fields @@ -204,6 +222,13 @@ class Model(with_metaclass(ModelBase)): data = self.__dict__ return {name: data[name] for name, field in fields} + @classmethod + def objects_in(cls, database): + ''' + Returns a `QuerySet` for selecting instances of this model class. + ''' + return QuerySet(cls, database) + class BufferModel(Model): diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py new file mode 100644 index 0000000..465e500 --- /dev/null +++ b/src/infi/clickhouse_orm/query.py @@ -0,0 +1,256 @@ +import six +import pytz +from copy import copy + + +# TODO +# - and/or between Q objects +# - check that field names are valid +# - qs slicing +# - operators for arrays: length, has, empty + +class Operator(object): + """ + Base class for filtering operators. + """ + + def to_sql(self, model_cls, field_name, value): + """ + Subclasses should implement this method. It returns an SQL string + that applies this operator on the given field and value. + """ + raise NotImplementedError + + +class SimpleOperator(Operator): + """ + A simple binary operator such as a=b, ab etc. + """ + + def __init__(self, sql_operator): + self._sql_operator = sql_operator + + def to_sql(self, model_cls, field_name, value): + field = getattr(model_cls, field_name) + value = field.to_db_string(field.to_python(value, pytz.utc)) + return ' '.join([field_name, self._sql_operator, value]) + + +class InOperator(Operator): + """ + An operator that implements IN. + Accepts 3 different types of values: + - a list or tuple of simple values + - a string (used verbatim as the contents of the parenthesis) + - a queryset (subquery) + """ + + def to_sql(self, model_cls, field_name, value): + field = getattr(model_cls, field_name) + if isinstance(value, QuerySet): + value = value.as_sql() + elif isinstance(value, six.string_types): + pass + else: + value = ', '.join([field.to_db_string(field.to_python(v, pytz.utc)) for v in value]) + return '%s IN (%s)' % (field_name, value) + + +class LikeOperator(Operator): + """ + A LIKE operator that matches the field to a given pattern. Can be + case sensitive or insensitive. + """ + + def __init__(self, pattern, case_sensitive=True): + self._pattern = pattern + self._case_sensitive = case_sensitive + + def to_sql(self, model_cls, field_name, value): + field = getattr(model_cls, field_name) + value = field.to_db_string(field.to_python(value, pytz.utc), quote=False) + value = value.replace('\\', '\\\\').replace('%', '\\\\%').replace('_', '\\\\_') + pattern = self._pattern.format(value) + if self._case_sensitive: + return '%s LIKE \'%s\'' % (field_name, pattern) + else: + return 'lowerUTF8(%s) LIKE lowerUTF8(\'%s\')' % (field_name, pattern) + + +class IExactOperator(Operator): + """ + An operator for case insensitive string comparison. + """ + + def to_sql(self, model_cls, field_name, value): + field = getattr(model_cls, field_name) + value = field.to_db_string(field.to_python(value, pytz.utc)) + return 'lowerUTF8(%s) = lowerUTF8(%s)' % (field_name, value) + + +# Define the set of builtin operators + +_operators = {} + +def register_operator(name, sql): + _operators[name] = sql + +register_operator('eq', SimpleOperator('=')) +register_operator('gt', SimpleOperator('>')) +register_operator('gte', SimpleOperator('>=')) +register_operator('lt', SimpleOperator('<')) +register_operator('lte', SimpleOperator('<=')) +register_operator('in', InOperator()) +register_operator('contains', LikeOperator('%{}%')) +register_operator('startswith', LikeOperator('{}%')) +register_operator('endswith', LikeOperator('%{}')) +register_operator('icontains', LikeOperator('%{}%', False)) +register_operator('istartswith', LikeOperator('{}%', False)) +register_operator('iendswith', LikeOperator('%{}', False)) +register_operator('iexact', IExactOperator()) + + +class FOV(object): + """ + An object for storing Field + Operator + Value. + """ + + def __init__(self, field_name, operator, value): + self._field_name = field_name + self._operator = _operators[operator] + self._value = value + + def to_sql(self, model_cls): + return self._operator.to_sql(model_cls, self._field_name, self._value) + + +class Q(object): + + def __init__(self, **kwargs): + self._fovs = [self._build_fov(k, v) for k, v in six.iteritems(kwargs)] + self._negate = False + + def _build_fov(self, key, value): + if '__' in key: + field_name, operator = key.rsplit('__', 1) + else: + field_name, operator = key, 'eq' + return FOV(field_name, operator, value) + + def to_sql(self, model_cls): + if not self._fovs: + return '1' + sql = ' AND '.join(fov.to_sql(model_cls) for fov in self._fovs) + if self._negate: + sql = 'NOT (%s)' % sql + return sql + + def __invert__(self): + q = copy(self) + q._negate = True + return q + + +class QuerySet(object): + """ + A queryset is an object that represents a database query using a specific `Model`. + It is lazy, meaning that it does not hit the database until you iterate over its + matching rows (model instances). + """ + + def __init__(self, model_cls, database): + """ + Initializer. It is possible to create a queryset like this, but the standard + way is to use `MyModel.objects_in(database)`. + """ + self._model_cls = model_cls + self._database = database + self._order_by = [f[0] for f in model_cls._fields] + self._q = [] + self._fields = [] + + def __iter__(self): + """ + Iterates over the model instances matching this queryset + """ + return self._database.select(self.as_sql(), self._model_cls) + + def __bool__(self): + """ + Returns true if this queryset matches any rows. + """ + return bool(self.count()) + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) + + def __unicode__(self): + return self.as_sql() + + def as_sql(self): + """ + Returns the whole query as a SQL string. + """ + fields = '*' + if self._fields: + fields = ', '.join('`%s`' % field for field in self._fields) + params = (fields, self._database.db_name, self._model_cls.table_name(), self.conditions_as_sql(), self.order_by_as_sql()) + return u'SELECT %s\nFROM `%s`.`%s`\nWHERE %s\nORDER BY %s' % params + + def order_by_as_sql(self): + """ + Returns the contents of the query's `ORDER BY` clause as a string. + """ + return u', '.join([ + '%s DESC' % field[1:] if field[0] == '-' else field + for field in self._order_by + ]) + + def conditions_as_sql(self): + """ + Returns the contents of the query's `WHERE` clause as a string. + """ + if self._q: + return u' AND '.join([q.to_sql(self._model_cls) for q in self._q]) + else: + return u'1' + + def count(self): + """ + Returns the number of matching model instances. + """ + return self._database.count(self._model_cls, self.conditions_as_sql()) + + def order_by(self, *field_names): + """ + Returns a new `QuerySet` instance with the ordering changed. + """ + qs = copy(self) + qs._order_by = field_names + return qs + + def only(self, *field_names): + """ + Returns a new `QuerySet` instance limited to the specified field names. + Useful when there are large fields that are not needed, + or for creating a subquery to use with an IN operator. + """ + qs = copy(self) + qs._fields = field_names + return qs + + def filter(self, **kwargs): + """ + Returns a new `QuerySet` instance that includes only rows matching the conditions. + """ + qs = copy(self) + qs._q = list(self._q) + [Q(**kwargs)] + return qs + + def exclude(self, **kwargs): + """ + Returns a new `QuerySet` instance that excludes all rows matching the conditions. + """ + qs = copy(self) + qs._q = list(self._q) + [~Q(**kwargs)] + return qs diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py new file mode 100644 index 0000000..d50d311 --- /dev/null +++ b/tests/base_test_with_data.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +import unittest + +from infi.clickhouse_orm.database import Database +from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.fields import * +from infi.clickhouse_orm.engines import * + +import logging +logging.getLogger("requests").setLevel(logging.WARNING) + + +class TestCaseWithData(unittest.TestCase): + + def setUp(self): + self.database = Database('test-db') + self.database.create_table(Person) + + def tearDown(self): + self.database.drop_table(Person) + self.database.drop_database() + + def _insert_and_check(self, data, count): + self.database.insert(data) + self.assertEquals(count, self.database.count(Person)) + for instance in data: + self.assertEquals(self.database, instance.get_database()) + + def _sample_data(self): + for entry in data: + yield Person(**entry) + + +class Person(Model): + + first_name = StringField() + last_name = StringField() + birthday = DateField() + height = Float32Field() + + engine = MergeTree('birthday', ('first_name', 'last_name', 'birthday')) + + +data = [ + {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63"}, + {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74"}, + {"first_name": "Adena", "last_name": "Norman", "birthday": "1979-05-14", "height": "1.66"}, + {"first_name": "Aline", "last_name": "Crane", "birthday": "1988-05-01", "height": "1.62"}, + {"first_name": "Althea", "last_name": "Barrett", "birthday": "2004-07-28", "height": "1.71"}, + {"first_name": "Amanda", "last_name": "Vang", "birthday": "1973-02-23", "height": "1.68"}, + {"first_name": "Angela", "last_name": "Sanders", "birthday": "2016-01-08", "height": "1.66"}, + {"first_name": "Anne", "last_name": "Rasmussen", "birthday": "1995-04-03", "height": "1.77"}, + {"first_name": "Ariana", "last_name": "Cole", "birthday": "1977-12-20", "height": "1.72"}, + {"first_name": "Ashton", "last_name": "Fuller", "birthday": "1995-11-17", "height": "1.75"}, + {"first_name": "Ava", "last_name": "Sanders", "birthday": "1997-08-10", "height": "1.60"}, + {"first_name": "Barrett", "last_name": "Clemons", "birthday": "1985-07-03", "height": "1.71"}, + {"first_name": "Beatrice", "last_name": "Gregory", "birthday": "1992-01-19", "height": "1.80"}, + {"first_name": "Buffy", "last_name": "Webb", "birthday": "1990-03-06", "height": "1.68"}, + {"first_name": "Callie", "last_name": "Wiley", "birthday": "1987-11-24", "height": "1.69"}, + {"first_name": "Cara", "last_name": "Fox", "birthday": "2004-05-15", "height": "1.71"}, + {"first_name": "Caryn", "last_name": "Sears", "birthday": "1999-02-17", "height": "1.71"}, + {"first_name": "Cassady", "last_name": "Knapp", "birthday": "1977-12-15", "height": "1.72"}, + {"first_name": "Cassady", "last_name": "Rogers", "birthday": "2013-11-04", "height": "1.71"}, + {"first_name": "Catherine", "last_name": "Hicks", "birthday": "1989-05-23", "height": "1.80"}, + {"first_name": "Cathleen", "last_name": "Frank", "birthday": "1977-09-04", "height": "1.61"}, + {"first_name": "Celeste", "last_name": "James", "birthday": "1990-03-08", "height": "1.67"}, + {"first_name": "Chelsea", "last_name": "Castro", "birthday": "2001-08-10", "height": "1.71"}, + {"first_name": "Ciaran", "last_name": "Carver", "birthday": "2016-12-25", "height": "1.76"}, + {"first_name": "Ciaran", "last_name": "Hurley", "birthday": "1995-10-25", "height": "1.65"}, + {"first_name": "Clementine", "last_name": "Moon", "birthday": "1994-03-29", "height": "1.73"}, + {"first_name": "Connor", "last_name": "Jenkins", "birthday": "1999-07-23", "height": "1.67"}, + {"first_name": "Courtney", "last_name": "Cannon", "birthday": "1997-10-26", "height": "1.76"}, + {"first_name": "Courtney", "last_name": "Hoffman", "birthday": "1994-11-07", "height": "1.65"}, + {"first_name": "Denton", "last_name": "Sanchez", "birthday": "1971-10-16", "height": "1.72"}, + {"first_name": "Dominique", "last_name": "Sandoval", "birthday": "1972-02-01", "height": "1.72"}, + {"first_name": "Dora", "last_name": "Cabrera", "birthday": "2016-04-26", "height": "1.68"}, + {"first_name": "Eagan", "last_name": "Dodson", "birthday": "2015-10-22", "height": "1.67"}, + {"first_name": "Edan", "last_name": "Dennis", "birthday": "1989-09-18", "height": "1.73"}, + {"first_name": "Ella", "last_name": "Castillo", "birthday": "1973-03-28", "height": "1.73"}, + {"first_name": "Elton", "last_name": "Ayers", "birthday": "1994-06-20", "height": "1.68"}, + {"first_name": "Elton", "last_name": "Smith", "birthday": "1982-06-20", "height": "1.66"}, + {"first_name": "Emma", "last_name": "Clements", "birthday": "1996-08-07", "height": "1.75"}, + {"first_name": "Evangeline", "last_name": "Weber", "birthday": "1984-06-03", "height": "1.70"}, + {"first_name": "Faith", "last_name": "Emerson", "birthday": "1989-12-30", "height": "1.62"}, + {"first_name": "Fritz", "last_name": "Atkinson", "birthday": "2011-06-15", "height": "1.73"}, + {"first_name": "Galvin", "last_name": "Phillips", "birthday": "2004-01-17", "height": "1.74"}, + {"first_name": "Georgia", "last_name": "Kennedy", "birthday": "1974-12-29", "height": "1.66"}, + {"first_name": "Griffith", "last_name": "Henry", "birthday": "1985-04-02", "height": "1.66"}, + {"first_name": "Hedy", "last_name": "Strong", "birthday": "2001-10-04", "height": "1.60"}, + {"first_name": "Hu", "last_name": "May", "birthday": "1976-10-01", "height": "1.76"}, + {"first_name": "Hyacinth", "last_name": "Kent", "birthday": "1971-07-18", "height": "1.72"}, + {"first_name": "Idola", "last_name": "Fulton", "birthday": "1974-11-27", "height": "1.66"}, + {"first_name": "Jarrod", "last_name": "Gibbs", "birthday": "1987-06-13", "height": "1.62"}, + {"first_name": "Jesse", "last_name": "Gomez", "birthday": "2011-01-28", "height": "1.71"}, + {"first_name": "Josiah", "last_name": "Hodges", "birthday": "2011-09-04", "height": "1.68"}, + {"first_name": "Karleigh", "last_name": "Bartlett", "birthday": "1991-10-24", "height": "1.69"}, + {"first_name": "Keelie", "last_name": "Mathis", "birthday": "1993-10-26", "height": "1.69"}, + {"first_name": "Kieran", "last_name": "Solomon", "birthday": "1993-10-30", "height": "1.69"}, + {"first_name": "Laith", "last_name": "Howell", "birthday": "1991-07-07", "height": "1.70"}, + {"first_name": "Leroy", "last_name": "Pacheco", "birthday": "1998-12-30", "height": "1.70"}, + {"first_name": "Lesley", "last_name": "Stephenson", "birthday": "2010-04-10", "height": "1.64"}, + {"first_name": "Macaulay", "last_name": "Rowe", "birthday": "1982-03-02", "height": "1.68"}, + {"first_name": "Macey", "last_name": "Griffin", "birthday": "1971-09-18", "height": "1.63"}, + {"first_name": "Madeline", "last_name": "Kidd", "birthday": "1984-12-09", "height": "1.69"}, + {"first_name": "Maia", "last_name": "Hyde", "birthday": "1972-06-09", "height": "1.74"}, + {"first_name": "Mary", "last_name": "Kirkland", "birthday": "1987-10-09", "height": "1.73"}, + {"first_name": "Molly", "last_name": "Salas", "birthday": "1994-04-23", "height": "1.70"}, + {"first_name": "Montana", "last_name": "Bruce", "birthday": "1982-06-28", "height": "1.66"}, + {"first_name": "Naomi", "last_name": "Hays", "birthday": "2004-11-27", "height": "1.70"}, + {"first_name": "Norman", "last_name": "Santos", "birthday": "1989-01-10", "height": "1.68"}, + {"first_name": "Octavius", "last_name": "Floyd", "birthday": "1985-02-22", "height": "1.68"}, + {"first_name": "Odette", "last_name": "Mcneil", "birthday": "1978-05-21", "height": "1.76"}, + {"first_name": "Oliver", "last_name": "Ashley", "birthday": "2004-08-13", "height": "1.68"}, + {"first_name": "Quon", "last_name": "Wiggins", "birthday": "1992-05-06", "height": "1.74"}, + {"first_name": "Rafael", "last_name": "Parker", "birthday": "2016-01-24", "height": "1.76"}, + {"first_name": "Reese", "last_name": "Noel", "birthday": "1996-11-04", "height": "1.77"}, + {"first_name": "Rhona", "last_name": "Camacho", "birthday": "1976-12-17", "height": "1.59"}, + {"first_name": "Rigel", "last_name": "Oneal", "birthday": "1993-11-05", "height": "1.63"}, + {"first_name": "Roary", "last_name": "Simmons", "birthday": "1986-07-23", "height": "1.63"}, + {"first_name": "Russell", "last_name": "Pruitt", "birthday": "1979-05-04", "height": "1.63"}, + {"first_name": "Sawyer", "last_name": "Fischer", "birthday": "1995-04-01", "height": "1.78"}, + {"first_name": "Scarlett", "last_name": "Durham", "birthday": "2005-09-29", "height": "1.65"}, + {"first_name": "Seth", "last_name": "Serrano", "birthday": "2017-06-02", "height": "1.71"}, + {"first_name": "Shad", "last_name": "Bradshaw", "birthday": "1998-08-25", "height": "1.72"}, + {"first_name": "Shana", "last_name": "Jarvis", "birthday": "1997-05-21", "height": "1.72"}, + {"first_name": "Sharon", "last_name": "Shelton", "birthday": "1970-05-02", "height": "1.65"}, + {"first_name": "Shoshana", "last_name": "Solis", "birthday": "1998-07-18", "height": "1.65"}, + {"first_name": "Stephen", "last_name": "Baxter", "birthday": "2004-09-24", "height": "1.74"}, + {"first_name": "Sydney", "last_name": "Stevens", "birthday": "1989-07-11", "height": "1.70"}, + {"first_name": "Tasha", "last_name": "Campos", "birthday": "1984-02-11", "height": "1.72"}, + {"first_name": "Ulla", "last_name": "Arnold", "birthday": "1990-06-04", "height": "1.63"}, + {"first_name": "Vaughan", "last_name": "Schmidt", "birthday": "1985-06-19", "height": "1.61"}, + {"first_name": "Velma", "last_name": "English", "birthday": "1999-01-18", "height": "1.65"}, + {"first_name": "Venus", "last_name": "Hurst", "birthday": "1993-10-22", "height": "1.72"}, + {"first_name": "Victor", "last_name": "Woods", "birthday": "1989-06-23", "height": "1.67"}, + {"first_name": "Victoria", "last_name": "Slater", "birthday": "2009-07-19", "height": "1.72"}, + {"first_name": "Wang", "last_name": "Goodwin", "birthday": "1983-05-15", "height": "1.66"}, + {"first_name": "Warren", "last_name": "Bowen", "birthday": "2000-07-20", "height": "1.76"}, + {"first_name": "Warren", "last_name": "Dudley", "birthday": "1995-10-23", "height": "1.59"}, + {"first_name": "Whilemina", "last_name": "Blankenship", "birthday": "1970-07-14", "height": "1.66"}, + {"first_name": "Whitney", "last_name": "Durham", "birthday": "1977-09-15", "height": "1.72"}, + {"first_name": "Whitney", "last_name": "Scott", "birthday": "1971-07-04", "height": "1.70"}, + {"first_name": "Wynter", "last_name": "Garcia", "birthday": "1975-01-10", "height": "1.69"}, + {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1997-02-25", "height": "1.74"} +]; diff --git a/tests/test_buffer.py b/tests/test_buffer.py new file mode 100644 index 0000000..91bf656 --- /dev/null +++ b/tests/test_buffer.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import unittest + +from infi.clickhouse_orm.models import BufferModel +from infi.clickhouse_orm.engines import * +from .base_test_with_data import * + + +class BufferTestCase(TestCaseWithData): + + def _insert_and_check_buffer(self, data, count): + self.database.insert(data) + self.assertEquals(count, self.database.count(PersonBuffer)) + + def _sample_buffer_data(self): + for entry in data: + yield PersonBuffer(**entry) + + def test_insert_buffer(self): + self.database.create_table(PersonBuffer) + self._insert_and_check_buffer(self._sample_buffer_data(), len(data)) + + +class PersonBuffer(BufferModel, Person): + + engine = Buffer(Person) + + diff --git a/tests/test_database.py b/tests/test_database.py index 1897d8f..8f94230 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -3,35 +3,10 @@ import unittest from infi.clickhouse_orm.database import Database, DatabaseException -from infi.clickhouse_orm.models import Model, BufferModel -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * - -import logging -logging.getLogger("requests").setLevel(logging.WARNING) +from .base_test_with_data import * -class DatabaseTestCase(unittest.TestCase): - - def setUp(self): - self.database = Database('test-db') - self.database.create_table(Person) - self.database.create_table(PersonBuffer) - - def tearDown(self): - self.database.drop_table(PersonBuffer) - self.database.drop_table(Person) - self.database.drop_database() - - def _insert_and_check(self, data, count): - self.database.insert(data) - self.assertEquals(count, self.database.count(Person)) - for instance in data: - self.assertEquals(self.database, instance.get_database()) - - def _insert_and_check_buffer(self, data, count): - self.database.insert(data) - self.assertEquals(count, self.database.count(PersonBuffer)) +class DatabaseTestCase(TestCaseWithData): def test_insert__generator(self): self._insert_and_check(self._sample_data(), len(data)) @@ -133,6 +108,11 @@ class DatabaseTestCase(unittest.TestCase): with self.assertRaises(ValueError): self.database.paginate(Person, 'first_name, last_name', page_num, 100) + def test_pagination_with_conditions(self): + self._insert_and_check(self._sample_data(), len(data)) + page = self.database.paginate(Person, 'first_name, last_name', 1, 100, conditions="first_name < 'Ava'") + self.assertEquals(page.number_of_objects, 10) + def test_special_chars(self): s = u'אבגד \\\'"`,.;éåäöšž\n\t\0\b\r' p = Person(first_name=s) @@ -140,171 +120,12 @@ class DatabaseTestCase(unittest.TestCase): p = list(self.database.select("SELECT * from $table", Person))[0] self.assertEquals(p.first_name, s) - def test_readonly(self): - orig_database = self.database - self.database = Database(orig_database.db_name, readonly=True) - with self.assertRaises(DatabaseException): - self._insert_and_check(self._sample_data(), len(data)) - self.assertEquals(self.database.count(Person), 0) - with self.assertRaises(DatabaseException): - self.database.drop_table(Person) - with self.assertRaises(DatabaseException): - self.database.drop_database() - self.database = orig_database - - def test_insert_buffer(self): - self._insert_and_check_buffer(self._sample_buffer_data(), len(data)) - - def _sample_data(self): - for entry in data: - yield Person(**entry) - def test_raw(self): self._insert_and_check(self._sample_data(), len(data)) query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" results = self.database.raw(query) self.assertEqual(results, "Whitney\tDurham\t1977-09-15\t1.72\nWhitney\tScott\t1971-07-04\t1.7\n") - def test_insert_readonly(self): - m = ReadOnlyModel(name='readonly') + def test_invalid_user(self): with self.assertRaises(DatabaseException): - self.database.insert([m]) - - def test_create_readonly_table(self): - with self.assertRaises(DatabaseException): - self.database.create_table(ReadOnlyModel) - - def test_drop_readonly_table(self): - with self.assertRaises(DatabaseException): - self.database.drop_table(ReadOnlyModel) - - def _sample_buffer_data(self): - for entry in data: - yield PersonBuffer(**entry) - - - -class Person(Model): - - first_name = StringField() - last_name = StringField() - birthday = DateField() - height = Float32Field() - - engine = MergeTree('birthday', ('first_name', 'last_name', 'birthday')) - - -class ReadOnlyModel(Model): - readonly = True - - name = StringField() - - -class PersonBuffer(BufferModel, Person): - - engine = Buffer(Person) - - - -data = [ - {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63"}, - {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74"}, - {"first_name": "Adena", "last_name": "Norman", "birthday": "1979-05-14", "height": "1.66"}, - {"first_name": "Aline", "last_name": "Crane", "birthday": "1988-05-01", "height": "1.62"}, - {"first_name": "Althea", "last_name": "Barrett", "birthday": "2004-07-28", "height": "1.71"}, - {"first_name": "Amanda", "last_name": "Vang", "birthday": "1973-02-23", "height": "1.68"}, - {"first_name": "Angela", "last_name": "Sanders", "birthday": "2016-01-08", "height": "1.66"}, - {"first_name": "Anne", "last_name": "Rasmussen", "birthday": "1995-04-03", "height": "1.77"}, - {"first_name": "Ariana", "last_name": "Cole", "birthday": "1977-12-20", "height": "1.72"}, - {"first_name": "Ashton", "last_name": "Fuller", "birthday": "1995-11-17", "height": "1.75"}, - {"first_name": "Ava", "last_name": "Sanders", "birthday": "1997-08-10", "height": "1.60"}, - {"first_name": "Barrett", "last_name": "Clemons", "birthday": "1985-07-03", "height": "1.71"}, - {"first_name": "Beatrice", "last_name": "Gregory", "birthday": "1992-01-19", "height": "1.80"}, - {"first_name": "Buffy", "last_name": "Webb", "birthday": "1990-03-06", "height": "1.68"}, - {"first_name": "Callie", "last_name": "Wiley", "birthday": "1987-11-24", "height": "1.69"}, - {"first_name": "Cara", "last_name": "Fox", "birthday": "2004-05-15", "height": "1.71"}, - {"first_name": "Caryn", "last_name": "Sears", "birthday": "1999-02-17", "height": "1.71"}, - {"first_name": "Cassady", "last_name": "Knapp", "birthday": "1977-12-15", "height": "1.72"}, - {"first_name": "Cassady", "last_name": "Rogers", "birthday": "2013-11-04", "height": "1.71"}, - {"first_name": "Catherine", "last_name": "Hicks", "birthday": "1989-05-23", "height": "1.80"}, - {"first_name": "Cathleen", "last_name": "Frank", "birthday": "1977-09-04", "height": "1.61"}, - {"first_name": "Celeste", "last_name": "James", "birthday": "1990-03-08", "height": "1.67"}, - {"first_name": "Chelsea", "last_name": "Castro", "birthday": "2001-08-10", "height": "1.71"}, - {"first_name": "Ciaran", "last_name": "Carver", "birthday": "2016-12-25", "height": "1.76"}, - {"first_name": "Ciaran", "last_name": "Hurley", "birthday": "1995-10-25", "height": "1.65"}, - {"first_name": "Clementine", "last_name": "Moon", "birthday": "1994-03-29", "height": "1.73"}, - {"first_name": "Connor", "last_name": "Jenkins", "birthday": "1999-07-23", "height": "1.67"}, - {"first_name": "Courtney", "last_name": "Cannon", "birthday": "1997-10-26", "height": "1.76"}, - {"first_name": "Courtney", "last_name": "Hoffman", "birthday": "1994-11-07", "height": "1.65"}, - {"first_name": "Denton", "last_name": "Sanchez", "birthday": "1971-10-16", "height": "1.72"}, - {"first_name": "Dominique", "last_name": "Sandoval", "birthday": "1972-02-01", "height": "1.72"}, - {"first_name": "Dora", "last_name": "Cabrera", "birthday": "2016-04-26", "height": "1.68"}, - {"first_name": "Eagan", "last_name": "Dodson", "birthday": "2015-10-22", "height": "1.67"}, - {"first_name": "Edan", "last_name": "Dennis", "birthday": "1989-09-18", "height": "1.73"}, - {"first_name": "Ella", "last_name": "Castillo", "birthday": "1973-03-28", "height": "1.73"}, - {"first_name": "Elton", "last_name": "Ayers", "birthday": "1994-06-20", "height": "1.68"}, - {"first_name": "Elton", "last_name": "Smith", "birthday": "1982-06-20", "height": "1.66"}, - {"first_name": "Emma", "last_name": "Clements", "birthday": "1996-08-07", "height": "1.75"}, - {"first_name": "Evangeline", "last_name": "Weber", "birthday": "1984-06-03", "height": "1.70"}, - {"first_name": "Faith", "last_name": "Emerson", "birthday": "1989-12-30", "height": "1.62"}, - {"first_name": "Fritz", "last_name": "Atkinson", "birthday": "2011-06-15", "height": "1.73"}, - {"first_name": "Galvin", "last_name": "Phillips", "birthday": "2004-01-17", "height": "1.74"}, - {"first_name": "Georgia", "last_name": "Kennedy", "birthday": "1974-12-29", "height": "1.66"}, - {"first_name": "Griffith", "last_name": "Henry", "birthday": "1985-04-02", "height": "1.66"}, - {"first_name": "Hedy", "last_name": "Strong", "birthday": "2001-10-04", "height": "1.60"}, - {"first_name": "Hu", "last_name": "May", "birthday": "1976-10-01", "height": "1.76"}, - {"first_name": "Hyacinth", "last_name": "Kent", "birthday": "1971-07-18", "height": "1.72"}, - {"first_name": "Idola", "last_name": "Fulton", "birthday": "1974-11-27", "height": "1.66"}, - {"first_name": "Jarrod", "last_name": "Gibbs", "birthday": "1987-06-13", "height": "1.62"}, - {"first_name": "Jesse", "last_name": "Gomez", "birthday": "2011-01-28", "height": "1.71"}, - {"first_name": "Josiah", "last_name": "Hodges", "birthday": "2011-09-04", "height": "1.68"}, - {"first_name": "Karleigh", "last_name": "Bartlett", "birthday": "1991-10-24", "height": "1.69"}, - {"first_name": "Keelie", "last_name": "Mathis", "birthday": "1993-10-26", "height": "1.69"}, - {"first_name": "Kieran", "last_name": "Solomon", "birthday": "1993-10-30", "height": "1.69"}, - {"first_name": "Laith", "last_name": "Howell", "birthday": "1991-07-07", "height": "1.70"}, - {"first_name": "Leroy", "last_name": "Pacheco", "birthday": "1998-12-30", "height": "1.70"}, - {"first_name": "Lesley", "last_name": "Stephenson", "birthday": "2010-04-10", "height": "1.64"}, - {"first_name": "Macaulay", "last_name": "Rowe", "birthday": "1982-03-02", "height": "1.68"}, - {"first_name": "Macey", "last_name": "Griffin", "birthday": "1971-09-18", "height": "1.63"}, - {"first_name": "Madeline", "last_name": "Kidd", "birthday": "1984-12-09", "height": "1.69"}, - {"first_name": "Maia", "last_name": "Hyde", "birthday": "1972-06-09", "height": "1.74"}, - {"first_name": "Mary", "last_name": "Kirkland", "birthday": "1987-10-09", "height": "1.73"}, - {"first_name": "Molly", "last_name": "Salas", "birthday": "1994-04-23", "height": "1.70"}, - {"first_name": "Montana", "last_name": "Bruce", "birthday": "1982-06-28", "height": "1.66"}, - {"first_name": "Naomi", "last_name": "Hays", "birthday": "2004-11-27", "height": "1.70"}, - {"first_name": "Norman", "last_name": "Santos", "birthday": "1989-01-10", "height": "1.68"}, - {"first_name": "Octavius", "last_name": "Floyd", "birthday": "1985-02-22", "height": "1.68"}, - {"first_name": "Odette", "last_name": "Mcneil", "birthday": "1978-05-21", "height": "1.76"}, - {"first_name": "Oliver", "last_name": "Ashley", "birthday": "2004-08-13", "height": "1.68"}, - {"first_name": "Quon", "last_name": "Wiggins", "birthday": "1992-05-06", "height": "1.74"}, - {"first_name": "Rafael", "last_name": "Parker", "birthday": "2016-01-24", "height": "1.76"}, - {"first_name": "Reese", "last_name": "Noel", "birthday": "1996-11-04", "height": "1.77"}, - {"first_name": "Rhona", "last_name": "Camacho", "birthday": "1976-12-17", "height": "1.59"}, - {"first_name": "Rigel", "last_name": "Oneal", "birthday": "1993-11-05", "height": "1.63"}, - {"first_name": "Roary", "last_name": "Simmons", "birthday": "1986-07-23", "height": "1.63"}, - {"first_name": "Russell", "last_name": "Pruitt", "birthday": "1979-05-04", "height": "1.63"}, - {"first_name": "Sawyer", "last_name": "Fischer", "birthday": "1995-04-01", "height": "1.78"}, - {"first_name": "Scarlett", "last_name": "Durham", "birthday": "2005-09-29", "height": "1.65"}, - {"first_name": "Seth", "last_name": "Serrano", "birthday": "2017-06-02", "height": "1.71"}, - {"first_name": "Shad", "last_name": "Bradshaw", "birthday": "1998-08-25", "height": "1.72"}, - {"first_name": "Shana", "last_name": "Jarvis", "birthday": "1997-05-21", "height": "1.72"}, - {"first_name": "Sharon", "last_name": "Shelton", "birthday": "1970-05-02", "height": "1.65"}, - {"first_name": "Shoshana", "last_name": "Solis", "birthday": "1998-07-18", "height": "1.65"}, - {"first_name": "Stephen", "last_name": "Baxter", "birthday": "2004-09-24", "height": "1.74"}, - {"first_name": "Sydney", "last_name": "Stevens", "birthday": "1989-07-11", "height": "1.70"}, - {"first_name": "Tasha", "last_name": "Campos", "birthday": "1984-02-11", "height": "1.72"}, - {"first_name": "Ulla", "last_name": "Arnold", "birthday": "1990-06-04", "height": "1.63"}, - {"first_name": "Vaughan", "last_name": "Schmidt", "birthday": "1985-06-19", "height": "1.61"}, - {"first_name": "Velma", "last_name": "English", "birthday": "1999-01-18", "height": "1.65"}, - {"first_name": "Venus", "last_name": "Hurst", "birthday": "1993-10-22", "height": "1.72"}, - {"first_name": "Victor", "last_name": "Woods", "birthday": "1989-06-23", "height": "1.67"}, - {"first_name": "Victoria", "last_name": "Slater", "birthday": "2009-07-19", "height": "1.72"}, - {"first_name": "Wang", "last_name": "Goodwin", "birthday": "1983-05-15", "height": "1.66"}, - {"first_name": "Warren", "last_name": "Bowen", "birthday": "2000-07-20", "height": "1.76"}, - {"first_name": "Warren", "last_name": "Dudley", "birthday": "1995-10-23", "height": "1.59"}, - {"first_name": "Whilemina", "last_name": "Blankenship", "birthday": "1970-07-14", "height": "1.66"}, - {"first_name": "Whitney", "last_name": "Durham", "birthday": "1977-09-15", "height": "1.72"}, - {"first_name": "Whitney", "last_name": "Scott", "birthday": "1971-07-04", "height": "1.70"}, - {"first_name": "Wynter", "last_name": "Garcia", "birthday": "1975-01-10", "height": "1.69"}, - {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1997-02-25", "height": "1.74"} -]; + Database(self.database.db_name, username='default', password='wrong') \ No newline at end of file diff --git a/tests/test_engines.py b/tests/test_engines.py index d3d8865..3639960 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -38,6 +38,11 @@ class EnginesTestCase(unittest.TestCase): engine = MergeTree('date', ('date', 'event_id', 'event_group'), index_granularity=4096) self._create_and_insert(TestModel) + def test_replicated_merge_tree(self): + engine = MergeTree('date', ('date', 'event_id', 'event_group'), replica_table_path='/clickhouse/tables/{layer}-{shard}/hits', replica_name='{replica}') + expected = "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', date, (date, event_id, event_group), 8192)" + self.assertEquals(engine.create_table_sql(), expected) + def test_collapsing_merge_tree(self): class TestModel(SampleModel): engine = CollapsingMergeTree('date', ('date', 'event_id', 'event_group'), 'event_version') @@ -53,6 +58,21 @@ class EnginesTestCase(unittest.TestCase): engine = ReplacingMergeTree('date', ('date', 'event_id', 'event_group'), 'event_uversion') self._create_and_insert(TestModel) + def test_tiny_log(self): + class TestModel(SampleModel): + engine = TinyLog() + self._create_and_insert(TestModel) + + def test_log(self): + class TestModel(SampleModel): + engine = Log() + self._create_and_insert(TestModel) + + def test_memory(self): + class TestModel(SampleModel): + engine = Memory() + self._create_and_insert(TestModel) + class SampleModel(Model): diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py new file mode 100644 index 0000000..f9490af --- /dev/null +++ b/tests/test_fixed_string_fields.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +import unittest + +from infi.clickhouse_orm.database import Database +from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.fields import * +from infi.clickhouse_orm.engines import * + + +class FixedStringFieldsTest(unittest.TestCase): + + def setUp(self): + self.database = Database('test-db') + self.database.create_table(FixedStringModel) + + def tearDown(self): + self.database.drop_database() + + def _insert_sample_data(self): + self.database.insert([ + FixedStringModel(date_field='2016-08-30', fstr_field=''), + FixedStringModel(date_field='2016-08-30'), + FixedStringModel(date_field='2016-08-31', fstr_field='foo'), + FixedStringModel(date_field='2016-08-31', fstr_field=u'לילה') + ]) + + def _assert_sample_data(self, results): + self.assertEquals(len(results), 4) + self.assertEquals(results[0].fstr_field, '') + self.assertEquals(results[1].fstr_field, 'ABCDEFGHIJK') + self.assertEquals(results[2].fstr_field, 'foo') + self.assertEquals(results[3].fstr_field, u'לילה') + + def test_insert_and_select(self): + self._insert_sample_data() + query = 'SELECT * from $table ORDER BY date_field' + results = list(self.database.select(query, FixedStringModel)) + self._assert_sample_data(results) + + def test_ad_hoc_model(self): + self._insert_sample_data() + query = 'SELECT * from $db.fixedstringmodel ORDER BY date_field' + results = list(self.database.select(query)) + self._assert_sample_data(results) + + def test_assignment_error(self): + for value in (17, 'this is too long', u'זה ארוך', None, 99.9): + with self.assertRaises(ValueError): + FixedStringModel(fstr_field=value) + + +class FixedStringModel(Model): + + date_field = DateField() + fstr_field = FixedStringField(12, default='ABCDEFGHIJK') + + engine = MergeTree('date_field', ('date_field',)) diff --git a/tests/test_querysets.py b/tests/test_querysets.py new file mode 100644 index 0000000..27e6c5c --- /dev/null +++ b/tests/test_querysets.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +import unittest + +from infi.clickhouse_orm.database import Database +from .base_test_with_data import * +import logging +from datetime import date, datetime + +try: + Enum # exists in Python 3.4+ +except NameError: + from enum import Enum # use the enum34 library instead + + +class QuerySetTestCase(TestCaseWithData): + + def setUp(self): + super(QuerySetTestCase, self).setUp() + self.database.insert(self._sample_data()) + + def _test_qs(self, qs, expected_count): + logging.info(qs.as_sql()) + for instance in qs: + logging.info('\t%s' % instance.to_dict()) + self.assertEquals(qs.count(), expected_count) + + def test_no_filtering(self): + qs = Person.objects_in(self.database) + self._test_qs(qs, len(data)) + + def test_truthiness(self): + qs = Person.objects_in(self.database) + self.assertTrue(qs.filter(first_name='Connor')) + self.assertFalse(qs.filter(first_name='Willy')) + + def test_filter_string_field(self): + qs = Person.objects_in(self.database) + self._test_qs(qs.filter(first_name='Ciaran'), 2) + self._test_qs(qs.filter(first_name='ciaran'), 0) # case sensitive + self._test_qs(qs.filter(first_name__iexact='ciaran'), 2) # case insensitive + self._test_qs(qs.filter(first_name__gt='Whilemina'), 4) + self._test_qs(qs.filter(first_name__gte='Whilemina'), 5) + self._test_qs(qs.filter(first_name__lt='Adam'), 1) + self._test_qs(qs.filter(first_name__lte='Adam'), 2) + self._test_qs(qs.filter(first_name__in=('Connor', 'Courtney')), 3) # in tuple + self._test_qs(qs.filter(first_name__in=['Connor', 'Courtney']), 3) # in list + self._test_qs(qs.filter(first_name__in="'Connor', 'Courtney'"), 3) # in string + self._test_qs(qs.filter(first_name__contains='sh'), 3) # case sensitive + self._test_qs(qs.filter(first_name__icontains='sh'), 6) # case insensitive + self._test_qs(qs.filter(first_name__startswith='le'), 0) # case sensitive + self._test_qs(qs.filter(first_name__istartswith='Le'), 2) # case insensitive + self._test_qs(qs.filter(first_name__istartswith=''), 100) # empty prefix + self._test_qs(qs.filter(first_name__endswith='IA'), 0) # case sensitive + self._test_qs(qs.filter(first_name__iendswith='ia'), 3) # case insensitive + self._test_qs(qs.filter(first_name__iendswith=''), 100) # empty suffix + + def test_filter_unicode_string(self): + self.database.insert([ + Person(first_name=u'דונלד', last_name=u'דאק') + ]) + qs = Person.objects_in(self.database) + self._test_qs(qs.filter(first_name=u'דונלד'), 1) + + def test_filter_float_field(self): + qs = Person.objects_in(self.database) + self._test_qs(qs.filter(height__gt=2), 0) + self._test_qs(qs.filter(height__lt=1.61), 4) + self._test_qs(qs.filter(height__lt='1.61'), 4) + self._test_qs(qs.exclude(height__lt='1.61'), 96) + self._test_qs(qs.filter(height__gt=0), 100) + self._test_qs(qs.exclude(height__gt=0), 0) + + def test_filter_date_field(self): + qs = Person.objects_in(self.database) + self._test_qs(qs.filter(birthday='1970-12-02'), 1) + self._test_qs(qs.filter(birthday=date(1970, 12, 2)), 1) + self._test_qs(qs.filter(birthday__lte=date(1970, 12, 2)), 3) + + def test_only(self): + qs = Person.objects_in(self.database).only('first_name', 'last_name') + for person in qs: + self.assertTrue(person.first_name) + self.assertTrue(person.last_name) + self.assertFalse(person.height) + self.assertEquals(person.birthday, date(1970, 1, 1)) + + def test_order_by(self): + qs = Person.objects_in(self.database) + person = list(qs.order_by('first_name', 'last_name'))[0] + self.assertEquals(person.first_name, 'Abdul') + person = list(qs.order_by('-first_name', '-last_name'))[0] + self.assertEquals(person.first_name, 'Yolanda') + person = list(qs.order_by('height'))[0] + self.assertEquals(person.height, 1.59) + person = list(qs.order_by('-height'))[0] + self.assertEquals(person.height, 1.8) + + def test_in_subquery(self): + qs = Person.objects_in(self.database) + self._test_qs(qs.filter(height__in='SELECT max(height) FROM $table'), 2) + self._test_qs(qs.filter(first_name__in=qs.only('last_name')), 2) + + def _insert_sample_model(self): + self.database.create_table(SampleModel) + now = datetime.now() + self.database.insert([ + SampleModel(timestamp=now, num=1, color=Color.red), + SampleModel(timestamp=now, num=2, color=Color.red), + SampleModel(timestamp=now, num=3, color=Color.blue), + SampleModel(timestamp=now, num=4, color=Color.white), + ]) + + def test_filter_enum_field(self): + self._insert_sample_model() + qs = SampleModel.objects_in(self.database) + self._test_qs(qs.filter(color=Color.red), 2) + self._test_qs(qs.exclude(color=Color.white), 3) + # Different ways to specify blue + self._test_qs(qs.filter(color__gt=Color.blue), 1) + self._test_qs(qs.filter(color__gt='blue'), 1) + self._test_qs(qs.filter(color__gt=2), 1) + + def test_filter_int_field(self): + self._insert_sample_model() + qs = SampleModel.objects_in(self.database) + self._test_qs(qs.filter(num=1), 1) + self._test_qs(qs.filter(num__gt=1), 3) + self._test_qs(qs.filter(num__gte=1), 4) + self._test_qs(qs.filter(num__in=(1, 2, 3)), 3) + self._test_qs(qs.filter(num__in=range(1, 4)), 3) + + +Color = Enum('Color', u'red blue green yellow brown white black') + + +class SampleModel(Model): + + timestamp = DateTimeField() + materialized_date = DateField(materialized='toDate(timestamp)') + num = Int32Field() + color = Enum8Field(Color) + + engine = MergeTree('materialized_date', ('materialized_date',)) \ No newline at end of file diff --git a/tests/test_readonly.py b/tests/test_readonly.py new file mode 100644 index 0000000..371fdcb --- /dev/null +++ b/tests/test_readonly.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import unittest + +from infi.clickhouse_orm.database import Database, DatabaseException +from infi.clickhouse_orm.models import Model +from infi.clickhouse_orm.fields import * +from infi.clickhouse_orm.engines import * +from .base_test_with_data import * + + +class ReadonlyTestCase(TestCaseWithData): + + def _test_readonly_db(self, username): + self._insert_and_check(self._sample_data(), len(data)) + orig_database = self.database + try: + self.database = Database(orig_database.db_name, username=username, readonly=True) + with self.assertRaises(DatabaseException): + self._insert_and_check(self._sample_data(), len(data)) + self.assertEquals(self.database.count(Person), 100) + list(self.database.select('SELECT * from $table', Person)) + with self.assertRaises(DatabaseException): + self.database.drop_table(Person) + with self.assertRaises(DatabaseException): + self.database.drop_database() + except DatabaseException as e: + if 'Unknown user' in unicode(e): + raise unittest.SkipTest('Database user "%s" is not defined' % username) + else: + raise + finally: + self.database = orig_database + + def test_readonly_db_with_default_user(self): + self._test_readonly_db('default') + + def test_readonly_db_with_readonly_user(self): + self._test_readonly_db('readonly') + + def test_insert_readonly(self): + m = ReadOnlyModel(name='readonly') + with self.assertRaises(DatabaseException): + self.database.insert([m]) + + def test_create_readonly_table(self): + with self.assertRaises(DatabaseException): + self.database.create_table(ReadOnlyModel) + + def test_drop_readonly_table(self): + with self.assertRaises(DatabaseException): + self.database.drop_table(ReadOnlyModel) + + +class ReadOnlyModel(Model): + readonly = True + + name = StringField() +