mirror of
https://github.com/Infinidat/infi.clickhouse_orm.git
synced 2025-07-25 15:29:50 +03:00
Add simple engines: TinyLog, Log, Memory
This commit is contained in:
parent
64bf3b423e
commit
be474b3aed
|
@ -1,8 +1,37 @@
|
||||||
Table Engines
|
Table Engines
|
||||||
=============
|
=============
|
||||||
|
|
||||||
|
See: [ClickHouse Documentation](https://clickhouse.yandex/reference_en.html#Table+engines)
|
||||||
|
|
||||||
Each model must have an engine instance, used when creating the table in ClickHouse.
|
Each model must have an engine instance, used when creating the table in ClickHouse.
|
||||||
|
|
||||||
|
The following engines are supported by the ORM:
|
||||||
|
|
||||||
|
- TinyLog
|
||||||
|
- Log
|
||||||
|
- Memory
|
||||||
|
- MergeTree / ReplicatedMergeTree
|
||||||
|
- CollapsingMergeTree / ReplicatedCollapsingMergeTree
|
||||||
|
- SummingMergeTree / ReplicatedSummingMergeTree
|
||||||
|
- ReplacingMergeTree / ReplicatedReplacingMergeTree
|
||||||
|
- Buffer
|
||||||
|
|
||||||
|
|
||||||
|
Simple Engines
|
||||||
|
--------------
|
||||||
|
|
||||||
|
`TinyLog`, `Log` and `Memory` engines do not require any parameters:
|
||||||
|
|
||||||
|
engine = engines.TinyLog()
|
||||||
|
|
||||||
|
engine = engines.Log()
|
||||||
|
|
||||||
|
engine = engines.Memory()
|
||||||
|
|
||||||
|
|
||||||
|
Engines in the MergeTree Family
|
||||||
|
-------------------------------
|
||||||
|
|
||||||
To define a `MergeTree` engine, supply the date column name and the names (or expressions) for the key columns:
|
To define a `MergeTree` engine, supply the date column name and the names (or expressions) for the key columns:
|
||||||
|
|
||||||
engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'))
|
engine = engines.MergeTree('EventDate', ('CounterID', 'EventDate'))
|
||||||
|
@ -24,32 +53,7 @@ For a `ReplacingMergeTree` you can optionally specify the version column:
|
||||||
|
|
||||||
engine = engines.ReplacingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), ver_col='Version')
|
engine = engines.ReplacingMergeTree('EventDate', ('OrderID', 'EventDate', 'BannerID'), ver_col='Version')
|
||||||
|
|
||||||
A `Buffer` engine is available for BufferModels. (See below how to use BufferModel). You can specify following parameters:
|
### Data Replication
|
||||||
|
|
||||||
engine = engines.Buffer(Person) # you need to initialize engine with main Model. Other default parameters will be used
|
|
||||||
# or:
|
|
||||||
engine = engines.Buffer(Person, num_layers=16, min_time=10,
|
|
||||||
max_time=100, min_rows=10000, max_rows=1000000,
|
|
||||||
min_bytes=10000000, max_bytes=100000000)
|
|
||||||
|
|
||||||
Buffer Models
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Here's how you can define Model for Buffer Engine. The Buffer Model should be inherited from models.BufferModel and main Model:
|
|
||||||
|
|
||||||
class PersonBuffer(models.BufferModel, Person):
|
|
||||||
|
|
||||||
engine = engines.Buffer(Person)
|
|
||||||
|
|
||||||
Then you can insert objects into Buffer model and they will be handled by ClickHouse properly:
|
|
||||||
|
|
||||||
db.create_table(PersonBuffer)
|
|
||||||
suzy = PersonBuffer(first_name='Suzy', last_name='Jones')
|
|
||||||
dan = PersonBuffer(first_name='Dan', last_name='Schwartz')
|
|
||||||
db.insert([dan, suzy])
|
|
||||||
|
|
||||||
Data Replication
|
|
||||||
----------------
|
|
||||||
|
|
||||||
Any of the above engines can be converted to a replicated engine (e.g. `ReplicatedMergeTree`) by adding two parameters, `replica_table_path` and `replica_name`:
|
Any of the above engines can be converted to a replicated engine (e.g. `ReplicatedMergeTree`) by adding two parameters, `replica_table_path` and `replica_name`:
|
||||||
|
|
||||||
|
@ -58,6 +62,31 @@ Any of the above engines can be converted to a replicated engine (e.g. `Replicat
|
||||||
replica_name='{replica}')
|
replica_name='{replica}')
|
||||||
|
|
||||||
|
|
||||||
|
Buffer Engine
|
||||||
|
-------------
|
||||||
|
|
||||||
|
A `Buffer` engine is only used in conjunction with a `BufferModel`.
|
||||||
|
The model should be a subclass of both `models.BufferModel` and the main model.
|
||||||
|
The main model is also passed to the engine:
|
||||||
|
|
||||||
|
class PersonBuffer(models.BufferModel, Person):
|
||||||
|
|
||||||
|
engine = engines.Buffer(Person)
|
||||||
|
|
||||||
|
Additional buffer parameters can optionally be specified:
|
||||||
|
|
||||||
|
engine = engines.Buffer(Person, num_layers=16, min_time=10,
|
||||||
|
max_time=100, min_rows=10000, max_rows=1000000,
|
||||||
|
min_bytes=10000000, max_bytes=100000000)
|
||||||
|
|
||||||
|
Then you can insert objects into Buffer model and they will be handled by ClickHouse properly:
|
||||||
|
|
||||||
|
db.create_table(PersonBuffer)
|
||||||
|
suzy = PersonBuffer(first_name='Suzy', last_name='Jones')
|
||||||
|
dan = PersonBuffer(first_name='Dan', last_name='Schwartz')
|
||||||
|
db.insert([dan, suzy])
|
||||||
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
[<< Field Types](field_types.md) | [Table of Contents](toc.md) | [Schema Migrations >>](schema_migrations.md)
|
[<< Field Types](field_types.md) | [Table of Contents](toc.md) | [Schema Migrations >>](schema_migrations.md)
|
|
@ -5,6 +5,24 @@ class Engine(object):
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
class TinyLog(Engine):
|
||||||
|
|
||||||
|
def create_table_sql(self):
|
||||||
|
return 'TinyLog'
|
||||||
|
|
||||||
|
|
||||||
|
class Log(Engine):
|
||||||
|
|
||||||
|
def create_table_sql(self):
|
||||||
|
return 'Log'
|
||||||
|
|
||||||
|
|
||||||
|
class Memory(Engine):
|
||||||
|
|
||||||
|
def create_table_sql(self):
|
||||||
|
return 'Memory'
|
||||||
|
|
||||||
|
|
||||||
class MergeTree(Engine):
|
class MergeTree(Engine):
|
||||||
|
|
||||||
def __init__(self, date_col, key_cols, sampling_expr=None,
|
def __init__(self, date_col, key_cols, sampling_expr=None,
|
||||||
|
|
|
@ -53,6 +53,21 @@ class EnginesTestCase(unittest.TestCase):
|
||||||
engine = ReplacingMergeTree('date', ('date', 'event_id', 'event_group'), 'event_uversion')
|
engine = ReplacingMergeTree('date', ('date', 'event_id', 'event_group'), 'event_uversion')
|
||||||
self._create_and_insert(TestModel)
|
self._create_and_insert(TestModel)
|
||||||
|
|
||||||
|
def test_tiny_log(self):
|
||||||
|
class TestModel(SampleModel):
|
||||||
|
engine = TinyLog()
|
||||||
|
self._create_and_insert(TestModel)
|
||||||
|
|
||||||
|
def test_log(self):
|
||||||
|
class TestModel(SampleModel):
|
||||||
|
engine = Log()
|
||||||
|
self._create_and_insert(TestModel)
|
||||||
|
|
||||||
|
def test_memory(self):
|
||||||
|
class TestModel(SampleModel):
|
||||||
|
engine = Memory()
|
||||||
|
self._create_and_insert(TestModel)
|
||||||
|
|
||||||
|
|
||||||
class SampleModel(Model):
|
class SampleModel(Model):
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user