From 93747f77583851af4ce230ab8fa490e97b841bcc Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Sat, 8 Feb 2020 12:38:23 +0200 Subject: [PATCH] Improve docs --- docs/class_reference.md | 1167 +++++++++++++++++++++- docs/expressions.md | 85 ++ docs/field_options.md | 112 +++ docs/field_types.md | 108 +- docs/models_and_databases.md | 26 +- docs/querysets.md | 16 +- docs/table_engines.md | 2 +- docs/toc.md | 14 +- scripts/generate_ref.py | 4 + scripts/generate_toc.sh | 1 + scripts/html_to_markdown_toc.py | 6 +- src/infi/clickhouse_orm/engines.py | 6 +- src/infi/clickhouse_orm/fields.py | 11 +- src/infi/clickhouse_orm/funcs.py | 13 +- src/infi/clickhouse_orm/query.py | 2 +- src/infi/clickhouse_orm/system_models.py | 58 +- 16 files changed, 1464 insertions(+), 167 deletions(-) create mode 100644 docs/expressions.md create mode 100644 docs/field_options.md diff --git a/docs/class_reference.md b/docs/class_reference.md index e98e33b..7f038d0 100644 --- a/docs/class_reference.md +++ b/docs/class_reference.md @@ -209,6 +209,13 @@ Returns `None` unless the instance was read from the database or written to it. Gets a `Field` instance given its name, or `None` if not found. +#### Model.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + #### Model.is_read_only() @@ -242,6 +249,12 @@ class name converted to lowercase. Override this if you want to use a different table name. +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + #### to_dict(include_readonly=True, field_names=None) @@ -251,6 +264,15 @@ Returns the instance's column values as a dict. - `field_names`: an iterable of field names to return (optional) +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + #### to_tsv(include_readonly=True) @@ -317,6 +339,13 @@ Returns `None` unless the instance was read from the database or written to it. Gets a `Field` instance given its name, or `None` if not found. +#### BufferModel.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + #### BufferModel.is_read_only() @@ -350,6 +379,12 @@ class name converted to lowercase. Override this if you want to use a different table name. +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + #### to_dict(include_readonly=True, field_names=None) @@ -359,6 +394,15 @@ Returns the instance's column values as a dict. - `field_names`: an iterable of field names to return (optional) +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + #### to_tsv(include_readonly=True) @@ -458,6 +502,13 @@ Returns `None` unless the instance was read from the database or written to it. Gets a `Field` instance given its name, or `None` if not found. +#### DistributedModel.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + #### DistributedModel.is_read_only() @@ -487,6 +538,12 @@ class name converted to lowercase. Override this if you want to use a different table name. +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + #### to_dict(include_readonly=True, field_names=None) @@ -496,6 +553,15 @@ Returns the instance's column values as a dict. - `field_names`: an iterable of field names to return (optional) +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + #### to_tsv(include_readonly=True) @@ -605,6 +671,8 @@ Extends BaseEnumField ### Field +Extends FunctionOperatorsMixin + Abstract base class for all field types. @@ -632,6 +700,20 @@ Extends BaseFloatField #### Float64Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +### IPv4Field + +Extends Field + +#### IPv4Field(default=None, alias=None, materialized=None, readonly=None, codec=None) + + +### IPv6Field + +Extends Field + +#### IPv6Field(default=None, alias=None, materialized=None, readonly=None, codec=None) + + ### Int16Field Extends BaseIntField @@ -781,11 +863,11 @@ https://clickhouse.yandex/docs/en/table_engines/distributed.html #### Distributed(cluster, table=None, sharding_key=None) -:param cluster: what cluster to access data from -:param table: underlying table that actually stores data. +- `cluster`: what cluster to access data from +- `table`: underlying table that actually stores data. If you are not specifying any table here, ensure that it can be inferred from your model's superclass (see models.DistributedModel.fix_engine_table) -:param sharding_key: how to distribute data among shards when inserting +- `sharding_key`: how to distribute data among shards when inserting straightly into Distributed table, optional @@ -876,6 +958,9 @@ Returns a copy of this queryset that excludes all rows matching the conditions. Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE. +#### extra(**kwargs) + + #### filter(*q, **kwargs) @@ -999,6 +1084,9 @@ Returns a copy of this queryset that excludes all rows matching the conditions. Pass `prewhere=True` to apply the conditions as PREWHERE instead of WHERE. +#### extra(**kwargs) + + #### filter(*q, **kwargs) @@ -1075,3 +1163,1076 @@ with aggregate function calculated across all the rows. More information: https://clickhouse.yandex/docs/en/query_language/select/#with-totals-modifier +infi.clickhouse_orm.funcs +------------------------- + +### F + +Extends Cond, FunctionOperatorsMixin + + +Represents a database function call and its arguments. +It doubles as a query condition when the function returns a boolean result. + +#### CAST(type) + + +#### CRC32() + + +#### IPv4CIDRToRange(cidr) + + +#### IPv4NumToString() + + +#### IPv4NumToStringClassC() + + +#### IPv4StringToNum() + + +#### IPv4ToIPv6() + + +#### IPv6CIDRToRange(cidr) + + +#### IPv6NumToString() + + +#### IPv6StringToNum() + + +#### MD5() + + +#### SHA1() + + +#### SHA224() + + +#### SHA256() + + +#### URLHash(n=None) + + +#### UUIDNumToString() + + +#### UUIDStringToNum() + + +#### F(name, *args) + + +Initializer. + + +#### abs() + + +#### acos() + + +#### addDays(n, timezone=None) + + +#### addHours(n, timezone=None) + + +#### addMinutes(n, timezone=None) + + +#### addMonths(n, timezone=None) + + +#### addQuarters(n, timezone=None) + + +#### addSeconds(n, timezone=None) + + +#### addWeeks(n, timezone=None) + + +#### addYears(n, timezone=None) + + +#### alphaTokens() + + +#### appendTrailingCharIfAbsent(c) + + +#### array() + + +#### arrayAll() + + +#### arrayConcat() + + +#### arrayCount() + + +#### arrayCumSum() + + +#### arrayCumSumNonNegative() + + +#### arrayDifference() + + +#### arrayDistinct() + + +#### arrayElement(n) + + +#### arrayEnumerate() + + +#### arrayEnumerateDense() + + +#### arrayEnumerateDenseRanked() + + +#### arrayEnumerateUniq() + + +#### arrayEnumerateUniqRanked() + + +#### arrayExists() + + +#### arrayIntersect() + + +#### arrayJoin() + + +#### arrayPopBack() + + +#### arrayPopFront() + + +#### arrayPushBack(x) + + +#### arrayPushFront(x) + + +#### arrayReduce(*args) + + +#### arrayResize(size, extender=None) + + +#### arrayReverse() + + +#### arrayReverseSort() + + +#### arraySlice(offset, length=None) + + +#### arraySort() + + +#### arrayStringConcat(sep=None) + + +#### arraySum() + + +#### arrayUniq() + + +#### asin() + + +#### atan() + + +#### base64Decode() + + +#### base64Encode() + + +#### bitAnd(y) + + +#### bitNot() + + +#### bitOr(y) + + +#### bitRotateLeft(y) + + +#### bitRotateRight(y) + + +#### bitShiftLeft(y) + + +#### bitShiftRight(y) + + +#### bitTest(y) + + +#### bitTestAll(*args) + + +#### bitTestAny(*args) + + +#### bitXor(y) + + +#### bitmapAnd(y) + + +#### bitmapAndCardinality(y) + + +#### bitmapAndnot(y) + + +#### bitmapAndnotCardinality(y) + + +#### bitmapBuild() + + +#### bitmapCardinality() + + +#### bitmapContains(needle) + + +#### bitmapHasAll(y) + + +#### bitmapHasAny(y) + + +#### bitmapOr(y) + + +#### bitmapOrCardinality(y) + + +#### bitmapToArray() + + +#### bitmapXor(y) + + +#### bitmapXorCardinality(y) + + +#### bitmaskToArray() + + +#### bitmaskToList() + + +#### cbrt() + + +#### ceiling(n=None) + + +#### ceiling(n=None) + + +#### cityHash64() + + +#### concat() + + +#### convertCharset(from_charset, to_charset) + + +#### cos() + + +#### countEqual(x) + + +#### divide(**kwargs) + + +#### e() + + +#### empty() + + +#### emptyArrayDate() + + +#### emptyArrayDateTime() + + +#### emptyArrayFloat32() + + +#### emptyArrayFloat64() + + +#### emptyArrayInt16() + + +#### emptyArrayInt32() + + +#### emptyArrayInt64() + + +#### emptyArrayInt8() + + +#### emptyArrayString() + + +#### emptyArrayToSingle() + + +#### emptyArrayUInt16() + + +#### emptyArrayUInt32() + + +#### emptyArrayUInt64() + + +#### emptyArrayUInt8() + + +#### endsWith(suffix) + + +#### equals(**kwargs) + + +#### erf() + + +#### erfc() + + +#### exp() + + +#### exp10() + + +#### exp2() + + +#### farmHash64() + + +#### floor(n=None) + + +#### formatDateTime(format, timezone="") + + +#### gcd(b) + + +#### generateUUIDv4() + + +#### greater(**kwargs) + + +#### greaterOrEquals(**kwargs) + + +#### halfMD5() + + +#### has(x) + + +#### hasAll(x) + + +#### hasAny(x) + + +#### hex() + + +#### hiveHash() + + +#### indexOf(x) + + +#### intDiv(b) + + +#### intDivOrZero(b) + + +#### intExp10() + + +#### intExp2() + + +#### intHash32() + + +#### intHash64() + + +#### javaHash() + + +#### jumpConsistentHash(buckets) + + +#### lcm(b) + + +#### length() + + +#### lengthUTF8() + + +#### less(**kwargs) + + +#### lessOrEquals(**kwargs) + + +#### lgamma() + + +#### log() + + +#### log() + + +#### log10() + + +#### log2() + + +#### lower() + + +#### lowerUTF8() + + +#### metroHash64() + + +#### minus(**kwargs) + + +#### modulo(**kwargs) + + +#### multiply(**kwargs) + + +#### murmurHash2_32() + + +#### murmurHash2_64() + + +#### murmurHash3_128() + + +#### murmurHash3_32() + + +#### murmurHash3_64() + + +#### negate() + + +#### notEmpty() + + +#### notEquals(**kwargs) + + +#### now() + + +#### parseDateTimeBestEffort(timezone=None) + + +#### parseDateTimeBestEffortOrNull(timezone=None) + + +#### parseDateTimeBestEffortOrZero(timezone=None) + + +#### pi() + + +#### plus(**kwargs) + + +#### power(y) + + +#### power(y) + + +#### rand() + + +#### rand64() + + +#### randConstant() + + +#### range() + + +#### regexpQuoteMeta() + + +#### replace(pattern, replacement) + + +#### replaceAll(pattern, replacement) + + +#### replaceOne(pattern, replacement) + + +#### replaceRegexpAll(pattern, replacement) + + +#### replaceRegexpOne(pattern, replacement) + + +#### reverse() + + +#### reverseUTF8() + + +#### round(n=None) + + +#### roundAge() + + +#### roundDown(y) + + +#### roundDuration() + + +#### roundToExp2() + + +#### sin() + + +#### sipHash128() + + +#### sipHash64() + + +#### splitByChar(s) + + +#### splitByString(s) + + +#### sqrt() + + +#### startsWith(prefix) + + +#### substring(offset, length) + + +#### substringUTF8(offset, length) + + +#### subtractDays(n, timezone=None) + + +#### subtractHours(n, timezone=None) + + +#### subtractMinutes(n, timezone=None) + + +#### subtractMonths(n, timezone=None) + + +#### subtractQuarters(n, timezone=None) + + +#### subtractSeconds(n, timezone=None) + + +#### subtractWeeks(n, timezone=None) + + +#### subtractYears(n, timezone=None) + + +#### tan() + + +#### tgamma() + + +#### timeSlot() + + +#### timeSlots(duration) + + +#### toDate() + + +#### toDateTime() + + +#### toDayOfMonth() + + +#### toDayOfWeek() + + +#### toDecimal128(scale) + + +#### toDecimal32(scale) + + +#### toDecimal64(scale) + + +#### toFixedString(length) + + +#### toFloat32() + + +#### toFloat32OrZero() + + +#### toFloat64() + + +#### toFloat64OrZero() + + +#### toHour() + + +#### toIPv4() + + +#### toIPv6() + + +#### toInt16() + + +#### toInt16OrZero() + + +#### toInt32() + + +#### toInt32OrZero() + + +#### toInt64() + + +#### toInt64OrZero() + + +#### toInt8() + + +#### toInt8OrZero() + + +#### toMinute() + + +#### toMonday() + + +#### toMonth() + + +#### toRelativeDayNum(timezone="") + + +#### toRelativeHourNum(timezone="") + + +#### toRelativeMinuteNum(timezone="") + + +#### toRelativeMonthNum(timezone="") + + +#### toRelativeSecondNum(timezone="") + + +#### toRelativeWeekNum(timezone="") + + +#### toRelativeYearNum(timezone="") + + +#### toSecond() + + +#### toStartOfDay() + + +#### toStartOfFifteenMinutes() + + +#### toStartOfFiveMinute() + + +#### toStartOfHour() + + +#### toStartOfMinute() + + +#### toStartOfMonth() + + +#### toStartOfQuarter() + + +#### toStartOfYear() + + +#### toString() + + +#### toStringCutToZero() + + +#### toTime(timezone="") + + +#### toUInt16() + + +#### toUInt16OrZero() + + +#### toUInt32() + + +#### toUInt32OrZero() + + +#### toUInt64() + + +#### toUInt64OrZero() + + +#### toUInt8() + + +#### toUInt8OrZero() + + +#### toUUID() + + +#### toYear() + + +#### to_sql(*args) + + +Generates an SQL string for this function and its arguments. +For example if the function name is a symbol of a binary operator: + (2.54 * `height`) +For other functions: + gcd(12, 300) + + +#### today() + + +#### trimBoth() + + +#### trimLeft() + + +#### trimRight() + + +#### tryBase64Decode() + + +#### unhex() + + +#### upper() + + +#### upperUTF8() + + +#### xxHash32() + + +#### xxHash64() + + +#### yesterday() + + +infi.clickhouse_orm.system_models +--------------------------------- + +### SystemPart + +Extends Model + + +Contains information about parts of a table in the MergeTree family. +This model operates only fields, described in the reference. Other fields are ignored. +https://clickhouse.yandex/docs/en/system_tables/system.parts/ + +#### SystemPart(**kwargs) + + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### attach(settings=None) + + + Add a new part or partition from the 'detached' directory to the table. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.create_table_sql(db) + + +Returns the SQL command for creating a table for this model. + + +#### detach(settings=None) + + +Move a partition to the 'detached' directory and forget it. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### drop(settings=None) + + +Delete a partition + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.drop_table_sql(db) + + +Returns the SQL command for deleting this model's table. + + +#### fetch(zookeeper_path, settings=None) + + +Download a partition from another server. + +- `zookeeper_path`: Path in zookeeper to fetch from +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.fields(writable=False) + + +Returns an `OrderedDict` of the model's fields (from name to `Field` instance). +If `writable` is true, only writable fields are included. +Callers should not modify the dictionary. + + +#### freeze(settings=None) + + +Create a backup of a partition. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None) + + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. +- `database`: if given, sets the database that this instance belongs to. + + +#### SystemPart.get(database, conditions="") + + +Get all data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database condition is added automatically + +Returns: A list of SystemPart objects + + +#### SystemPart.get_active(database, conditions="") + + +Gets active data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database and active conditions are added automatically + +Returns: A list of SystemPart objects + + +#### get_database() + + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + + +Gets a `Field` instance given its name, or `None` if not found. + + +#### SystemPart.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + +#### SystemPart.is_read_only() + + +Returns true if the model is marked as read only. + + +#### SystemPart.is_system_model() + + +Returns true if the model represents a system table. + + +#### SystemPart.objects_in(database) + + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### SystemPart.table_name() + + +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + +#### to_dict(include_readonly=True, field_names=None) + + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +#### to_tsv(include_readonly=True) + + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + diff --git a/docs/expressions.md b/docs/expressions.md new file mode 100644 index 0000000..66e16e2 --- /dev/null +++ b/docs/expressions.md @@ -0,0 +1,85 @@ + +Expressions +=========== + +One of the ORM's core concepts is _expressions_, which are composed using functions, operators and model fields. Expressions are used in multiple places in the ORM: + +- When defining [field options](field_options.md) - `default`, `alias` and `materialized`. +- In [table engine](table_engines.md) parameters for engines in the `MergeTree` family. +- In [queryset](querysets.md) methods such as `filter`, `exclude`, `order_by`, `extra`, `aggregate` and `limit_by`. + +Using Expressions +----------------- + +Expressions usually include ClickHouse database functions, which are made available by the `F` class. Here's a simple function: +```python +from infi.clickhouse_orm.models import F +expr = F.today() +``` + +Functions that accept arguments can be composed, just like when using SQL: +```python +expr = F.toDayOfWeek(F.today()) +``` + +You can see the SQL expression that is represented by an ORM expression by calling its `to_sql` or `repr` methods: +```python +>>> print(expr.to_sql()) +toDayOfWeek(today()) +``` + +### Operators + +ORM expressions support Python's standard arithmetic operators, so you can compose expressions using `+`, `-`, `*`, `/` and `%`. For example: +```python +# A random integer between 1 and 10 +F.rand() % 10 + 1 +``` + +There is also support for comparison operators (`<`, `<=`, `==`, `>=`, `>`, `!=`) and logical operators (`&`, `|`, `~`, `^`) which are often used for filtering querysets: +```python +# Is it Friday the 13th? +(F.toDayOfWeek(F.today()) == 6) & (F.toDayOfMonth(F.today()) == 13) +``` + +### Referring to model fields + +To refer to a model field inside an expression, use `.` syntax, for example: +```python +# Convert the temperature from Celsius to Fahrenheit +Sensor.temperature * 1.8 + 32 +``` + +Inside model class definitions omit the class name: +```python +class Person(Model): + height_cm = fields.Float32Field() + height_inch = fields.Float32Field(alias=height_cm/2.54) + ... +``` + +### Creating new "functions" + +Since expressions are just Python objects until they get converted to SQL, it is possible to invent new "functions" by combining existing ones into useful building blocks. For example, we can create a reusable expression that takes a string and trims whitespace, converts it to uppercase, and changes blanks to underscores: +```python +def normalize_string(s): + return F.replaceAll(F.upper(F.trimBoth(s)), ' ', '_') +``` + +Then we can use this expression anywhere we need it: +```python +class Event(Model): + code = fields.StringField() + normalized_code = fields.StringField(materialized=normalize_string(code)) +``` + +### Which functions are available? + +ClickHouse has many hundreds of functions, and new ones often get added. If you encounter a function that the database supports but is not available in the `F` class, please report this via a GitHub issue. You can still use the function by providing its name: +```python +expr = F("someFunctionName", arg1, arg2, ...) +``` + +--- + +[<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Querysets >>](querysets.md) \ No newline at end of file diff --git a/docs/field_options.md b/docs/field_options.md new file mode 100644 index 0000000..db3e58f --- /dev/null +++ b/docs/field_options.md @@ -0,0 +1,112 @@ +Field Options +============= + +All field types accept the following arguments: + + - default + - alias + - materialized + - readonly + - codec + +Note that `default`, `alias` and `materialized` are mutually exclusive - you cannot use more than one of them in a single field. + +## default + +Specifies a default value to use for the field. If not given, the field will have a default value based on its type: empty string for string fields, zero for numeric fields, etc. +The default value can be a Python value suitable for the field type, or an expression. For example: +```python +class Event(models.Model): + + name = fields.StringField(default="EVENT") + repeated = fields.UInt32Field(default=1) + created = fields.DateTimeField(default=F.now()) + + engine = engines.Memory() + ... +``` +When creating a model instance, any fields you do not specify get their default value. Fields that use a default expression are assigned a sentinel value of `infi.clickhouse_orm.models.NO_VALUE` instead. For example: +```python +>>> event = Event() +>>> print(event.to_dict()) +{'name': 'EVENT', 'repeated': 1, 'created': } +``` +:warning: Due to a bug in ClickHouse versions prior to 20.1.2.4, insertion of records with expressions for default values may fail. + +## alias / materialized + +The `alias` and `materialized` attributes expect an expression that gets calculated by the database. The difference is that `alias` fields are calculated on the fly, while `materialized` fields are calculated when the record is inserted, and are stored on disk. +You can use any expression, and can refer to other model fields. For example: +```python +class Event(models.Model): + + created = fields.DateTimeField() + created_date = fields.DateTimeField(materialized=F.toDate(created)) + name = fields.StringField() + normalized_name = fields.StringField(alias=F.upper(F.trim(name))) + + engine = engines.Memory() +``` +For backwards compatibility with older versions of the ORM, you can pass the expression as an SQL string: +```python + created_date = fields.DateTimeField(materialized="toDate(created)") +``` +Both field types can't be inserted into the database directly, so they are ignored when using the `Database.insert()` method. ClickHouse does not return the field values if you use `"SELECT * FROM ..."` - you have to list these field names explicitly in the query. + +Usage: +```python +obj = Event(created=datetime.now(), name='MyEvent') +db = Database('my_test_db') +db.insert([obj]) +# All values will be retrieved from database +db.select('SELECT created, created_date, username, name FROM $db.event', model_class=Event) +# created_date and username will contain a default value +db.select('SELECT * FROM $db.event', model_class=Event) +``` +When creating a model instance, any alias or materialized fields are assigned a sentinel value of `infi.clickhouse_orm.models.NO_VALUE` since their real values can only be known after insertion to the database. + +## codec + +This attribute specifies the compression algorithm to use for the field (instead of the default data compression algorithm defined in server settings). + +Supported compression algorithms: + +| Codec | Argument | Comment +| -------------------- | -------------------------------------------| ---------------------------------------------------- +| NONE | None | No compression. +| LZ4 | None | LZ4 compression. +| LZ4HC(`level`) | Possible `level` range: [3, 12]. | Default value: 9. Greater values stands for better compression and higher CPU usage. Recommended value range: [4,9]. +| ZSTD(`level`) | Possible `level`range: [1, 22]. | Default value: 1. Greater values stands for better compression and higher CPU usage. Levels >= 20, should be used with caution, as they require more memory. +| Delta(`delta_bytes`) | Possible `delta_bytes` range: 1, 2, 4 , 8. | Default value for `delta_bytes` is `sizeof(type)` if it is equal to 1, 2,4 or 8 and equals to 1 otherwise. + +Codecs can be combined by separating their names with commas. The default database codec is not included into pipeline (if it should be applied to a field, you have to specify it explicitly in pipeline). + +Recommended usage for codecs: +- When values for particular metric do not differ significantly from point to point, delta-encoding allows to reduce disk space usage significantly. +- DateTime works great with pipeline of Delta, ZSTD and the column size can be compressed to 2-3% of its original size (given a smooth datetime data) +- Numeric types usually enjoy best compression rates with ZSTD +- String types enjoy good compression rates with LZ4HC + +Example: +```python +class Stats(models.Model): + + id = fields.UInt64Field(codec='ZSTD(10)') + timestamp = fields.DateTimeField(codec='Delta,ZSTD') + timestamp_date = fields.DateField(codec='Delta(4),ZSTD(22)') + metadata_id = fields.Int64Field(codec='LZ4') + status = fields.StringField(codec='LZ4HC(10)') + calculation = fields.NullableField(fields.Float32Field(), codec='ZSTD') + alerts = fields.ArrayField(fields.FixedStringField(length=15), codec='Delta(2),LZ4HC') + + engine = MergeTree('timestamp_date', ('id', 'timestamp')) +``` +Note: This feature is supported on ClickHouse version 19.1.16 and above. Codec arguments will be ignored by the ORM for older versions of ClickHouse. + +## readonly + +This attribute is set automatically for fields with `alias` or `materialized` attributes, you do not need to pass it yourself. + +--- + +[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md) \ No newline at end of file diff --git a/docs/field_types.md b/docs/field_types.md index 3c2d2bf..434564a 100644 --- a/docs/field_types.md +++ b/docs/field_types.md @@ -33,112 +33,6 @@ The following field types are supported: | ArrayField | Array | list | See below | NullableField | Nullable | See below | See below -Field Options ----------------- -All field types accept the following arguments: - - - default - - alias - - materialized - - readonly - - codec - -Note that `default`, `alias` and `materialized` are mutually exclusive - you cannot use more than one of them in a single field. - -### default - -Specifies a default value to use for the field. If not given, the field will have a default value based on its type: empty string for string fields, zero for numeric fields, etc. -The default value can be a Python value suitable for the field type, or an expression. For example: -```python -class Event(models.Model): - - name = fields.StringField(default="EVENT") - repeated = fields.UInt32Field(default=1) - created = fields.DateTimeField(default=F.now()) - - engine = engines.Memory() - ... -``` -When creating a model instance, any fields you do not specify get their default value. Fields that use a default expression are assigned a sentinel value of `infi.clickhouse_orm.models.NO_VALUE` instead. For example: -```python ->>> event = Event() ->>> print(event.to_dict()) -{'name': 'EVENT', 'repeated': 1, 'created': } -``` -:warning: Due to a bug in ClickHouse versions prior to 20.1.2.4, insertion of records with expressions for default values may fail. - -### alias / materialized - -The `alias` and `materialized` attributes expect an expression that gets calculated by the database. The difference is that `alias` fields are calculated on the fly, while `materialized` fields are calculated when the record is inserted, and are stored on disk. -You can use any expression, and can refer to other model fields. For example: -```python -class Event(models.Model): - - created = fields.DateTimeField() - created_date = fields.DateTimeField(materialized=F.toDate(created)) - name = fields.StringField() - normalized_name = fields.StringField(alias=F.upper(F.trim(name))) - - engine = engines.Memory() -``` -For backwards compatibility with older versions of the ORM, you can pass the expression as an SQL string: -```python - created_date = fields.DateTimeField(materialized="toDate(created)") -``` -Both field types can't be inserted into the database directly, so they are ignored when using the `Database.insert()` method. ClickHouse does not return the field values if you use `"SELECT * FROM ..."` - you have to list these field names explicitly in the query. - -Usage: -```python -obj = Event(created=datetime.now(), name='MyEvent') -db = Database('my_test_db') -db.insert([obj]) -# All values will be retrieved from database -db.select('SELECT created, created_date, username, name FROM $db.event', model_class=Event) -# created_date and username will contain a default value -db.select('SELECT * FROM $db.event', model_class=Event) -``` -When creating a model instance, any alias or materialized fields are assigned a sentinel value of `infi.clickhouse_orm.models.NO_VALUE` since their real values can only be known after insertion to the database. - -### readonly - -This attribute is set automatically for fields with `alias` or `materialized` attributes, you do not need to pass it yourself. - -### codec -This attribute specifies the compression algorithm to use for the field (instead of the default data compression algorithm defined in server settings). - -Supported compression algorithms: - -| Codec | Argument | Comment -| -------------------- | -------------------------------------------| ---------------------------------------------------- -| NONE | None | No compression. -| LZ4 | None | LZ4 compression. -| LZ4HC(`level`) | Possible `level` range: [3, 12]. | Default value: 9. Greater values stands for better compression and higher CPU usage. Recommended value range: [4,9]. -| ZSTD(`level`) | Possible `level`range: [1, 22]. | Default value: 1. Greater values stands for better compression and higher CPU usage. Levels >= 20, should be used with caution, as they require more memory. -| Delta(`delta_bytes`) | Possible `delta_bytes` range: 1, 2, 4 , 8. | Default value for `delta_bytes` is `sizeof(type)` if it is equal to 1, 2,4 or 8 and equals to 1 otherwise. - -Codecs can be combined by separating their names with commas. The default database codec is not included into pipeline (if it should be applied to a field, you have to specify it explicitly in pipeline). - -Recommended usage for codecs: -- When values for particular metric do not differ significantly from point to point, delta-encoding allows to reduce disk space usage significantly. -- DateTime works great with pipeline of Delta, ZSTD and the column size can be compressed to 2-3% of its original size (given a smooth datetime data) -- Numeric types usually enjoy best compression rates with ZSTD -- String types enjoy good compression rates with LZ4HC - -Example: -```python -class Stats(models.Model): - - id = fields.UInt64Field(codec='ZSTD(10)') - timestamp = fields.DateTimeField(codec='Delta,ZSTD') - timestamp_date = fields.DateField(codec='Delta(4),ZSTD(22)') - metadata_id = fields.Int64Field(codec='LZ4') - status = fields.StringField(codec='LZ4HC(10)') - calculation = fields.NullableField(fields.Float32Field(), codec='ZSTD') - alerts = fields.ArrayField(fields.FixedStringField(length=15), codec='Delta(2),LZ4HC') - - engine = MergeTree('timestamp_date', ('id', 'timestamp')) -``` -Note: This feature is supported on ClickHouse version 19.1.16 and above. Codec arguments will be ignored by the ORM for older versions of ClickHouse. DateTimeField and Time Zones ---------------------------- @@ -294,4 +188,4 @@ class BooleanField(Field): --- -[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md) \ No newline at end of file +[<< Field Options](field_options.md) | [Table of Contents](toc.md) | [Table Engines >>](table_engines.md) \ No newline at end of file diff --git a/docs/models_and_databases.md b/docs/models_and_databases.md index 59947db..c6ce1ca 100644 --- a/docs/models_and_databases.md +++ b/docs/models_and_databases.md @@ -31,6 +31,8 @@ Each field has a "natural" default value - empty string for string fields, zero first_name = fields.StringField(default="anonymous") +For additional details see [here](field_options.md). + ### Null values To allow null values in a field, wrap it inside a `NullableField`: @@ -39,25 +41,27 @@ To allow null values in a field, wrap it inside a `NullableField`: In this case, the default value for that field becomes `null` unless otherwise specified. +For more information about `NullableField` see [Field Types](field_types.md). + ### Materialized fields The value of a materialized field is calculated from other fields in the model. For example: - year_born = fields.Int16Field(materialized="toYear(birthday)") + year_born = fields.Int16Field(materialized=F.toYear(birthday)) Materialized fields are read-only, meaning that their values are not sent to the database when inserting records. -It is not possible to specify a default value for a materialized field. +For additional details see [here](field_options.md). ### Alias fields An alias field is a field whose value is calculated by ClickHouse on the fly, as a function of other fields. It is not physically stored by the database. For example: - weekday_born = field.UInt8Field(alias="toDayOfWeek(birthday)") + weekday_born = field.UInt8Field(alias=F.toDayOfWeek(birthday)) Alias fields are read-only, meaning that their values are not sent to the database when inserting records. -It is not possible to specify a default value for an alias field. +For additional details see [here](field_options.md). ### Table Names @@ -121,19 +125,19 @@ Reading from the Database Loading model instances from the database is simple: for person in db.select("SELECT * FROM my_test_db.person", model_class=Person): - print person.first_name, person.last_name + print(person.first_name, person.last_name) Do not include a `FORMAT` clause in the query, since the ORM automatically sets the format to `TabSeparatedWithNamesAndTypes`. It is possible to select only a subset of the columns, and the rest will receive their default values: for person in db.select("SELECT first_name FROM my_test_db.person WHERE last_name='Smith'", model_class=Person): - print person.first_name + print(person.first_name) The ORM provides a way to build simple queries without writing SQL by hand. The previous snippet can be written like this: for person in Person.objects_in(db).filter(last_name='Smith').only('first_name'): - print person.first_name + print(person.first_name) See [Querysets](querysets.md) for more information. @@ -144,7 +148,7 @@ Reading without a Model When running a query, specifying a model class is not required. In case you do not provide a model class, an ad-hoc class will be defined based on the column names and types returned by the query: for row in db.select("SELECT max(height) as max_height FROM my_test_db.person"): - print row.max_height + print(row.max_height) This is a very convenient feature that saves you the need to define a model for each query, while still letting you work with Pythonic column values and an elegant syntax. @@ -180,9 +184,9 @@ It is possible to paginate through model instances: >>> order_by = 'first_name, last_name' >>> page = db.paginate(Person, order_by, page_num=1, page_size=10) - >>> print page.number_of_objects + >>> print(page.number_of_objects) 2507 - >>> print page.pages_total + >>> print(page.pages_total) 251 >>> for person in page.objects: >>> # do something @@ -204,4 +208,4 @@ Note that `order_by` must be chosen so that the ordering is unique, otherwise th --- -[<< Overview](index.md) | [Table of Contents](toc.md) | [Querysets >>](querysets.md) \ No newline at end of file +[<< Overview](index.md) | [Table of Contents](toc.md) | [Expressions >>](expressions.md) \ No newline at end of file diff --git a/docs/querysets.md b/docs/querysets.md index 056e794..d5dfb25 100644 --- a/docs/querysets.md +++ b/docs/querysets.md @@ -8,7 +8,7 @@ A queryset is an object that represents a database query using a specific Model. This queryset matches all Person instances in the database. You can get these instances using iteration: for person in qs: - print person.first_name, person.last_name + print(person.first_name, person.last_name) Filtering --------- @@ -128,7 +128,7 @@ Adds a DISTINCT clause to the query, meaning that any duplicate rows in the resu Final -------- -This method can be used only with CollapsingMergeTree engine. +This method can be used only with `CollapsingMergeTree` engine. Adds a FINAL modifier to the query, meaning data is selected fully "collapsed" by sign field. >>> Person.objects_in(database).count() @@ -162,9 +162,9 @@ Similar to `Database.paginate`, you can go over the queryset results one page at >>> qs = Person.objects_in(database).order_by('last_name', 'first_name') >>> page = qs.paginate(page_num=1, page_size=10) - >>> print page.number_of_objects + >>> print(page.number_of_objects) 2507 - >>> print page.pages_total + >>> print(page.pages_total) 251 >>> for person in page.objects: >>> # do something @@ -185,9 +185,9 @@ Aggregation It is possible to use aggregation functions over querysets using the `aggregate` method. The simplest form of aggregation works over all rows in the queryset: >>> qs = Person.objects_in(database).aggregate(average_height='avg(height)') - >>> print qs.count() + >>> print(qs.count()) 1 - >>> for row in qs: print row.average_height + >>> for row in qs: print(row.average_height) 1.71 The returned row or rows are no longer instances of the base model (`Person` in this example), but rather instances of an ad-hoc model that includes only the fields specified in the call to `aggregate`. @@ -215,7 +215,7 @@ To achieve this, you can use `with_totals` method. It will return extra row (las values aggregated for all rows suitable for filters. qs = Person.objects_in(database).aggregate('first_name', num='count()').with_totals().order_by('-count')[:3] - >>> print qs.count() + >>> print(qs.count()) 4 >>> for row in qs: >>> print("'{}': {}".format(row.first_name, row.count)) @@ -225,4 +225,4 @@ values aggregated for all rows suitable for filters. --- -[<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md) \ No newline at end of file +[<< Expressions](expressions.md) | [Table of Contents](toc.md) | [Field Options >>](field_options.md) \ No newline at end of file diff --git a/docs/table_engines.md b/docs/table_engines.md index d4ba905..eb213ff 100644 --- a/docs/table_engines.md +++ b/docs/table_engines.md @@ -1,7 +1,7 @@ Table Engines ============= -See: [ClickHouse Documentation](https://clickhouse.yandex/docs/en/table_engines/) +See: [ClickHouse Documentation](https://clickhouse.tech/docs/en/operations/table_engines/) Each model must have an engine instance, used when creating the table in ClickHouse. diff --git a/docs/toc.md b/docs/toc.md index 0c81cb3..6a9f29a 100644 --- a/docs/toc.md +++ b/docs/toc.md @@ -30,13 +30,17 @@ * [Pagination](querysets.md#pagination) * [Aggregation](querysets.md#aggregation) + * [Field Options](field_options.md#field-options) + * [default](field_options.md#default) + * [alias / materialized](field_options.md#alias-/-materialized) + * [codec](field_options.md#codec) + * [readonly](field_options.md#readonly) + * [Field Types](field_types.md#field-types) * [DateTimeField and Time Zones](field_types.md#datetimefield-and-time-zones) * [Working with enum fields](field_types.md#working-with-enum-fields) * [Working with array fields](field_types.md#working-with-array-fields) - * [Working with materialized and alias fields](field_types.md#working-with-materialized-and-alias-fields) * [Working with nullable fields](field_types.md#working-with-nullable-fields) - * [Working with field compression codecs](field_types.md#working-with-field-compression-codecs) * [Working with LowCardinality fields](field_types.md#working-with-lowcardinality-fields) * [Creating custom field types](field_types.md#creating-custom-field-types) @@ -84,6 +88,8 @@ * [FixedStringField](class_reference.md#fixedstringfield) * [Float32Field](class_reference.md#float32field) * [Float64Field](class_reference.md#float64field) + * [IPv4Field](class_reference.md#ipv4field) + * [IPv6Field](class_reference.md#ipv6field) * [Int16Field](class_reference.md#int16field) * [Int32Field](class_reference.md#int32field) * [Int64Field](class_reference.md#int64field) @@ -111,4 +117,8 @@ * [infi.clickhouse_orm.query](class_reference.md#infi.clickhouse_orm.query) * [QuerySet](class_reference.md#queryset) * [AggregateQuerySet](class_reference.md#aggregatequeryset) + * [infi.clickhouse_orm.funcs](class_reference.md#infi.clickhouse_orm.funcs) + * [F](class_reference.md#f) + * [infi.clickhouse_orm.system_models](class_reference.md#infi.clickhouse_orm.system_models) + * [SystemPart](class_reference.md#systempart) diff --git a/scripts/generate_ref.py b/scripts/generate_ref.py index 0cbdb0d..bb9df4e 100644 --- a/scripts/generate_ref.py +++ b/scripts/generate_ref.py @@ -125,6 +125,8 @@ if __name__ == '__main__': from infi.clickhouse_orm import engines from infi.clickhouse_orm import models from infi.clickhouse_orm import query + from infi.clickhouse_orm import funcs + from infi.clickhouse_orm import system_models print('Class Reference') print('===============') @@ -134,3 +136,5 @@ if __name__ == '__main__': module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False) module_doc([engines.Engine] + all_subclasses(engines.Engine), False) module_doc([query.QuerySet, query.AggregateQuerySet]) + module_doc([funcs.F]) + module_doc([system_models.SystemPart]) diff --git a/scripts/generate_toc.sh b/scripts/generate_toc.sh index 7ed82ce..a77aaaa 100755 --- a/scripts/generate_toc.sh +++ b/scripts/generate_toc.sh @@ -9,6 +9,7 @@ printf "# Table of Contents\n\n" > toc.md generate_one "index.md" generate_one "models_and_databases.md" generate_one "querysets.md" +generate_one "field_options.md" generate_one "field_types.md" generate_one "table_engines.md" generate_one "schema_migrations.md" diff --git a/scripts/html_to_markdown_toc.py b/scripts/html_to_markdown_toc.py index 169e698..9ddd41b 100644 --- a/scripts/html_to_markdown_toc.py +++ b/scripts/html_to_markdown_toc.py @@ -1,4 +1,4 @@ -from HTMLParser import HTMLParser +from html.parser import HTMLParser import sys @@ -18,7 +18,7 @@ class HeadersToMarkdownParser(HTMLParser): if tag.lower() in HEADER_TAGS: indent = ' ' * int(self.inside[1]) fragment = self.text.lower().replace(' ', '-') - print '%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment) + print('%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment)) self.inside = None self.text = '' @@ -28,4 +28,4 @@ class HeadersToMarkdownParser(HTMLParser): HeadersToMarkdownParser().feed(sys.stdin.read()) -print +print('') diff --git a/src/infi/clickhouse_orm/engines.py b/src/infi/clickhouse_orm/engines.py index d01e389..905daf6 100644 --- a/src/infi/clickhouse_orm/engines.py +++ b/src/infi/clickhouse_orm/engines.py @@ -216,11 +216,11 @@ class Distributed(Engine): """ def __init__(self, cluster, table=None, sharding_key=None): """ - :param cluster: what cluster to access data from - :param table: underlying table that actually stores data. + - `cluster`: what cluster to access data from + - `table`: underlying table that actually stores data. If you are not specifying any table here, ensure that it can be inferred from your model's superclass (see models.DistributedModel.fix_engine_table) - :param sharding_key: how to distribute data among shards when inserting + - `sharding_key`: how to distribute data among shards when inserting straightly into Distributed table, optional """ self.cluster = cluster diff --git a/src/infi/clickhouse_orm/fields.py b/src/infi/clickhouse_orm/fields.py index 96127a7..4f12691 100644 --- a/src/infi/clickhouse_orm/fields.py +++ b/src/infi/clickhouse_orm/fields.py @@ -74,9 +74,10 @@ class Field(FunctionOperatorsMixin): def get_sql(self, with_default_expression=True, db=None): ''' Returns an SQL expression describing the field (e.g. for CREATE TABLE). - :param with_default_expression: If True, adds default value to sql. + + - `with_default_expression`: If True, adds default value to sql. It doesn't affect fields with alias and materialized values. - :param db: Database, used for checking supported features. + - `db`: Database, used for checking supported features. ''' sql = self.db_type if with_default_expression: @@ -102,8 +103,10 @@ class Field(FunctionOperatorsMixin): """ Checks if the instance if one of the types provided or if any of the inner_field child is one of the types provided, returns True if field or any inner_field is one of ths provided, False otherwise - :param types: Iterable of types to check inclusion of instance - :return: Boolean + + - `types`: Iterable of types to check inclusion of instance + + Returns: Boolean """ if isinstance(self, types): return True diff --git a/src/infi/clickhouse_orm/funcs.py b/src/infi/clickhouse_orm/funcs.py index 31febe3..bd8e26f 100644 --- a/src/infi/clickhouse_orm/funcs.py +++ b/src/infi/clickhouse_orm/funcs.py @@ -110,10 +110,17 @@ class F(Cond, FunctionOperatorsMixin): It doubles as a query condition when the function returns a boolean result. """ def __init__(self, name, *args): + """ + Initializer. + + """ self.name = name self.args = args self.is_binary_operator = False + def __repr__(self): + return self.to_sql() + def to_sql(self, *args): # FIXME why *args ? """ Generates an SQL string for this function and its arguments. @@ -128,11 +135,11 @@ class F(Cond, FunctionOperatorsMixin): else: prefix = self.name sep = ', ' - arg_strs = (F.arg_to_sql(arg) for arg in self.args) + arg_strs = (F._arg_to_sql(arg) for arg in self.args) return prefix + '(' + sep.join(arg_strs) + ')' @staticmethod - def arg_to_sql(arg): + def _arg_to_sql(arg): """ Converts a function argument to SQL string according to its type. Supports functions, model fields, strings, dates, datetimes, booleans, @@ -156,7 +163,7 @@ class F(Cond, FunctionOperatorsMixin): if arg is None: return 'NULL' if is_iterable(arg): - return '[' + comma_join(F.arg_to_sql(x) for x in arg) + ']' + return '[' + comma_join(F._arg_to_sql(x) for x in arg) + ']' return str(arg) # Arithmetic functions diff --git a/src/infi/clickhouse_orm/query.py b/src/infi/clickhouse_orm/query.py index c28462d..06a3266 100644 --- a/src/infi/clickhouse_orm/query.py +++ b/src/infi/clickhouse_orm/query.py @@ -205,7 +205,7 @@ class Q(object): def is_empty(self): """ Checks if there are any conditions in Q object - :return: Boolean + Returns: Boolean """ return not bool(self._conds or self._children) diff --git a/src/infi/clickhouse_orm/system_models.py b/src/infi/clickhouse_orm/system_models.py index cb0cca6..dfbe46b 100644 --- a/src/infi/clickhouse_orm/system_models.py +++ b/src/infi/clickhouse_orm/system_models.py @@ -60,10 +60,12 @@ class SystemPart(Model): def _partition_operation_sql(self, operation, settings=None, from_part=None): """ Performs some operation over partition - :param db: Database object to execute operation on - :param operation: Operation to execute from SystemPart.OPERATIONS set - :param settings: Settings for executing request to ClickHouse over db.raw() method - :return: Operation execution result + + - `db`: Database object to execute operation on + - `operation`: Operation to execute from SystemPart.OPERATIONS set + - `settings`: Settings for executing request to ClickHouse over db.raw() method + + Returns: Operation execution result """ operation = operation.upper() assert operation in self.OPERATIONS, "operation must be in [%s]" % comma_join(self.OPERATIONS) @@ -76,41 +78,51 @@ class SystemPart(Model): def detach(self, settings=None): """ Move a partition to the 'detached' directory and forget it. - :param settings: Settings for executing request to ClickHouse over db.raw() method - :return: SQL Query + + - `settings`: Settings for executing request to ClickHouse over db.raw() method + + Returns: SQL Query """ return self._partition_operation_sql('DETACH', settings=settings) def drop(self, settings=None): """ Delete a partition - :param settings: Settings for executing request to ClickHouse over db.raw() method - :return: SQL Query + + - `settings`: Settings for executing request to ClickHouse over db.raw() method + + Returns: SQL Query """ return self._partition_operation_sql('DROP', settings=settings) def attach(self, settings=None): """ Add a new part or partition from the 'detached' directory to the table. - :param settings: Settings for executing request to ClickHouse over db.raw() method - :return: SQL Query + + - `settings`: Settings for executing request to ClickHouse over db.raw() method + + Returns: SQL Query """ return self._partition_operation_sql('ATTACH', settings=settings) def freeze(self, settings=None): """ Create a backup of a partition. - :param settings: Settings for executing request to ClickHouse over db.raw() method - :return: SQL Query + + - `settings`: Settings for executing request to ClickHouse over db.raw() method + + Returns: SQL Query """ return self._partition_operation_sql('FREEZE', settings=settings) def fetch(self, zookeeper_path, settings=None): """ Download a partition from another server. - :param zookeeper_path: Path in zookeeper to fetch from - :param settings: Settings for executing request to ClickHouse over db.raw() method - :return: SQL Query + + - `zookeeper_path`: Path in zookeeper to fetch from + - `settings`: Settings for executing request to ClickHouse over db.raw() method + + Returns: SQL Query """ return self._partition_operation_sql('FETCH', settings=settings, from_part=zookeeper_path) @@ -118,9 +130,11 @@ class SystemPart(Model): def get(cls, database, conditions=""): """ Get all data from system.parts table - :param database: A database object to fetch data from. - :param conditions: WHERE clause conditions. Database condition is added automatically - :return: A list of SystemPart objects + + - `database`: A database object to fetch data from. + - `conditions`: WHERE clause conditions. Database condition is added automatically + + Returns: A list of SystemPart objects """ assert isinstance(database, Database), "database must be database.Database class instance" assert isinstance(conditions, str), "conditions must be a string" @@ -134,9 +148,11 @@ class SystemPart(Model): def get_active(cls, database, conditions=""): """ Gets active data from system.parts table - :param database: A database object to fetch data from. - :param conditions: WHERE clause conditions. Database and active conditions are added automatically - :return: A list of SystemPart objects + + - `database`: A database object to fetch data from. + - `conditions`: WHERE clause conditions. Database and active conditions are added automatically + + Returns: A list of SystemPart objects """ if conditions: conditions += ' AND '