From 1a366ce62500344b52c0c89ab17fde2917f9eb69 Mon Sep 17 00:00:00 2001 From: sw <935405794@qq.com> Date: Sat, 4 Jun 2022 12:47:35 +0800 Subject: [PATCH] tuple names maybe not exists --- .github/workflows/ci.yml | 64 --- .github/workflows/coverage.yml | 44 ++ .github/workflows/pr-python310.yml | 41 ++ .github/workflows/pr-python37.yml | 41 ++ .github/workflows/pr-python38.yml | 41 ++ .github/workflows/pr-python39.yml | 41 ++ README.md | 1 + docs/async_databases.md | 60 +++ docs/class_reference.md | 679 ++++++++++++++++++++++++++--- docs/contributing.md | 21 +- docs/expressions.md | 2 +- docs/field_options.md | 19 +- docs/field_types.md | 81 ++-- docs/importing_orm_classes.md | 30 +- docs/index.md | 8 +- docs/models_and_databases.md | 29 +- docs/querysets.md | 51 ++- docs/toc.md | 18 +- docs/whats_new_in_version_2.md | 4 +- scripts/generate_ref.py | 25 +- scripts/generate_toc.sh | 3 +- src/clickhouse_orm/aio/database.py | 11 +- src/clickhouse_orm/database.py | 2 +- src/clickhouse_orm/models.py | 14 +- tests/base_test_with_data.py | 26 ++ tests/test_aiodatabase.py | 331 ++++++++++++++ tests/test_alias_fields.py | 6 +- tests/test_array_fields.py | 4 +- tests/test_temporary_models.py | 46 ++ tests/test_tuple_fields.py | 63 +++ 30 files changed, 1548 insertions(+), 258 deletions(-) delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/pr-python310.yml create mode 100644 .github/workflows/pr-python37.yml create mode 100644 .github/workflows/pr-python38.yml create mode 100644 .github/workflows/pr-python39.yml create mode 100644 docs/async_databases.md create mode 100644 tests/test_aiodatabase.py create mode 100644 tests/test_temporary_models.py create mode 100644 tests/test_tuple_fields.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index a9db1d9..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: ci - -on: - push: - # Publish `master` as Docker `latest` image. - branches: - - master - - develop - - # Publish `v1.2.3` tags as releases. - tags: - - v* - - # Run tests for any PRs. - pull_request: - -env: - IMAGE_NAME: ch_orm - -jobs: - # Run tests. - # See also https://docs.docker.com/docker-hub/builds/automated-testing/ - test: - runs-on: ubuntu-latest - services: - clickhouse: - image: clickhouse/clickhouse-server - ports: - - 8123:8123 - - 9000:9000 - options: --ulimit nofile=262144:262144 - strategy: - matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10" ] - steps: - - uses: actions/checkout@v2 - - name: Build and Install - run: | - pip install build - python -m build - pip install dist/* - pip install coveralls - - name: UnitTest - run: | - coverage run --source=clickhouse_orm -m unittest - - name: Upload Coverage - run: coveralls --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: ${{ matrix.python-version }} - COVERALLS_PARALLEL: true - - coveralls: - name: Finish Coveralls - needs: test - runs-on: ubuntu-latest - container: python:3-slim - steps: - - name: Finished - run: | - pip3 install --upgrade coveralls - coveralls --finish - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..e5e1b93 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,44 @@ +name: Coverage check +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.9] + os: [ubuntu-latest] + fail-fast: false + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies 🔨 + run: | + python -m pip install --upgrade pip + pip install build + python -m build + pip install dist/* + pip install coveralls + - name: Run coverage + run: | + coverage run --source=clickhouse_orm -m unittest + - name: Upload Coverage + run: coveralls --service=github + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: ${{ matrix.python-version }} + COVERALLS_PARALLEL: true + - name: Finished + run: | + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pr-python310.yml b/.github/workflows/pr-python310.yml new file mode 100644 index 0000000..d3e2677 --- /dev/null +++ b/.github/workflows/pr-python310.yml @@ -0,0 +1,41 @@ +name: Python 3.10 Tests +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + - develop + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + branches: + - master + - develop + +jobs: + testPy37: + runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server + ports: + - 8123:8123 + - 9000:9000 + options: --ulimit nofile=262144:262144 + strategy: + matrix: + python-version: [ "3.10" ] + steps: + - uses: actions/checkout@v2 + - name: Build and Install + run: | + pip install build + python -m build + pip install dist/* + - name: Run Unit Tests + run: | + python -m unittest diff --git a/.github/workflows/pr-python37.yml b/.github/workflows/pr-python37.yml new file mode 100644 index 0000000..ae59e28 --- /dev/null +++ b/.github/workflows/pr-python37.yml @@ -0,0 +1,41 @@ +name: Python 3.7 Tests +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + - develop + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + branches: + - master + - develop + +jobs: + testPy37: + runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server + ports: + - 8123:8123 + - 9000:9000 + options: --ulimit nofile=262144:262144 + strategy: + matrix: + python-version: [ "3.7" ] + steps: + - uses: actions/checkout@v2 + - name: Build and Install + run: | + pip install build + python -m build + pip install dist/* + - name: Run Unit Tests + run: | + python -m unittest diff --git a/.github/workflows/pr-python38.yml b/.github/workflows/pr-python38.yml new file mode 100644 index 0000000..239542b --- /dev/null +++ b/.github/workflows/pr-python38.yml @@ -0,0 +1,41 @@ +name: Python 3.8 Tests +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + - develop + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + branches: + - master + - develop + +jobs: + testPy37: + runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server + ports: + - 8123:8123 + - 9000:9000 + options: --ulimit nofile=262144:262144 + strategy: + matrix: + python-version: [ "3.8" ] + steps: + - uses: actions/checkout@v2 + - name: Build and Install + run: | + pip install build + python -m build + pip install dist/* + - name: Run Unit Tests + run: | + python -m unittest diff --git a/.github/workflows/pr-python39.yml b/.github/workflows/pr-python39.yml new file mode 100644 index 0000000..c3d59df --- /dev/null +++ b/.github/workflows/pr-python39.yml @@ -0,0 +1,41 @@ +name: Python 3.9 Tests +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + - develop + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + branches: + - master + - develop + +jobs: + testPy37: + runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server + ports: + - 8123:8123 + - 9000:9000 + options: --ulimit nofile=262144:262144 + strategy: + matrix: + python-version: [ "3.9" ] + steps: + - uses: actions/checkout@v2 + - name: Build and Install + run: | + pip install build + python -m build + pip install dist/* + - name: Run Unit Tests + run: | + python -m unittest diff --git a/README.md b/README.md index 8a23ff5..76cbc35 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ A fork of [infi.clikchouse_orm](https://github.com/Infinidat/infi.clickhouse_orm This repository expects to use more type hints, and will drop support for Python 2.x. +Supports both synchronous and asynchronous ways to interact with the clickhouse server. Means you can use asyncio to perform asynchronous queries, although the asynchronous mode is not well tested. | Build | [![Coverage Status](https://github.com/sswest/ch-orm/workflows/ci/badge.svg)](https://github.com/sswest/ch-orm/actions?query=workflow:ci)[![Coverage Status](https://coveralls.io/repos/github/sswest/ch-orm/badge.svg?branch=develop)](https://coveralls.io/github/sswest/ch-orm?branch=develop) | | ------- | ------------------------------------------------------------ | diff --git a/docs/async_databases.md b/docs/async_databases.md new file mode 100644 index 0000000..66a306b --- /dev/null +++ b/docs/async_databases.md @@ -0,0 +1,60 @@ +Async Databases +==================== + +Databases in async mode have basically the same API. In most cases you just need to add `await`. + +Insert from the AioDatabase +------------------------- + +To write your instances to ClickHouse, you need a `AioDatabase` instance: + +```python +from clickhouse_orm.aio.database import AioDatabase + +db = AioDatabase('my_test_db') + +async def main(): + await db.init() + ... +``` + +**Unlike the previous Database instance, you have to use an asynchronous method to initialize the db.** + + +Using the `AioDatabase` instance you can create a table for your model, and insert instances to it: +```python +from clickhouse_orm.aio.database import AioDatabase + +db = AioDatabase('my_test_db') + +async def main(): + await db.init() + await db.create_table(Person) + await db.insert([dan, suzy]) +``` + +The `insert` method can take any iterable of model instances, but they all must belong to the same model class. + +Reading from the AioDatabase +------------------------- + +Loading model instances from the database is easy, use the `async for` keyword: +```python +async for person in db.select("SELECT * FROM my_test_db.person", model_class=Person): + print(person.first_name, person.last_name) +``` +**Note: AioDatabase does not support QuerySet value by index** + +```python +async def main(): + await db.init() + + # incorrect example + person = await Person.objects_in(db).filter[5] + + # correct + person = [_ async for _ in Person.objects_in(db).filter[5:5]][0] +``` + + +[<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Expressions >>](expressions.md) \ No newline at end of file diff --git a/docs/class_reference.md b/docs/class_reference.md index 08716a5..174230b 100644 --- a/docs/class_reference.md +++ b/docs/class_reference.md @@ -1,8 +1,8 @@ Class Reference =============== -infi.clickhouse_orm.database ----------------------------- +clickhouse_orm.database +----------------------- ### Database @@ -10,7 +10,7 @@ infi.clickhouse_orm.database Database instances connect to a specific ClickHouse database for running queries, inserting data and other operations. -#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False, autocreate=True, timeout=60, verify_ssl_cert=True, log_statements=False) +#### Database(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False, auto_create=True, timeout=60, verify_ssl_cert=True, log_statements=False) Initializes a database instance. Unless it's readonly, the database will be @@ -21,7 +21,8 @@ created on the ClickHouse server if it does not already exist. - `username`: optional connection credentials. - `password`: optional connection credentials. - `readonly`: use a read-only connection. -- `autocreate`: automatically create the database if it does not exist (unless in readonly mode). +- `auto_create`: automatically create the database + if it does not exist (unless in readonly mode). - `timeout`: the connection timeout in seconds. - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS. - `log_statements`: when True, all database statements are logged. @@ -88,13 +89,17 @@ for example system tables. - `system_table`: whether the table is a system table, or belongs to the current database +#### init() + + #### insert(model_instances, batch_size=1000) Insert records into the database. - `model_instances`: any iterable containing instances of a single model class. -- `batch_size`: number of records to send per chunk (use a lower number if your records are very large). +- `batch_size`: number of records to send per chunk + (use a lower number if your records are very large). #### migrate(migrations_package_name, up_to=9999) @@ -152,8 +157,164 @@ Extends Exception Raised when a database operation fails. -infi.clickhouse_orm.models --------------------------- +clickhouse_orm.aio.database +--------------------------- + +### AioDatabase + +Extends Database + +#### AioDatabase(db_name, db_url="http://localhost:8123/", username=None, password=None, readonly=False, auto_create=True, timeout=60, verify_ssl_cert=True, log_statements=False) + + +Initializes a database instance. Unless it's readonly, the database will be +created on the ClickHouse server if it does not already exist. + +- `db_name`: name of the database to connect to. +- `db_url`: URL of the ClickHouse server. +- `username`: optional connection credentials. +- `password`: optional connection credentials. +- `readonly`: use a read-only connection. +- `auto_create`: automatically create the database + if it does not exist (unless in readonly mode). +- `timeout`: the connection timeout in seconds. +- `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS. +- `log_statements`: when True, all database statements are logged. + + +#### add_setting(name, value) + + +Adds a database setting that will be sent with every request. +For example, `db.add_setting("max_execution_time", 10)` will +limit query execution time to 10 seconds. +The name must be string, and the value is converted to string in case +it isn't. To remove a setting, pass `None` as the value. + + +#### close() + + +#### count(model_class, conditions=None) + + +Counts the number of records in the model's table. + +- `model_class`: the model to count. +- `conditions`: optional SQL conditions (contents of the WHERE clause). + + +#### create_database() + + +Creates the database on the ClickHouse server if it does not already exist. + + +#### create_table(model_class) + + +Creates a table for the given model class, if it does not exist already. + + +#### create_temporary_table(model_class, table_name=None) + + +Creates a temporary table for the given model class, if it does not exist already. +And you can specify the temporary table name explicitly. + + +#### does_table_exist(model_class) + + +Checks whether a table for the given model class already exists. +Note that this only checks for existence of a table with the expected name. + + +#### drop_database() + + +Deletes the database on the ClickHouse server. + + +#### drop_table(model_class) + + +Drops the database table of the given model class, if it exists. + + +#### get_model_for_table(table_name, system_table=False) + + +Generates a model class from an existing table in the database. +This can be used for querying tables which don't have a corresponding model class, +for example system tables. + +- `table_name`: the table to create a model for +- `system_table`: whether the table is a system table, or belongs to the current database + + +#### init() + + +#### insert(model_instances, batch_size=1000) + + +Insert records into the database. + +- `model_instances`: any iterable containing instances of a single model class. +- `batch_size`: number of records to send per chunk (use a lower number if your records are very large). + + +#### migrate(migrations_package_name, up_to=9999) + + +Executes schema migrations. + +- `migrations_package_name` - fully qualified name of the Python package + containing the migrations. +- `up_to` - number of the last migration to apply. + + +#### paginate(model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None) + + +Selects records and returns a single page of model instances. + +- `model_class`: the model class matching the query's table, + or `None` for getting back instances of an ad-hoc model. +- `order_by`: columns to use for sorting the query (contents of the ORDER BY clause). +- `page_num`: the page number (1-based), or -1 to get the last page. +- `page_size`: number of records to return per page. +- `conditions`: optional SQL conditions (contents of the WHERE clause). +- `settings`: query settings to send as HTTP GET parameters + +The result is a namedtuple containing `objects` (list), `number_of_objects`, +`pages_total`, `number` (of the current page), and `page_size`. + + +#### raw(query, settings=None, stream=False) + + +Performs a query and returns its output as text. + +- `query`: the SQL query to execute. +- `settings`: query settings to send as HTTP GET parameters +- `stream`: if true, the HTTP response from ClickHouse will be streamed. + + +#### select(query, model_class=None, settings=None) + + +Performs a query and returns a generator of model instances. + +- `query`: the SQL query to execute. +- `model_class`: the model class matching the query's table, + or `None` for getting back instances of an ad-hoc model. +- `settings`: query settings to send as HTTP GET parameters + + +clickhouse_orm.models +--------------------- ### Model @@ -239,6 +400,12 @@ Returns true if the model is marked as read only. Returns true if the model represents a system table. +#### Model.is_temporary_model() + + +Returns true if the model represents a temporary table. + + #### Model.objects_in(database) @@ -369,6 +536,12 @@ Returns true if the model is marked as read only. Returns true if the model represents a system table. +#### BufferModel.is_temporary_model() + + +Returns true if the model represents a temporary table. + + #### BufferModel.objects_in(database) @@ -504,6 +677,12 @@ Returns true if the model is marked as read only. Returns true if the model represents a system table. +#### MergeModel.is_temporary_model() + + +Returns true if the model represents a temporary table. + + #### MergeModel.objects_in(database) @@ -670,6 +849,12 @@ Returns true if the model is marked as read only. Returns true if the model represents a system table. +#### DistributedModel.is_temporary_model() + + +Returns true if the model represents a temporary table. + + #### DistributedModel.objects_in(database) @@ -811,14 +996,14 @@ separated by non-alphanumeric characters. - `random_seed` — The seed for Bloom filter hash functions. -infi.clickhouse_orm.fields --------------------------- +clickhouse_orm.fields +--------------------- ### ArrayField Extends Field -#### ArrayField(inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None) +#### ArrayField(inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### BaseEnumField @@ -828,7 +1013,7 @@ Extends Field Abstract base class for all enum-type fields. -#### BaseEnumField(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None) +#### BaseEnumField(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### BaseFloatField @@ -838,7 +1023,7 @@ Extends Field Abstract base class for all float-type fields. -#### BaseFloatField(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### BaseFloatField(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### BaseIntField @@ -848,49 +1033,49 @@ Extends Field Abstract base class for all integer-type fields. -#### BaseIntField(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### BaseIntField(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### DateField Extends Field -#### DateField(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### DateField(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### DateTime64Field Extends DateTimeField -#### DateTime64Field(default=None, alias=None, materialized=None, readonly=None, codec=None, timezone=None, precision=6) +#### DateTime64Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None, timezone=None, precision=6) ### DateTimeField Extends Field -#### DateTimeField(default=None, alias=None, materialized=None, readonly=None, codec=None, timezone=None) +#### DateTimeField(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None, timezone=None) ### Decimal128Field Extends DecimalField -#### Decimal128Field(scale, default=None, alias=None, materialized=None, readonly=None) +#### Decimal128Field(scale, default=None, alias=None, materialized=None, readonly=None, db_column=None) ### Decimal32Field Extends DecimalField -#### Decimal32Field(scale, default=None, alias=None, materialized=None, readonly=None) +#### Decimal32Field(scale, default=None, alias=None, materialized=None, readonly=None, db_column=None) ### Decimal64Field Extends DecimalField -#### Decimal64Field(scale, default=None, alias=None, materialized=None, readonly=None) +#### Decimal64Field(scale, default=None, alias=None, materialized=None, readonly=None, db_column=None) ### DecimalField @@ -900,21 +1085,21 @@ Extends Field Base class for all decimal fields. Can also be used directly. -#### DecimalField(precision, scale, default=None, alias=None, materialized=None, readonly=None) +#### DecimalField(precision, scale, default=None, alias=None, materialized=None, readonly=None, db_column=None) ### Enum16Field Extends BaseEnumField -#### Enum16Field(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Enum16Field(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Enum8Field Extends BaseEnumField -#### Enum8Field(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Enum8Field(enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Field @@ -924,130 +1109,137 @@ Extends FunctionOperatorsMixin Abstract base class for all field types. -#### Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### FixedStringField Extends StringField -#### FixedStringField(length, default=None, alias=None, materialized=None, readonly=None) +#### FixedStringField(length, default=None, alias=None, materialized=None, readonly=None, db_column=None) ### Float32Field Extends BaseFloatField -#### Float32Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Float32Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Float64Field Extends BaseFloatField -#### Float64Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Float64Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### IPv4Field Extends Field -#### IPv4Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### IPv4Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### IPv6Field Extends Field -#### IPv6Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### IPv6Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Int16Field Extends BaseIntField -#### Int16Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Int16Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Int32Field Extends BaseIntField -#### Int32Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Int32Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Int64Field Extends BaseIntField -#### Int64Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Int64Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### Int8Field Extends BaseIntField -#### Int8Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### Int8Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### LowCardinalityField Extends Field -#### LowCardinalityField(inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None) +#### LowCardinalityField(inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### NullableField Extends Field -#### NullableField(inner_field, default=None, alias=None, materialized=None, extra_null_values=None, codec=None) +#### NullableField(inner_field, default=None, alias=None, materialized=None, extra_null_values=None, codec=None, db_column=None) ### StringField Extends Field -#### StringField(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### StringField(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) + + +### TupleField + +Extends Field + +#### TupleField(name_fields, default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### UInt16Field Extends BaseIntField -#### UInt16Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### UInt16Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### UInt32Field Extends BaseIntField -#### UInt32Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### UInt32Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### UInt64Field Extends BaseIntField -#### UInt64Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### UInt64Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### UInt8Field Extends BaseIntField -#### UInt8Field(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### UInt8Field(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) ### UUIDField Extends Field -#### UUIDField(default=None, alias=None, materialized=None, readonly=None, codec=None) +#### UUIDField(default=None, alias=None, materialized=None, readonly=None, codec=None, db_column=None) -infi.clickhouse_orm.engines ---------------------------- +clickhouse_orm.engines +---------------------- ### Engine @@ -1140,11 +1332,13 @@ Extends MergeTree #### ReplacingMergeTree(date_col=None, order_by=(), ver_col=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None, partition_key=None, primary_key=None) -infi.clickhouse_orm.query -------------------------- +clickhouse_orm.query +-------------------- ### QuerySet +Extends Generic + A queryset is an object that represents a database query using a specific `Model`. It is lazy, meaning that it does not hit the database until you iterate over its @@ -1290,7 +1484,7 @@ Extends QuerySet A queryset used for aggregation. -#### AggregateQuerySet(base_qs, grouping_fields, calculated_fields) +#### AggregateQuerySet(base_queryset, grouping_fields, calculated_fields) Initializer. Normally you should not call this but rather use `QuerySet.aggregate()`. @@ -1299,7 +1493,9 @@ The grouping fields should be a list/tuple of field names from the model. For ex ``` ('event_type', 'event_subtype') ``` -The calculated fields should be a mapping from name to a ClickHouse aggregation function. For example: +The calculated fields should be a mapping from name to a ClickHouse aggregation function. + +For example: ``` {'weekday': 'toDayOfWeek(event_date)', 'number_of_events': 'count()'} ``` @@ -1443,8 +1639,8 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier #### to_sql(model_cls) -infi.clickhouse_orm.funcs -------------------------- +clickhouse_orm.funcs +-------------------- ### F @@ -2012,7 +2208,7 @@ Initializer. #### floor(n=None) -#### formatDateTime(format, timezone="") +#### formatDateTime(format, timezone=NO_VALUE) #### gcd(b) @@ -2021,6 +2217,9 @@ Initializer. #### generateUUIDv4() +#### geohashEncode(y, precision=12) + + #### greater(**kwargs) @@ -2717,6 +2916,42 @@ Initializer. #### startsWith(prefix) +#### stddevPop(**kwargs) + + +#### stddevPopIf(cond) + + +#### stddevPopOrDefault() + + +#### stddevPopOrDefaultIf(cond) + + +#### stddevPopOrNull() + + +#### stddevPopOrNullIf(cond) + + +#### stddevSamp(**kwargs) + + +#### stddevSampIf(cond) + + +#### stddevSampOrDefault() + + +#### stddevSampOrDefaultIf(cond) + + +#### stddevSampOrNull() + + +#### stddevSampOrNullIf(cond) + + #### substring(**kwargs) @@ -2870,10 +3105,10 @@ Initializer. #### toIPv6() -#### toISOWeek(timezone="") +#### toISOWeek(timezone=NO_VALUE) -#### toISOYear(timezone="") +#### toISOYear(timezone=NO_VALUE) #### toInt16(**kwargs) @@ -2945,28 +3180,28 @@ Initializer. #### toMonth() -#### toQuarter(timezone="") +#### toQuarter(timezone=NO_VALUE) -#### toRelativeDayNum(timezone="") +#### toRelativeDayNum(timezone=NO_VALUE) -#### toRelativeHourNum(timezone="") +#### toRelativeHourNum(timezone=NO_VALUE) -#### toRelativeMinuteNum(timezone="") +#### toRelativeMinuteNum(timezone=NO_VALUE) -#### toRelativeMonthNum(timezone="") +#### toRelativeMonthNum(timezone=NO_VALUE) -#### toRelativeSecondNum(timezone="") +#### toRelativeSecondNum(timezone=NO_VALUE) -#### toRelativeWeekNum(timezone="") +#### toRelativeWeekNum(timezone=NO_VALUE) -#### toRelativeYearNum(timezone="") +#### toRelativeYearNum(timezone=NO_VALUE) #### toSecond() @@ -3011,7 +3246,7 @@ Initializer. #### toStringCutToZero() -#### toTime(timezone="") +#### toTime(timezone=NO_VALUE) #### toTimeZone(timezone) @@ -3056,19 +3291,19 @@ Initializer. #### toUUID() -#### toUnixTimestamp(timezone="") +#### toUnixTimestamp(timezone=NO_VALUE) -#### toWeek(mode=0, timezone="") +#### toWeek(mode=0, timezone=NO_VALUE) -#### toYYYYMM(timezone="") +#### toYYYYMM(timezone=NO_VALUE) -#### toYYYYMMDD(timezone="") +#### toYYYYMMDD(timezone=NO_VALUE) -#### toYYYYMMDDhhmmss(timezone="") +#### toYYYYMMDDhhmmss(timezone=NO_VALUE) #### toYear() @@ -3135,6 +3370,9 @@ For other functions: #### tryBase64Decode() +#### tupleElement(n) + + #### unhex() @@ -3144,3 +3382,314 @@ For other functions: #### uniqExact(**kwargs) +#### uniqExactIf() + + +#### uniqExactOrDefault() + + +#### uniqExactOrDefaultIf() + + +#### uniqExactOrNull() + + +#### uniqExactOrNullIf() + + +#### uniqHLL12(**kwargs) + + +#### uniqHLL12If() + + +#### uniqHLL12OrDefault() + + +#### uniqHLL12OrDefaultIf() + + +#### uniqHLL12OrNull() + + +#### uniqHLL12OrNullIf() + + +#### uniqIf() + + +#### uniqOrDefault() + + +#### uniqOrDefaultIf() + + +#### uniqOrNull() + + +#### uniqOrNullIf() + + +#### upper(**kwargs) + + +#### upperUTF8() + + +#### varPop(**kwargs) + + +#### varPopIf(cond) + + +#### varPopOrDefault() + + +#### varPopOrDefaultIf(cond) + + +#### varPopOrNull() + + +#### varPopOrNullIf(cond) + + +#### varSamp(**kwargs) + + +#### varSampIf(cond) + + +#### varSampOrDefault() + + +#### varSampOrDefaultIf(cond) + + +#### varSampOrNull() + + +#### varSampOrNullIf(cond) + + +#### xxHash32() + + +#### xxHash64() + + +#### yesterday() + + +clickhouse_orm.system_models +---------------------------- + +### SystemPart + +Extends Model + + +Contains information about parts of a table in the MergeTree family. +This model operates only fields, described in the reference. Other fields are ignored. +https://clickhouse.tech/docs/en/system_tables/system.parts/ + +#### SystemPart(**kwargs) + + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### attach(settings=None) + + + Add a new part or partition from the 'detached' directory to the table. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.create_table_sql(db) + + +Returns the SQL statement for creating a table for this model. + + +#### detach(settings=None) + + +Move a partition to the 'detached' directory and forget it. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### drop(settings=None) + + +Delete a partition + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.drop_table_sql(db) + + +Returns the SQL command for deleting this model's table. + + +#### fetch(zookeeper_path, settings=None) + + +Download a partition from another server. + +- `zookeeper_path`: Path in zookeeper to fetch from +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.fields(writable=False) + + +Returns an `OrderedDict` of the model's fields (from name to `Field` instance). +If `writable` is true, only writable fields are included. +Callers should not modify the dictionary. + + +#### freeze(settings=None) + + +Create a backup of a partition. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None) + + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones. +- `database`: if given, sets the database that this instance belongs to. + + +#### SystemPart.get(database, conditions="") + + +Get all data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database condition is added automatically + +Returns: A list of SystemPart objects + + +#### SystemPart.get_active(database, conditions="") + + +Gets active data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database and active conditions are added automatically + +Returns: A list of SystemPart objects + + +#### get_database() + + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + + +Gets a `Field` instance given its name, or `None` if not found. + + +#### SystemPart.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + +#### SystemPart.is_read_only() + + +Returns true if the model is marked as read only. + + +#### SystemPart.is_system_model() + + +Returns true if the model represents a system table. + + +#### SystemPart.is_temporary_model() + + +Returns true if the model represents a temporary table. + + +#### SystemPart.objects_in(database) + + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### SystemPart.table_name() + + +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + +#### to_dict(include_readonly=True, field_names=None) + + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +#### to_tsv(include_readonly=True) + + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + diff --git a/docs/contributing.md b/docs/contributing.md index c173cb9..688919e 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,7 +1,7 @@ Contributing ============ -This project is hosted on GitHub - [https://github.com/Infinidat/infi.clickhouse_orm/](https://github.com/Infinidat/infi.clickhouse_orm/). +This project is hosted on GitHub - [https://github.com/sswest/ch-orm](https://github.com/sswest/ch-orm). Please open an issue there if you encounter a bug or want to request a feature. Pull requests are also welcome. @@ -11,29 +11,24 @@ Building After cloning the project, run the following commands: - easy_install -U infi.projector - cd infi.clickhouse_orm - projector devenv build + pip install build + python -m build -A `setup.py` file will be generated, which you can use to install the development version of the package: +A `dist` directory will be generated, which you can use to install the development version of the package: - python setup.py install + pip install dist/* Tests ----- To run the tests, ensure that the ClickHouse server is running on (this is the default), and run: - bin/nosetests + python -m unittest To see test coverage information run: - bin/nosetests --with-coverage --cover-package=infi.clickhouse_orm - -To test with tox, ensure that the setup.py is present (otherwise run `bin/buildout buildout:develop= setup.py`) and run: - - pip install tox - tox + coverage run --source=clickhouse_orm -m unittest + coverage report -m --- diff --git a/docs/expressions.md b/docs/expressions.md index d9237bd..b3c6a1d 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -96,4 +96,4 @@ Note that higher-order database functions (those that use lambda expressions) ar --- -[<< Models and Databases](models_and_databases.md) | [Table of Contents](toc.md) | [Importing ORM Classes >>](importing_orm_classes.md) +[<< Async Databases](async_databases.md) | [Table of Contents](toc.md) | [Importing ORM Classes >>](importing_orm_classes.md) diff --git a/docs/field_options.md b/docs/field_options.md index 3019c98..54bb4a2 100644 --- a/docs/field_options.md +++ b/docs/field_options.md @@ -8,6 +8,7 @@ All field types accept the following arguments: - materialized - readonly - codec + - db_column Note that `default`, `alias` and `materialized` are mutually exclusive - you cannot use more than one of them in a single field. @@ -25,7 +26,7 @@ class Event(Model): engine = Memory() ... ``` -When creating a model instance, any fields you do not specify get their default value. Fields that use a default expression are assigned a sentinel value of `infi.clickhouse_orm.utils.NO_VALUE` instead. For example: +When creating a model instance, any fields you do not specify get their default value. Fields that use a default expression are assigned a sentinel value of `clickhouse_orm.utils.NO_VALUE` instead. For example: ```python >>> event = Event() >>> print(event.to_dict()) @@ -33,6 +34,20 @@ When creating a model instance, any fields you do not specify get their default ``` :warning: Due to a bug in ClickHouse versions prior to 20.1.2.4, insertion of records with expressions for default values may fail. +## db_column + +db_column allows you to use the field names defined by the clickhouse backend, rather than Field instance names. + +```python +class Style(Model): + create_time = DateTimeField(default=F.now(), db_column="createTime") + + engine = Memory() +``` + +You can use the `create_time` field for all ORM operations, but the clickhouse will store the column named `createTime`. + + ## alias / materialized The `alias` and `materialized` attributes expect an expression that gets calculated by the database. The difference is that `alias` fields are calculated on the fly, while `materialized` fields are calculated when the record is inserted, and are stored on disk. @@ -63,7 +78,7 @@ db.select('SELECT created, created_date, username, name FROM $db.event', model_c # created_date and username will contain a default value db.select('SELECT * FROM $db.event', model_class=Event) ``` -When creating a model instance, any alias or materialized fields are assigned a sentinel value of `infi.clickhouse_orm.utils.NO_VALUE` since their real values can only be known after insertion to the database. +When creating a model instance, any alias or materialized fields are assigned a sentinel value of `clickhouse_orm.utils.NO_VALUE` since their real values can only be known after insertion to the database. ## codec diff --git a/docs/field_types.md b/docs/field_types.md index 7613276..8477398 100644 --- a/docs/field_types.md +++ b/docs/field_types.md @@ -5,34 +5,37 @@ See: [ClickHouse Documentation](https://clickhouse.tech/docs/en/sql-reference/da The following field types are supported: -| Class | DB Type | Pythonic Type | Comments -| ------------------ | ---------- | --------------------- | ----------------------------------------------------- -| StringField | String | str | Encoded as UTF-8 when written to ClickHouse -| FixedStringField | FixedString| str | Encoded as UTF-8 when written to ClickHouse -| DateField | Date | datetime.date | Range 1970-01-01 to 2105-12-31 -| DateTimeField | DateTime | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Timezone aware -| DateTime64Field | DateTime64 | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Timezone aware -| Int8Field | Int8 | int | Range -128 to 127 -| Int16Field | Int16 | int | Range -32768 to 32767 -| Int32Field | Int32 | int | Range -2147483648 to 2147483647 -| Int64Field | Int64 | int | Range -9223372036854775808 to 9223372036854775807 -| UInt8Field | UInt8 | int | Range 0 to 255 -| UInt16Field | UInt16 | int | Range 0 to 65535 -| UInt32Field | UInt32 | int | Range 0 to 4294967295 -| UInt64Field | UInt64 | int | Range 0 to 18446744073709551615 -| Float32Field | Float32 | float | -| Float64Field | Float64 | float | -| DecimalField | Decimal | Decimal | Pythonic values are rounded to fit the scale of the database field -| Decimal32Field | Decimal32 | Decimal | Ditto -| Decimal64Field | Decimal64 | Decimal | Ditto -| Decimal128Field | Decimal128 | Decimal | Ditto -| UUIDField | UUID | uuid.UUID | -| IPv4Field | IPv4 | ipaddress.IPv4Address | -| IPv6Field | IPv6 | ipaddress.IPv6Address | -| Enum8Field | Enum8 | Enum | See below -| Enum16Field | Enum16 | Enum | See below -| ArrayField | Array | list | See below -| NullableField | Nullable | See below | See below +| Class | DB Type | Pythonic Type | Comments +|------------------|-------------|------------------------| ----------------------------------------------------- +| StringField | String | str | Encoded as UTF-8 when written to ClickHouse +| FixedStringField | FixedString | str | Encoded as UTF-8 when written to ClickHouse +| DateField | Date | datetime.date | Range 1970-01-01 to 2105-12-31 +| DateTimeField | DateTime | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Timezone aware +| DateTime64Field | DateTime64 | datetime.datetime | Minimal value is 1970-01-01 00:00:00; Timezone aware +| Int8Field | Int8 | int | Range -128 to 127 +| Int16Field | Int16 | int | Range -32768 to 32767 +| Int32Field | Int32 | int | Range -2147483648 to 2147483647 +| Int64Field | Int64 | int | Range -9223372036854775808 to 9223372036854775807 +| UInt8Field | UInt8 | int | Range 0 to 255 +| UInt16Field | UInt16 | int | Range 0 to 65535 +| UInt32Field | UInt32 | int | Range 0 to 4294967295 +| UInt64Field | UInt64 | int | Range 0 to 18446744073709551615 +| Float32Field | Float32 | float | +| Float64Field | Float64 | float | +| DecimalField | Decimal | Decimal | Pythonic values are rounded to fit the scale of the database field +| Decimal32Field | Decimal32 | Decimal | Ditto +| Decimal64Field | Decimal64 | Decimal | Ditto +| Decimal128Field | Decimal128 | Decimal | Ditto +| UUIDField | UUID | uuid.UUID | +| IPv4Field | IPv4 | ipaddress.IPv4Address | +| IPv6Field | IPv6 | ipaddress.IPv6Address | +| Enum8Field | Enum8 | Enum | See below +| Enum16Field | Enum16 | Enum | See below +| ArrayField | Array | list | See below +| TupleField | Tuple | tuple | See below +| PointField | Point | contrib.geo.fields.Point | Experimental feature +| RingField | Ring | contrib.geo.fields.Ring | Experimental feature +| NullableField | Nullable | See below | See below DateTimeField and Time Zones @@ -96,6 +99,28 @@ data = SensorData(date=date.today(), temperatures=[25.5, 31.2, 28.7], humidity_l Note that multidimensional arrays are not supported yet by the ORM. +Working with tuple fields +------------------------- + +You can create tuple fields containing multiple data type, for example: + +```python +from datetime import date + +from clickhouse_orm.models import Model +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, Float32Field, UInt8Field, TupleField + +class SensorData(Model): + + date = DateField() + info = TupleField([('t', Float32Field()), ('h', UInt8Field())]) + + engine = MergeTree('date', ('date',)) + +data = SensorData(date=date.today(), info=(25.5, 41)) +``` + Working with nullable fields ---------------------------- [ClickHouse provides a NULL value support](https://clickhouse.tech/docs/en/sql-reference/data-types/nullable/). diff --git a/docs/importing_orm_classes.md b/docs/importing_orm_classes.md index 77d04e4..a7366fb 100644 --- a/docs/importing_orm_classes.md +++ b/docs/importing_orm_classes.md @@ -7,24 +7,24 @@ The ORM supports different styles of importing and referring to its classes, so Importing Everything -------------------- -It is safe to use `import *` from `infi.clickhouse_orm` or its submodules. Only classes that are needed by users of the ORM will get imported, and nothing else: +It is safe to use `import *` from `clickhouse_orm` or its submodules. Only classes that are needed by users of the ORM will get imported, and nothing else: ```python -from infi.clickhouse_orm import * +from clickhouse_orm import * ``` This is exactly equivalent to the following import statements: ```python -from infi.clickhouse_orm.database import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.funcs import * -from infi.clickhouse_orm.migrations import * -from infi.clickhouse_orm.models import * -from infi.clickhouse_orm.query import * -from infi.clickhouse_orm.system_models import * +from clickhouse_orm.database import * +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.funcs import * +from clickhouse_orm.migrations import * +from clickhouse_orm.models import * +from clickhouse_orm.query import * +from clickhouse_orm.system_models import * ``` By importing everything, all of the ORM's public classes can be used directly. For example: ```python -from infi.clickhouse_orm import * +from clickhouse_orm import * class Event(Model): @@ -40,8 +40,8 @@ Importing Everything into a Namespace To prevent potential name clashes and to make the code more readable, you can import the ORM's classes into a namespace of your choosing, e.g. `orm`. For brevity, it is recommended to import the `F` class explicitly: ```python -import infi.clickhouse_orm as orm -from infi.clickhouse_orm import F +import clickhouse_orm as orm +from clickhouse_orm import F class Event(orm.Model): @@ -57,7 +57,7 @@ Importing Specific Submodules It is possible to import only the submodules you need, and use their names to qualify the ORM's class names. This option is more verbose, but makes it clear where each class comes from. For example: ```python -from infi.clickhouse_orm import models, fields, engines, F +from clickhouse_orm import models, fields, engines, F class Event(models.Model): @@ -73,7 +73,7 @@ Importing Specific Classes If you prefer, you can import only the specific ORM classes that you need directly from `infi.clickhouse_orm`: ```python -from infi.clickhouse_orm import Model, StringField, UInt32Field, DateTimeField, F, Memory +from clickhouse_orm import Model, StringField, UInt32Field, DateTimeField, F, Memory class Event(Model): diff --git a/docs/index.md b/docs/index.md index db75910..ee71bee 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,14 +3,16 @@ Overview This project is simple ORM for working with the [ClickHouse database](https://clickhouse.tech/). It allows you to define model classes whose instances can be written to the database and read from it. -Version 1.x supports Python 2.7 and 3.5+. Version 2.x dropped support for Python 2.7, and works only with Python 3.5+. +This repository expects to use more type hints, and will drop support for Python 2.x. + +Supports both synchronous and asynchronous ways to interact with the clickhouse server. Means you can use asyncio to perform asynchronous queries, although the asynchronous mode is not well tested. Installation ------------ -To install infi.clickhouse_orm: +To install clickhouse_orm: - pip install infi.clickhouse_orm + pip install ch-orm --- diff --git a/docs/models_and_databases.md b/docs/models_and_databases.md index 28ab6ad..30c5867 100644 --- a/docs/models_and_databases.md +++ b/docs/models_and_databases.md @@ -10,7 +10,7 @@ Defining Models Models are defined in a way reminiscent of Django's ORM, by subclassing `Model`: ```python -from infi.clickhouse_orm import Model, StringField, DateField, Float32Field, MergeTree +from clickhouse_orm import Model, StringField, DateField, Float32Field, MergeTree class Person(Model): @@ -68,6 +68,8 @@ For additional details see [here](field_options.md). The table name used for the model is its class name, converted to lowercase. To override the default name, implement the `table_name` method: ```python +from clickhouse_orm.models import Model + class Person(Model): ... @@ -81,10 +83,14 @@ class Person(Model): It is possible to define constraints which ClickHouse verifies when data is inserted. Trying to insert invalid records will raise a `ServerError`. Each constraint has a name and an expression to validate. For example: ```python +from clickhouse_orm.models import Model, Constraint +from clickhouse_orm.funcs import F +from clickhouse_orm.fields import DateTimeField + class Person(Model): ... - + birthday = DateTimeField() # Ensure that the birthday is not a future date birthday_is_in_the_past = Constraint(birthday <= F.today()) ``` @@ -95,10 +101,17 @@ Models that use an engine from the `MergeTree` family can define additional inde For example: ```python +from clickhouse_orm.models import Model, Index +from clickhouse_orm.funcs import F +from clickhouse_orm.fields import StringField, Float32Field + class Person(Model): ... - + first_name = StringField() + last_name = StringField() + height = Float32Field() + # A minmax index that can help find people taller or shorter than some height height_index = Index(height, type=Index.minmax(), granularity=2) @@ -116,7 +129,7 @@ Once you have a model, you can create model instances: >>> dan = Person(first_name='Dan', last_name='Schwartz') >>> suzy = Person(first_name='Suzy', last_name='Jones') >>> dan.first_name - u'Dan' + 'Dan' When values are assigned to model fields, they are immediately converted to their Pythonic data type. In case the value is invalid, a `ValueError` is raised: @@ -133,9 +146,11 @@ Inserting to the Database To write your instances to ClickHouse, you need a `Database` instance: - from infi.clickhouse_orm import Database +```python +from clickhouse_orm import Database - db = Database('my_test_db') +db = Database('my_test_db') +``` This automatically connects to and creates a database called my_test_db, unless it already exists. If necessary, you can specify a different database URL and optional credentials: @@ -247,4 +262,4 @@ Note that `order_by` must be chosen so that the ordering is unique, otherwise th --- -[<< Overview](index.md) | [Table of Contents](toc.md) | [Expressions >>](expressions.md) \ No newline at end of file +[<< Overview](index.md) | [Table of Contents](toc.md) | [Async DataBase >>](async_databases.md) \ No newline at end of file diff --git a/docs/querysets.md b/docs/querysets.md index 76ecb0e..9837f5a 100644 --- a/docs/querysets.md +++ b/docs/querysets.md @@ -11,6 +11,11 @@ This queryset matches all Person instances in the database. You can get these in for person in qs: print(person.first_name, person.last_name) +For AioDatabase instances: + + async for person in qs: + print(person.first_name, person.last_name) + Filtering --------- @@ -88,24 +93,26 @@ qs = qs.filter(x__gt=100, y__lt=20, terrain='water') ``` Below are all the supported operators. -| Operator | Equivalent SQL | Comments | -| -------- | -------------------------------------------- | ---------------------------------- | -| `eq` | `field = value` | | -| `ne` | `field != value` | | -| `gt` | `field > value` | | -| `gte` | `field >= value` | | -| `lt` | `field < value` | | -| `lte` | `field <= value` | | -| `between` | `field BETWEEN value1 AND value2` | | -| `in` | `field IN (values)` | | -| `not_in` | `field NOT IN (values)` | | -| `contains` | `field LIKE '%value%'` | For string fields only | -| `startswith` | `field LIKE 'value%'` | For string fields only | -| `endswith` | `field LIKE '%value'` | For string fields only | -| `icontains` | `lowerUTF8(field) LIKE lowerUTF8('%value%')` | For string fields only | -| `istartswith` | `lowerUTF8(field) LIKE lowerUTF8('value%')` | For string fields only | -| `iendswith` | `lowerUTF8(field) LIKE lowerUTF8('%value')` | For string fields only | -| `iexact` | `lowerUTF8(field) = lowerUTF8(value)` | For string fields only | +| Operator | Equivalent SQL | Comments | +|---------------|----------------------------------------------| ---------------------------------- | +| `eq` | `field = value` | | +| `ne` | `field != value` | | +| `gt` | `field > value` | | +| `gte` | `field >= value` | | +| `lt` | `field < value` | | +| `lte` | `field <= value` | | +| `between` | `field BETWEEN value1 AND value2` | | +| `in` | `field IN (values)` | | +| `gin` | `field GLOBAL IN (values)` | | +| `not_in` | `field NOT IN (values)` | | +| `not_gin` | `field NOT GLOBAL IN (values)` | | +| `contains` | `field LIKE '%value%'` | For string fields only | +| `startswith` | `field LIKE 'value%'` | For string fields only | +| `endswith` | `field LIKE '%value'` | For string fields only | +| `icontains` | `lowerUTF8(field) LIKE lowerUTF8('%value%')` | For string fields only | +| `istartswith` | `lowerUTF8(field) LIKE lowerUTF8('value%')` | For string fields only | +| `iendswith` | `lowerUTF8(field) LIKE lowerUTF8('%value')` | For string fields only | +| `iexact` | `lowerUTF8(field) = lowerUTF8(value)` | For string fields only | Counting and Checking Existence ------------------------------- @@ -114,6 +121,9 @@ Use the `count` method to get the number of matches: Person.objects_in(database).count() + # aio + # await Person.objects_in(database).count() + To check if there are any matches at all, you can use any of the following equivalent options: if qs.count(): ... @@ -164,7 +174,7 @@ Adds a FINAL modifier to the query, meaning that the selected data is fully "col Slicing ------- -It is possible to get a specific item from the queryset by index: +It is possible to get a specific item from the queryset by index (**not applicable to AioDatabase)**: qs = Person.objects_in(database).order_by('last_name', 'first_name') first = qs[0] @@ -175,6 +185,9 @@ It is also possible to get a range a instances using a slice. This returns a que first_ten_people = list(qs[:10]) next_ten_people = list(qs[10:20]) + # first_ten_people = [_ async for _ in qs[:10]] + # next_ten_people = [_ async for _ in qs[10:20]] + You should use `order_by` to ensure a consistent ordering of the results. Trying to use negative indexes or a slice with a step (e.g. [0 : 100 : 2]) is not supported and will raise an `AssertionError`. diff --git a/docs/toc.md b/docs/toc.md index 2fd878f..2c19b5a 100644 --- a/docs/toc.md +++ b/docs/toc.md @@ -20,12 +20,16 @@ * [Counting](models_and_databases.md#counting) * [Pagination](models_and_databases.md#pagination) + * [Async Databases](async_databases.md#async-databases) + * [Insert from the AioDatabase](async_databases.md#insert-from-the-aiodatabase) + * [Reading from the AioDatabase](async_databases.md#reading-from-the-aiodatabase) + * [Querysets](querysets.md#querysets) * [Filtering](querysets.md#filtering) - * [Using IN and NOT IN](querysets.md#using-in-and-not-in) - * [Specifying PREWHERE conditions](querysets.md#specifying-prewhere-conditions) - * [Old-style filter conditions](querysets.md#old-style-filter-conditions) - * [Counting and Checking Existence](querysets.md#counting-and-checking-existence) + * [Using IN and NOT IN](querysets.md#using-in-and not-in) + * [Specifying PREWHERE conditions](querysets.md#specifying-prewhere conditions) + * [Old-style filter conditions](querysets.md#old-style-filter-conditions) + * [Counting and Checking Existence](querysets.md#counting-and-checking existence) * [Ordering](querysets.md#ordering) * [Omitting Fields](querysets.md#omitting-fields) * [Distinct](querysets.md#distinct) @@ -38,6 +42,7 @@ * [Field Options](field_options.md#field-options) * [default](field_options.md#default) + * [db_column](field_options.md#db_column) * [alias / materialized](field_options.md#alias-/-materialized) * [codec](field_options.md#codec) * [readonly](field_options.md#readonly) @@ -46,13 +51,14 @@ * [DateTimeField and Time Zones](field_types.md#datetimefield-and-time-zones) * [Working with enum fields](field_types.md#working-with-enum-fields) * [Working with array fields](field_types.md#working-with-array-fields) + * [Working with tuple fields](field_types.md#working-with-tuple-fields) * [Working with nullable fields](field_types.md#working-with-nullable-fields) - * [Working with LowCardinality fields](field_types.md#working-with-lowcardinality-fields) + * [Working with LowCardinality fields](field_types.md#working-with-lowcardinality fields) * [Creating custom field types](field_types.md#creating-custom-field-types) * [Table Engines](table_engines.md#table-engines) * [Simple Engines](table_engines.md#simple-engines) - * [Engines in the MergeTree Family](table_engines.md#engines-in-the-mergetree-family) + * [Engines in the MergeTree Family](table_engines.md#engines-in-the-mergetree family) * [Custom partitioning](table_engines.md#custom-partitioning) * [Primary key](table_engines.md#primary-key) * [Data Replication](table_engines.md#data-replication) diff --git a/docs/whats_new_in_version_2.md b/docs/whats_new_in_version_2.md index 378adca..df8a77c 100644 --- a/docs/whats_new_in_version_2.md +++ b/docs/whats_new_in_version_2.md @@ -50,9 +50,9 @@ for row in QueryLog.objects_in(db).filter(QueryLog.query_duration_ms > 10000): ## Convenient ways to import ORM classes -You can now import all ORM classes directly from `infi.clickhouse_orm`, without worrying about sub-modules. For example: +You can now import all ORM classes directly from `clickhouse_orm`, without worrying about sub-modules. For example: ```python -from infi.clickhouse_orm import Database, Model, StringField, DateTimeField, MergeTree +from clickhouse_orm import Database, Model, StringField, DateTimeField, MergeTree ``` See [Importing ORM Classes](importing_orm_classes.md). diff --git a/scripts/generate_ref.py b/scripts/generate_ref.py index 22850ab..bc17f8a 100644 --- a/scripts/generate_ref.py +++ b/scripts/generate_ref.py @@ -29,6 +29,7 @@ def _get_default_arg(args, defaults, arg_index): value = '"%s"' % value return DefaultArgSpec(True, value) + def get_method_sig(method): """ Given a function, it returns a string that pretty much looks how the function signature would be written in python. @@ -42,8 +43,8 @@ def get_method_sig(method): # list of defaults are returned in separate array. # eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'], # varargs=None, keywords=None, defaults=(42, 'something')) - argspec = inspect.getargspec(method) - arg_index=0 + argspec = inspect.getfullargspec(method) + arg_index = 0 args = [] # Use the args and defaults array returned by argspec and find out @@ -58,8 +59,8 @@ def get_method_sig(method): arg_index += 1 if argspec.varargs: args.append('*' + argspec.varargs) - if argspec.keywords: - args.append('**' + argspec.keywords) + if argspec.varkw: + args.append('**' + argspec.varkw) return "%s(%s)" % (method.__name__, ", ".join(args[1:])) @@ -120,18 +121,20 @@ def all_subclasses(cls): if __name__ == '__main__': - from infi.clickhouse_orm import database - from infi.clickhouse_orm import fields - from infi.clickhouse_orm import engines - from infi.clickhouse_orm import models - from infi.clickhouse_orm import query - from infi.clickhouse_orm import funcs - from infi.clickhouse_orm import system_models + from clickhouse_orm import database + from clickhouse_orm import fields + from clickhouse_orm import engines + from clickhouse_orm import models + from clickhouse_orm import query + from clickhouse_orm import funcs + from clickhouse_orm import system_models + from clickhouse_orm.aio import database as aio_database print('Class Reference') print('===============') print() module_doc([database.Database, database.DatabaseException]) + module_doc([aio_database.AioDatabase]) module_doc([models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index]) module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False) module_doc([engines.Engine] + all_subclasses(engines.Engine), False) diff --git a/scripts/generate_toc.sh b/scripts/generate_toc.sh index a77aaaa..f91a80e 100755 --- a/scripts/generate_toc.sh +++ b/scripts/generate_toc.sh @@ -1,13 +1,14 @@ generate_one() { # Converts Markdown to HTML using Pandoc, and then extracts the header tags - pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md + pandoc "$1" | python3 "../scripts/html_to_markdown_toc.py" "$1" >> toc.md } printf "# Table of Contents\n\n" > toc.md generate_one "index.md" generate_one "models_and_databases.md" +generate_one "async_databases.md" generate_one "querysets.md" generate_one "field_options.md" generate_one "field_types.md" diff --git a/src/clickhouse_orm/aio/database.py b/src/clickhouse_orm/aio/database.py index 2498314..fd36128 100644 --- a/src/clickhouse_orm/aio/database.py +++ b/src/clickhouse_orm/aio/database.py @@ -51,6 +51,7 @@ class AioDatabase(Database): ): r = await super()._send(data, settings, stream) if r.status_code != 200: + await r.aread() raise ServerError(r.text) return r @@ -85,11 +86,6 @@ class AioDatabase(Database): """ Creates the database on the ClickHouse server if it does not already exist. """ - if not self._init: - raise DatabaseException( - 'The AioDatabase object must execute the init method before it can be used' - ) - await self._send('CREATE DATABASE IF NOT EXISTS `%s`' % self.db_name) self.db_exists = True @@ -97,11 +93,6 @@ class AioDatabase(Database): """ Deletes the database on the ClickHouse server. """ - if not self._init: - raise DatabaseException( - 'The AioDatabase object must execute the init method before it can be used' - ) - await self._send('DROP DATABASE `%s`' % self.db_name) self.db_exists = False diff --git a/src/clickhouse_orm/database.py b/src/clickhouse_orm/database.py index 9921016..013207f 100644 --- a/src/clickhouse_orm/database.py +++ b/src/clickhouse_orm/database.py @@ -100,7 +100,7 @@ class Database: - `username`: optional connection credentials. - `password`: optional connection credentials. - `readonly`: use a read-only connection. - - `autocreate`: automatically create the database + - `auto_create`: automatically create the database if it does not exist (unless in readonly mode). - `timeout`: the connection timeout in seconds. - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS. diff --git a/src/clickhouse_orm/models.py b/src/clickhouse_orm/models.py index e0a2636..daf4a54 100644 --- a/src/clickhouse_orm/models.py +++ b/src/clickhouse_orm/models.py @@ -49,7 +49,7 @@ class Index: name: Optional[str] = None # this is set by the parent model parent: Optional[type["Model"]] = None # this is set by the parent model - def __init__(self, expr: F, type: str, granularity: int): + def __init__(self, expr: Field | F | tuple, type: str, granularity: int): """ Initializer. @@ -238,10 +238,14 @@ class ModelBase(type): return orm_fields.ArrayField(inner_field) # Tuples if db_type.startswith('Tuple'): - types = [s.strip() for s in db_type[6:-1].split(',')] - return orm_fields.TupleField(name_fields=[ - (str(i), cls.create_ad_hoc_field(type_name)) for i, type_name in enumerate(types)] - ) + types = [s.strip().split(' ') for s in db_type[6:-1].split(',')] + name_fields = [] + for i, tp in enumerate(types): + if len(tp) == 2: + name_fields.append((tp[0], cls.create_ad_hoc_field(tp[1]))) + else: + name_fields.append((str(i), cls.create_ad_hoc_field(tp[0]))) + return orm_fields.TupleField(name_fields=name_fields) # FixedString if db_type.startswith('FixedString'): length = int(db_type[12:-1]) diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index f48d11b..ccb70a2 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -5,6 +5,7 @@ from clickhouse_orm.database import Database from clickhouse_orm.models import Model from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.aio.database import AioDatabase import logging logging.getLogger("requests").setLevel(logging.WARNING) @@ -35,6 +36,31 @@ class TestCaseWithData(unittest.TestCase): yield Person(**entry) +class TestCaseWithAsyncData(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self): + self.database = AioDatabase('test-db', log_statements=True) + await self.database.init() + await self.database.create_table(Person) + + async def asyncTearDown(self): + await self.database.drop_table(Person) + await self.database.drop_database() + + async def _insert_all(self): + await self.database.insert(self._sample_data()) + self.assertTrue(await self.database.count(Person)) + + async def _insert_and_check(self, data, count, batch_size=1000): + await self.database.insert(data, batch_size=batch_size) + self.assertEqual(count, await self.database.count(Person)) + for instance in data: + self.assertEqual(self.database, instance.get_database()) + + def _sample_data(self): + for entry in data: + yield Person(**entry) + class Person(Model): diff --git a/tests/test_aiodatabase.py b/tests/test_aiodatabase.py new file mode 100644 index 0000000..1c9bea5 --- /dev/null +++ b/tests/test_aiodatabase.py @@ -0,0 +1,331 @@ +# -*- coding: utf-8 -*- +import unittest +import datetime + +from clickhouse_orm.database import ServerError, DatabaseException +from clickhouse_orm.query import Q +from clickhouse_orm.funcs import F +from tests.base_test_with_data import * + + +class DatabaseTestCase(TestCaseWithAsyncData): + + async def test_insert__generator(self): + await self._insert_and_check(self._sample_data(), len(data)) + + async def test_insert__list(self): + await self._insert_and_check(list(self._sample_data()), len(data)) + + async def test_insert__iterator(self): + await self._insert_and_check(iter(self._sample_data()), len(data)) + + async def test_insert__empty(self): + await self._insert_and_check([], 0) + + async def test_insert__small_batches(self): + await self._insert_and_check(self._sample_data(), len(data), batch_size=10) + + async def test_insert__medium_batches(self): + await self._insert_and_check(self._sample_data(), len(data), batch_size=100) + + async def test_insert__funcs_as_default_values(self): + if self.database.server_version < (20, 1, 2, 4): + raise unittest.SkipTest('Buggy in server versions before 20.1.2.4') + + class TestModel(Model): + a = DateTimeField(default=datetime.datetime(2020, 1, 1)) + b = DateField(default=F.toDate(a)) + c = Int32Field(default=7) + d = Int32Field(default=c * 5) + engine = Memory() + await self.database.create_table(TestModel) + await self.database.insert([TestModel()]) + with self.assertRaises(TypeError): + # AioDatabase does not support queryset object value by index + obj = TestModel.objects_in(self.database)[0] + async for t in TestModel.objects_in(self.database): + self.assertEqual(str(t.b), '2020-01-01') + self.assertEqual(t.d, 35) + + async def test_count(self): + await self.database.insert(self._sample_data()) + self.assertEqual(await self.database.count(Person), 100) + # Conditions as string + self.assertEqual(await self.database.count(Person, "first_name = 'Courtney'"), 2) + self.assertEqual(await self.database.count(Person, "birthday > '2000-01-01'"), 22) + self.assertEqual(await self.database.count(Person, "birthday < '1970-03-01'"), 0) + # Conditions as expression + self.assertEqual( + await self.database.count(Person, Person.birthday > datetime.date(2000, 1, 1)), 22 + ) + # Conditions as Q object + self.assertEqual( + await self.database.count(Person, Q(birthday__gt=datetime.date(2000, 1, 1))), 22 + ) + + async def test_select(self): + await self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = [person async for person in self.database.select(query, Person)] + self.assertEqual(len(results), 2) + self.assertEqual(results[0].last_name, 'Durham') + self.assertEqual(results[0].height, 1.72) + self.assertEqual(results[1].last_name, 'Scott') + self.assertEqual(results[1].height, 1.70) + self.assertEqual(results[0].get_database(), self.database) + self.assertEqual(results[1].get_database(), self.database) + + async def test_dollar_in_select(self): + query = "SELECT * FROM $table WHERE first_name = '$utm_source'" + [_ async for _ in self.database.select(query, Person)] + + async def test_select_partial_fields(self): + await self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT first_name, last_name FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = [person async for person in self.database.select(query, Person)] + self.assertEqual(len(results), 2) + self.assertEqual(results[0].last_name, 'Durham') + self.assertEqual(results[0].height, 0) # default value + self.assertEqual(results[1].last_name, 'Scott') + self.assertEqual(results[1].height, 0) # default value + self.assertEqual(results[0].get_database(), self.database) + self.assertEqual(results[1].get_database(), self.database) + + async def test_select_ad_hoc_model(self): + await self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = [person async for person in self.database.select(query)] + self.assertEqual(len(results), 2) + self.assertEqual(results[0].__class__.__name__, 'AdHocModel') + self.assertEqual(results[0].last_name, 'Durham') + self.assertEqual(results[0].height, 1.72) + self.assertEqual(results[1].last_name, 'Scott') + self.assertEqual(results[1].height, 1.70) + self.assertEqual(results[0].get_database(), self.database) + self.assertEqual(results[1].get_database(), self.database) + + async def test_select_with_totals(self): + await self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT last_name, sum(height) as height FROM `test-db`.person GROUP BY last_name WITH TOTALS" + results = [person async for person in self.database.select(query)] + total = sum(r.height for r in results[:-1]) + # Last line has an empty last name, and total of all heights + self.assertFalse(results[-1].last_name) + self.assertEqual(total, results[-1].height) + + async def test_pagination(self): + await self._insert_and_check(self._sample_data(), len(data)) + # Try different page sizes + for page_size in (1, 2, 7, 10, 30, 100, 150): + # Iterate over pages and collect all intances + page_num = 1 + instances = set() + while True: + page = await self.database.paginate(Person, 'first_name, last_name', page_num, page_size) + self.assertEqual(page.number_of_objects, len(data)) + self.assertGreater(page.pages_total, 0) + [instances.add(obj.to_tsv()) for obj in page.objects] + if page.pages_total == page_num: + break + page_num += 1 + # Verify that all instances were returned + self.assertEqual(len(instances), len(data)) + + async def test_pagination_last_page(self): + await self._insert_and_check(self._sample_data(), len(data)) + # Try different page sizes + for page_size in (1, 2, 7, 10, 30, 100, 150): + # Ask for the last page in two different ways and verify equality + page_a = await self.database.paginate(Person, 'first_name, last_name', -1, page_size) + page_b = await self.database.paginate(Person, 'first_name, last_name', + page_a.pages_total, page_size) + self.assertEqual(page_a[1:], page_b[1:]) + self.assertEqual( + [obj.to_tsv() for obj in page_a.objects], [obj.to_tsv() for obj in page_b.objects] + ) + + async def test_pagination_empty_page(self): + for page_num in (-1, 1, 2): + page = await self.database.paginate( + Person, 'first_name, last_name', page_num, 10, conditions="first_name = 'Ziggy'" + ) + self.assertEqual(page.number_of_objects, 0) + self.assertEqual(page.objects, []) + self.assertEqual(page.pages_total, 0) + self.assertEqual(page.number, max(page_num, 1)) + + async def test_pagination_invalid_page(self): + await self._insert_and_check(self._sample_data(), len(data)) + for page_num in (0, -2, -100): + with self.assertRaises(ValueError): + await self.database.paginate(Person, 'first_name, last_name', page_num, 100) + + async def test_pagination_with_conditions(self): + await self._insert_and_check(self._sample_data(), len(data)) + # Conditions as string + page = await self.database.paginate( + Person, 'first_name, last_name', 1, 100, conditions="first_name < 'Ava'" + ) + self.assertEqual(page.number_of_objects, 10) + # Conditions as expression + page = await self.database.paginate( + Person, 'first_name, last_name', 1, 100, conditions=Person.first_name < 'Ava' + ) + self.assertEqual(page.number_of_objects, 10) + # Conditions as Q object + page = await self.database.paginate( + Person, 'first_name, last_name', 1, 100, conditions=Q(first_name__lt='Ava') + ) + self.assertEqual(page.number_of_objects, 10) + + async def test_special_chars(self): + s = u'אבגד \\\'"`,.;éåäöšž\n\t\0\b\r' + p = Person(first_name=s) + await self.database.insert([p]) + p = [_ async for _ in self.database.select("SELECT * from $table", Person)][0] + self.assertEqual(p.first_name, s) + + async def test_raw(self): + await self._insert_and_check(self._sample_data(), len(data)) + query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" + results = await self.database.raw(query) + self.assertEqual(results, "Whitney\tDurham\t1977-09-15\t1.72\t\\N\nWhitney\tScott\t1971-07-04\t1.7\t\\N\n") + + async def test_not_init(self): + with self.assertRaises(DatabaseException) as cm: + db = AioDatabase(self.database.db_name) + await db.create_table(Person) + + exc = cm.exception + self.assertTrue(exc.args[0].startswith('The AioDatabase object must execute the init')) + + async def test_read_only(self): + with self.assertRaises(DatabaseException) as cm: + db = AioDatabase('test-db-2', readonly=True) + await db.init() + + exc = cm.exception + self.assertTrue(exc.args[0].startswith('Database does not exist')) + + async def test_invalid_user(self): + with self.assertRaises(ServerError) as cm: + db = AioDatabase(self.database.db_name, username='default', password='wrong') + await db.init() + + exc = cm.exception + print(exc.code, exc.message) + if exc.code == 193: # ClickHouse version < 20.3 + self.assertTrue(exc.message.startswith('Wrong password for user default')) + elif exc.code == 516: # ClickHouse version >= 20.3 + self.assertTrue(exc.message.startswith('default: Authentication failed')) + else: + raise Exception('Unexpected error code - %s %s' % (exc.code, exc.message)) + + async def test_nonexisting_db(self): + db = AioDatabase('db_not_here', auto_create=False) + await db.init() + with self.assertRaises(ServerError) as cm: + await db.create_table(Person) + exc = cm.exception + self.assertEqual(exc.code, 81) + self.assertTrue(exc.message.startswith("Database db_not_here doesn't exist")) + # Create and delete the db twice, to ensure db_exists gets updated + for i in range(2): + # Now create the database - should succeed + await db.create_database() + self.assertTrue(db.db_exists) + await db.create_table(Person) + # Drop the database + await db.drop_database() + self.assertFalse(db.db_exists) + + async def test_preexisting_db(self): + db = AioDatabase(self.database.db_name, auto_create=False) + await db.init() + await db.count(Person) + + async def test_missing_engine(self): + class EnginelessModel(Model): + float_field = Float32Field() + with self.assertRaises(DatabaseException) as cm: + await self.database.create_table(EnginelessModel) + self.assertEqual(str(cm.exception), 'EnginelessModel class must define an engine') + + async def test_potentially_problematic_field_names(self): + class Model1(Model): + system = StringField() + readonly = StringField() + engine = Memory() + instance = Model1(system='s', readonly='r') + self.assertEqual(instance.to_dict(), dict(system='s', readonly='r')) + await self.database.create_table(Model1) + await self.database.insert([instance]) + instance = [_ async for _ in Model1.objects_in(self.database)[0:10]][0] + self.assertEqual(instance.to_dict(), dict(system='s', readonly='r')) + + async def test_does_table_exist(self): + class Person2(Person): + pass + self.assertTrue(await self.database.does_table_exist(Person)) + self.assertFalse(await self.database.does_table_exist(Person2)) + + async def test_add_setting(self): + # Non-string setting name should not be accepted + with self.assertRaises(AssertionError): + self.database.add_setting(0, 1) + # Add a setting and see that it makes the query fail + self.database.add_setting('max_columns_to_read', 1) + with self.assertRaises(ServerError): + [_ async for _ in self.database.select('SELECT * from system.tables')] + # Remove the setting and see that now it works + self.database.add_setting('max_columns_to_read', None) + [_ async for _ in self.database.select('SELECT * from system.tables')] + + async def test_create_ad_hoc_field(self): + # Tests that create_ad_hoc_field works for all column types in the database + from clickhouse_orm.models import ModelBase + query = "SELECT DISTINCT type FROM system.columns" + async for row in self.database.select(query): + if row.type.startswith('Map'): + continue # Not supported yet + ModelBase.create_ad_hoc_field(row.type) + + async def test_get_model_for_table(self): + # Tests that get_model_for_table works for a non-system model + model = await self.database.get_model_for_table('person') + self.assertFalse(model.is_system_model()) + self.assertFalse(model.is_read_only()) + self.assertEqual(model.table_name(), 'person') + # Read a few records + [_ async for _ in model.objects_in(self.database)[:10]] + # Inserts should work too + await self.database.insert([ + model(first_name='aaa', last_name='bbb', height=1.77) + ]) + + async def test_get_model_for_table__system(self): + # Tests that get_model_for_table works for all system tables + query = "SELECT name FROM system.tables WHERE database='system'" + async for row in self.database.select(query): + print(row.name) + if row.name in ('distributed_ddl_queue',): + continue # Not supported + try: + model = await self.database.get_model_for_table(row.name, system_table=True) + except NotImplementedError: + continue # Table contains an unsupported field type + self.assertTrue(model.is_system_model()) + self.assertTrue(model.is_read_only()) + self.assertEqual(model.table_name(), row.name) + # Read a few records + try: + [_ async for _ in model.objects_in(self.database)[:10]] + except ServerError as e: + if 'Not enough privileges' in e.message: + pass + elif 'no certificate file has been specified' in e.message: + pass + elif 'table must contain condition' in e.message: + pass + else: + raise diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index 8ddcc3c..323f531 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -3,8 +3,8 @@ from datetime import date from clickhouse_orm.database import Database from clickhouse_orm.models import Model, NO_VALUE -from clickhouse_orm.fields import * -from clickhouse_orm.engines import * +from clickhouse_orm.fields import Int32Field, StringField, DateField +from clickhouse_orm.engines import MergeTree from clickhouse_orm.funcs import F @@ -70,7 +70,7 @@ class ModelWithAliasFields(Model): date_field = DateField() str_field = StringField() - alias_str = StringField(alias=u'str_field') + alias_str = StringField(alias='str_field') alias_int = Int32Field(alias='int_field') alias_date = DateField(alias='date_field') alias_func = Int32Field(alias=F.toYYYYMM(date_field)) diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py index 20972be..ba53901 100644 --- a/tests/test_array_fields.py +++ b/tests/test_array_fields.py @@ -3,8 +3,8 @@ from datetime import date from clickhouse_orm.database import Database from clickhouse_orm.models import Model -from clickhouse_orm.fields import * -from clickhouse_orm.engines import * +from clickhouse_orm.fields import ArrayField, DateField, StringField, Int32Field +from clickhouse_orm.engines import MergeTree class ArrayFieldsTest(unittest.TestCase): diff --git a/tests/test_temporary_models.py b/tests/test_temporary_models.py new file mode 100644 index 0000000..be425a5 --- /dev/null +++ b/tests/test_temporary_models.py @@ -0,0 +1,46 @@ +import unittest +from datetime import date + +import os + +from clickhouse_orm.database import Database, DatabaseException, ServerError +from clickhouse_orm.engines import Memory, MergeTree +from clickhouse_orm.fields import UUIDField, DateField +from clickhouse_orm.models import TemporaryModel, Model +from clickhouse_orm.session import in_session + + +class TemporaryTest(unittest.TestCase): + + def setUp(self): + self.database = Database('test-db', log_statements=True) + + def tearDown(self): + self.database.drop_database() + + def test_create_table(self): + with self.assertRaises(ServerError): + self.database.create_table(TemporaryTable) + with self.assertRaises(AssertionError): + self.database.create_table(TemporaryTable2) + with in_session(): + self.database.create_table(TemporaryTable) + count = TemporaryTable.objects_in(self.database).count() + self.assertEqual(count, 0) + # Check if temporary table is cleaned up + with self.assertRaises(ServerError): + TemporaryTable.objects_in(self.database).count() + + +class TemporaryTable(TemporaryModel): + date_field = DateField() + uuid = UUIDField() + + engine = Memory() + + +class TemporaryTable2(TemporaryModel): + date_field = DateField() + uuid = UUIDField() + + engine = MergeTree('date_field', ('date_field',)) diff --git a/tests/test_tuple_fields.py b/tests/test_tuple_fields.py new file mode 100644 index 0000000..b3cca61 --- /dev/null +++ b/tests/test_tuple_fields.py @@ -0,0 +1,63 @@ +import unittest +from datetime import date + +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import TupleField, DateField, StringField, Int32Field, ArrayField +from clickhouse_orm.engines import MergeTree + + +class TupleFieldsTest(unittest.TestCase): + + def setUp(self): + self.database = Database('test-db', log_statements=True) + self.database.create_table(ModelWithTuple) + + def tearDown(self): + self.database.drop_database() + + def test_insert_and_select(self): + instance = ModelWithTuple( + date_field='2016-08-30', + tuple_str=['goodbye,', 'cruel'], + tuple_date=['2010-01-01', '2020-01-01'], + ) + self.database.insert([instance]) + query = 'SELECT * from $db.modelwithtuple ORDER BY date_field' + for model_cls in (ModelWithTuple, None): + results = list(self.database.select(query, model_cls)) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].tuple_str, instance.tuple_str) + self.assertEqual(results[0].tuple_int, instance.tuple_int) + self.assertEqual(results[0].tuple_date, instance.tuple_date) + + def test_conversion(self): + instance = ModelWithTuple( + tuple_int=('1', '2'), + tuple_date=['2010-01-01', '2020-01-01'] + ) + self.assertEqual(instance.tuple_str, ('', '')) + self.assertEqual(instance.tuple_int, (1, 2)) + self.assertEqual(instance.tuple_date, (date(2010, 1, 1), date(2020, 1, 1))) + + def test_assignment_error(self): + instance = ModelWithTuple() + for value in (7, 'x', [date.today()], ['aaa'], [None]): + with self.assertRaises(ValueError): + instance.tuple_int = value + + def test_invalid_inner_field(self): + for x in ([('a', DateField)], [('b', None)], [('c', "")], [('d', ArrayField(StringField()))]): + with self.assertRaises(AssertionError): + TupleField(x) + + +class ModelWithTuple(Model): + + date_field = DateField() + tuple_str = TupleField([('a', StringField()), ('b', StringField())]) + tuple_int = TupleField([('a', Int32Field()), ('b', Int32Field())]) + tuple_date = TupleField([('a', DateField()), ('b', DateField())]) + tuple_mix = TupleField([('a', StringField()), ('b', Int32Field()), ('c', DateField())]) + + engine = MergeTree('date_field', ('date_field',))