From 0e9dea5bcb9a0ebda66d39826b0c046f402fef42 Mon Sep 17 00:00:00 2001 From: olliemath Date: Mon, 16 Aug 2021 09:35:59 +0100 Subject: [PATCH] Chore: update scripts/docs --- docs/class_reference.md | 391 ++++++++++++++++++++++++++++---- docs/toc.md | 14 +- pyproject.toml | 1 - scripts/docs2html.sh | 2 +- scripts/generate_all.sh | 3 +- scripts/generate_ref.py | 77 ++++--- scripts/generate_toc.sh | 4 +- scripts/html_to_markdown_toc.py | 17 +- scripts/test_python3.sh | 11 - setup.cfg | 1 - 10 files changed, 409 insertions(+), 112 deletions(-) delete mode 100755 scripts/test_python3.sh diff --git a/docs/class_reference.md b/docs/class_reference.md index 3d91cdc..490db16 100644 --- a/docs/class_reference.md +++ b/docs/class_reference.md @@ -2,7 +2,7 @@ Class Reference =============== clickhouse_orm.database ----------------------------- +----------------------- ### Database @@ -153,7 +153,7 @@ Extends Exception Raised when a database operation fails. clickhouse_orm.models --------------------------- +--------------------- ### Model @@ -812,7 +812,7 @@ separated by non-alphanumeric characters. clickhouse_orm.fields --------------------------- +--------------------- ### ArrayField @@ -1047,7 +1047,7 @@ Extends Field clickhouse_orm.engines ---------------------------- +---------------------- ### Engine @@ -1141,7 +1141,7 @@ Extends MergeTree clickhouse_orm.query -------------------------- +-------------------- ### QuerySet @@ -1444,7 +1444,7 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier clickhouse_orm.funcs -------------------------- +-------------------- ### F @@ -2012,7 +2012,7 @@ Initializer. #### floor(n=None) -#### formatDateTime(format, timezone="") +#### formatDateTime(format, timezone=NO_VALUE) #### gcd(b) @@ -2804,13 +2804,13 @@ Initializer. #### toDateTimeOrZero() -#### toDayOfMonth() +#### toDayOfMonth(timezone=NO_VALUE) -#### toDayOfWeek() +#### toDayOfWeek(timezone=NO_VALUE) -#### toDayOfYear() +#### toDayOfYear(timezone=NO_VALUE) #### toDecimal128(**kwargs) @@ -2861,7 +2861,7 @@ Initializer. #### toFloat64OrZero() -#### toHour() +#### toHour(timezone=NO_VALUE) #### toIPv4() @@ -2870,10 +2870,10 @@ Initializer. #### toIPv6() -#### toISOWeek(timezone="") +#### toISOWeek(timezone=NO_VALUE) -#### toISOYear(timezone="") +#### toISOYear(timezone=NO_VALUE) #### toInt16(**kwargs) @@ -2936,73 +2936,73 @@ Initializer. #### toIntervalYear() -#### toMinute() +#### toMinute(timezone=NO_VALUE) -#### toMonday() +#### toMonday(timezone=NO_VALUE) -#### toMonth() +#### toMonth(timezone=NO_VALUE) -#### toQuarter(timezone="") +#### toQuarter(timezone=NO_VALUE) -#### toRelativeDayNum(timezone="") +#### toRelativeDayNum(timezone=NO_VALUE) -#### toRelativeHourNum(timezone="") +#### toRelativeHourNum(timezone=NO_VALUE) -#### toRelativeMinuteNum(timezone="") +#### toRelativeMinuteNum(timezone=NO_VALUE) -#### toRelativeMonthNum(timezone="") +#### toRelativeMonthNum(timezone=NO_VALUE) -#### toRelativeSecondNum(timezone="") +#### toRelativeSecondNum(timezone=NO_VALUE) -#### toRelativeWeekNum(timezone="") +#### toRelativeWeekNum(timezone=NO_VALUE) -#### toRelativeYearNum(timezone="") +#### toRelativeYearNum(timezone=NO_VALUE) -#### toSecond() +#### toSecond(timezone=NO_VALUE) -#### toStartOfDay() +#### toStartOfDay(timezone=NO_VALUE) -#### toStartOfFifteenMinutes() +#### toStartOfFifteenMinutes(timezone=NO_VALUE) -#### toStartOfFiveMinute() +#### toStartOfFiveMinute(timezone=NO_VALUE) -#### toStartOfHour() +#### toStartOfHour(timezone=NO_VALUE) -#### toStartOfISOYear() +#### toStartOfISOYear(timezone=NO_VALUE) -#### toStartOfMinute() +#### toStartOfMinute(timezone=NO_VALUE) -#### toStartOfMonth() +#### toStartOfMonth(timezone=NO_VALUE) -#### toStartOfQuarter() +#### toStartOfQuarter(timezone=NO_VALUE) -#### toStartOfTenMinutes() +#### toStartOfTenMinutes(timezone=NO_VALUE) -#### toStartOfWeek(mode=0) +#### toStartOfWeek(timezone=NO_VALUE) -#### toStartOfYear() +#### toStartOfYear(timezone=NO_VALUE) #### toString() @@ -3011,7 +3011,7 @@ Initializer. #### toStringCutToZero() -#### toTime(timezone="") +#### toTime(timezone=NO_VALUE) #### toTimeZone(timezone) @@ -3056,22 +3056,22 @@ Initializer. #### toUUID() -#### toUnixTimestamp(timezone="") +#### toUnixTimestamp(timezone=NO_VALUE) -#### toWeek(mode=0, timezone="") +#### toWeek(mode=0, timezone=NO_VALUE) -#### toYYYYMM(timezone="") +#### toYYYYMM(timezone=NO_VALUE) -#### toYYYYMMDD(timezone="") +#### toYYYYMMDD(timezone=NO_VALUE) -#### toYYYYMMDDhhmmss(timezone="") +#### toYYYYMMDDhhmmss(timezone=NO_VALUE) -#### toYear() +#### toYear(timezone=NO_VALUE) #### to_sql(*args) @@ -3144,3 +3144,308 @@ For other functions: #### uniqExact(**kwargs) +#### uniqExactIf(*args) + + +#### uniqExactOrDefault() + + +#### uniqExactOrDefaultIf(*args) + + +#### uniqExactOrNull() + + +#### uniqExactOrNullIf(*args) + + +#### uniqHLL12(**kwargs) + + +#### uniqHLL12If(*args) + + +#### uniqHLL12OrDefault() + + +#### uniqHLL12OrDefaultIf(*args) + + +#### uniqHLL12OrNull() + + +#### uniqHLL12OrNullIf(*args) + + +#### uniqIf(*args) + + +#### uniqOrDefault() + + +#### uniqOrDefaultIf(*args) + + +#### uniqOrNull() + + +#### uniqOrNullIf(*args) + + +#### upper(**kwargs) + + +#### upperUTF8() + + +#### varPop(**kwargs) + + +#### varPopIf(cond) + + +#### varPopOrDefault() + + +#### varPopOrDefaultIf(cond) + + +#### varPopOrNull() + + +#### varPopOrNullIf(cond) + + +#### varSamp(**kwargs) + + +#### varSampIf(cond) + + +#### varSampOrDefault() + + +#### varSampOrDefaultIf(cond) + + +#### varSampOrNull() + + +#### varSampOrNullIf(cond) + + +#### xxHash32() + + +#### xxHash64() + + +#### yesterday() + + +clickhouse_orm.system_models +---------------------------- + +### SystemPart + +Extends Model + + +Contains information about parts of a table in the MergeTree family. +This model operates only fields, described in the reference. Other fields are ignored. +https://clickhouse.tech/docs/en/system_tables/system.parts/ + +#### SystemPart(**kwargs) + + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### attach(settings=None) + + + Add a new part or partition from the 'detached' directory to the table. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.create_table_sql(db) + + +Returns the SQL statement for creating a table for this model. + + +#### detach(settings=None) + + +Move a partition to the 'detached' directory and forget it. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### drop(settings=None) + + +Delete a partition + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.drop_table_sql(db) + + +Returns the SQL command for deleting this model's table. + + +#### fetch(zookeeper_path, settings=None) + + +Download a partition from another server. + +- `zookeeper_path`: Path in zookeeper to fetch from +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.fields(writable=False) + + +Returns an `OrderedDict` of the model's fields (from name to `Field` instance). +If `writable` is true, only writable fields are included. +Callers should not modify the dictionary. + + +#### freeze(settings=None) + + +Create a backup of a partition. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None) + + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones. +- `database`: if given, sets the database that this instance belongs to. + + +#### SystemPart.get(database, conditions="") + + +Get all data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database condition is added automatically + +Returns: A list of SystemPart objects + + +#### SystemPart.get_active(database, conditions="") + + +Gets active data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database and active conditions are added automatically + +Returns: A list of SystemPart objects + + +#### get_database() + + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + + +Gets a `Field` instance given its name, or `None` if not found. + + +#### SystemPart.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + +#### SystemPart.is_read_only() + + +Returns true if the model is marked as read only. + + +#### SystemPart.is_system_model() + + +Returns true if the model represents a system table. + + +#### SystemPart.objects_in(database) + + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### SystemPart.table_name() + + +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + +#### to_dict(include_readonly=True, field_names=None) + + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +#### to_tsv(include_readonly=True) + + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + diff --git a/docs/toc.md b/docs/toc.md index e973772..57b7f02 100644 --- a/docs/toc.md +++ b/docs/toc.md @@ -78,17 +78,17 @@ * [Tests](contributing.md#tests) * [Class Reference](class_reference.md#class-reference) - * [clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase) + * [clickhouse_orm.database](class_reference.md#clickhouse_ormdatabase) * [Database](class_reference.md#database) * [DatabaseException](class_reference.md#databaseexception) - * [clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels) + * [clickhouse_orm.models](class_reference.md#clickhouse_ormmodels) * [Model](class_reference.md#model) * [BufferModel](class_reference.md#buffermodel) * [MergeModel](class_reference.md#mergemodel) * [DistributedModel](class_reference.md#distributedmodel) * [Constraint](class_reference.md#constraint) * [Index](class_reference.md#index) - * [clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields) + * [clickhouse_orm.fields](class_reference.md#clickhouse_ormfields) * [ArrayField](class_reference.md#arrayfield) * [BaseEnumField](class_reference.md#baseenumfield) * [BaseFloatField](class_reference.md#basefloatfield) @@ -120,7 +120,7 @@ * [UInt64Field](class_reference.md#uint64field) * [UInt8Field](class_reference.md#uint8field) * [UUIDField](class_reference.md#uuidfield) - * [clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines) + * [clickhouse_orm.engines](class_reference.md#clickhouse_ormengines) * [Engine](class_reference.md#engine) * [TinyLog](class_reference.md#tinylog) * [Log](class_reference.md#log) @@ -132,10 +132,12 @@ * [CollapsingMergeTree](class_reference.md#collapsingmergetree) * [SummingMergeTree](class_reference.md#summingmergetree) * [ReplacingMergeTree](class_reference.md#replacingmergetree) - * [clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery) + * [clickhouse_orm.query](class_reference.md#clickhouse_ormquery) * [QuerySet](class_reference.md#queryset) * [AggregateQuerySet](class_reference.md#aggregatequeryset) * [Q](class_reference.md#q) - * [clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs) + * [clickhouse_orm.funcs](class_reference.md#clickhouse_ormfuncs) * [F](class_reference.md#f) + * [clickhouse_orm.system_models](class_reference.md#clickhouse_ormsystem_models) + * [SystemPart](class_reference.md#systempart) diff --git a/pyproject.toml b/pyproject.toml index 98859a5..8eb10c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ line-length = 120 extend-exclude = ''' /( | examples - | scripts )/ ''' diff --git a/scripts/docs2html.sh b/scripts/docs2html.sh index 15dc07e..73d6613 100755 --- a/scripts/docs2html.sh +++ b/scripts/docs2html.sh @@ -1,4 +1,4 @@ - +#!/bin/bash mkdir -p ../htmldocs find ./ -iname "*.md" -type f -exec sh -c 'echo "Converting ${0}"; pandoc "${0}" -s -o "../htmldocs/${0%.md}.html"' {} \; diff --git a/scripts/generate_all.sh b/scripts/generate_all.sh index eabf65c..db4a328 100755 --- a/scripts/generate_all.sh +++ b/scripts/generate_all.sh @@ -1,5 +1,6 @@ +#!/bin/bash # Class reference -../bin/python ../scripts/generate_ref.py > class_reference.md +poetry run python ../scripts/generate_ref.py > class_reference.md # Table of contents ../scripts/generate_toc.sh diff --git a/scripts/generate_ref.py b/scripts/generate_ref.py index 16473b4..7d1fe5a 100644 --- a/scripts/generate_ref.py +++ b/scripts/generate_ref.py @@ -1,11 +1,11 @@ - import inspect from collections import namedtuple -DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value') +DefaultArgSpec = namedtuple("DefaultArgSpec", "has_default default_value") + def _get_default_arg(args, defaults, arg_index): - """ Method that determines if an argument has default value or not, + """Method that determines if an argument has default value or not, and if yes what is the default value for the argument :param args: array of arguments, eg: ['first_arg', 'second_arg', 'third_arg'] @@ -25,12 +25,13 @@ def _get_default_arg(args, defaults, arg_index): return DefaultArgSpec(False, None) else: value = defaults[arg_index - args_with_no_defaults] - if (type(value) is str): + if type(value) is str: value = '"%s"' % value return DefaultArgSpec(True, value) + def get_method_sig(method): - """ Given a function, it returns a string that pretty much looks how the + """Given a function, it returns a string that pretty much looks how the function signature would be written in python. :param method: a python method @@ -42,31 +43,37 @@ def get_method_sig(method): # list of defaults are returned in separate array. # eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'], # varargs=None, keywords=None, defaults=(42, 'something')) - argspec = inspect.getargspec(method) - arg_index=0 + argspec = inspect.getfullargspec(method) args = [] # Use the args and defaults array returned by argspec and find out # which arguments has default - for arg in argspec.args: - default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index) + for idx, arg in enumerate(argspec.args): + default_arg = _get_default_arg(argspec.args, argspec.defaults, idx) + if default_arg.has_default: + val = default_arg.default_value + args.append("%s=%s" % (arg, val)) + else: + args.append(arg) + + for idx, arg in enumerate(argspec.kwonlyargs): + default_arg = _get_default_arg(argspec.kwonlyargs, argspec.kwonlydefaults, idx) if default_arg.has_default: val = default_arg.default_value args.append("%s=%s" % (arg, val)) else: args.append(arg) - arg_index += 1 if argspec.varargs: - args.append('*' + argspec.varargs) - if argspec.keywords: - args.append('**' + argspec.keywords) + args.append("*" + argspec.varargs) + if argspec.varkw: + args.append("**" + argspec.varkw) return "%s(%s)" % (method.__name__, ", ".join(args[1:])) def docstring(obj): - doc = (obj.__doc__ or '').rstrip() + doc = (obj.__doc__ or "").rstrip() if doc: - lines = doc.split('\n') + lines = doc.split("\n") # Find the length of the whitespace prefix common to all non-empty lines indentation = min(len(line) - len(line.lstrip()) for line in lines if line.strip()) # Output the lines without the indentation @@ -76,30 +83,30 @@ def docstring(obj): def class_doc(cls, list_methods=True): - bases = ', '.join([b.__name__ for b in cls.__bases__]) - print('###', cls.__name__) + bases = ", ".join([b.__name__ for b in cls.__bases__]) + print("###", cls.__name__) print() - if bases != 'object': - print('Extends', bases) + if bases != "object": + print("Extends", bases) print() docstring(cls) for name, method in inspect.getmembers(cls, lambda m: inspect.ismethod(m) or inspect.isfunction(m)): - if name == '__init__': + if name == "__init__": # Initializer - print('####', get_method_sig(method).replace(name, cls.__name__)) - elif name[0] == '_': + print("####", get_method_sig(method).replace(name, cls.__name__)) + elif name[0] == "_": # Private method continue - elif hasattr(method, '__self__') and method.__self__ == cls: + elif hasattr(method, "__self__") and method.__self__ == cls: # Class method if not list_methods: continue - print('#### %s.%s' % (cls.__name__, get_method_sig(method))) + print("#### %s.%s" % (cls.__name__, get_method_sig(method))) else: # Regular method if not list_methods: continue - print('####', get_method_sig(method)) + print("####", get_method_sig(method)) print() docstring(method) print() @@ -108,7 +115,7 @@ def class_doc(cls, list_methods=True): def module_doc(classes, list_methods=True): mdl = classes[0].__module__ print(mdl) - print('-' * len(mdl)) + print("-" * len(mdl)) print() for cls in classes: class_doc(cls, list_methods) @@ -118,21 +125,17 @@ def all_subclasses(cls): return cls.__subclasses__() + [g for s in cls.__subclasses__() for g in all_subclasses(s)] -if __name__ == '__main__': +if __name__ == "__main__": - from clickhouse_orm import database - from clickhouse_orm import fields - from clickhouse_orm import engines - from clickhouse_orm import models - from clickhouse_orm import query - from clickhouse_orm import funcs - from clickhouse_orm import system_models + from clickhouse_orm import database, engines, fields, funcs, models, query, system_models - print('Class Reference') - print('===============') + print("Class Reference") + print("===============") print() module_doc([database.Database, database.DatabaseException]) - module_doc([models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index]) + module_doc( + [models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index] + ) module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False) module_doc([engines.Engine] + all_subclasses(engines.Engine), False) module_doc([query.QuerySet, query.AggregateQuerySet, query.Q]) diff --git a/scripts/generate_toc.sh b/scripts/generate_toc.sh index a77aaaa..bc0db33 100755 --- a/scripts/generate_toc.sh +++ b/scripts/generate_toc.sh @@ -1,7 +1,7 @@ - +#!/bin/bash generate_one() { # Converts Markdown to HTML using Pandoc, and then extracts the header tags - pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md + pandoc "$1" | poetry run python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md } printf "# Table of Contents\n\n" > toc.md diff --git a/scripts/html_to_markdown_toc.py b/scripts/html_to_markdown_toc.py index 552137f..956732c 100644 --- a/scripts/html_to_markdown_toc.py +++ b/scripts/html_to_markdown_toc.py @@ -1,14 +1,13 @@ -from html.parser import HTMLParser import sys +from html.parser import HTMLParser - -HEADER_TAGS = ('h1', 'h2', 'h3') +HEADER_TAGS = ("h1", "h2", "h3") class HeadersToMarkdownParser(HTMLParser): inside = None - text = '' + text = "" def handle_starttag(self, tag, attrs): if tag.lower() in HEADER_TAGS: @@ -16,11 +15,11 @@ class HeadersToMarkdownParser(HTMLParser): def handle_endtag(self, tag): if tag.lower() in HEADER_TAGS: - indent = ' ' * int(self.inside[1]) - fragment = self.text.lower().replace(' ', '-').replace('.', '') - print('%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment)) + indent = " " * int(self.inside[1]) + fragment = self.text.lower().replace(" ", "-").replace(".", "") + print("%s* [%s](%s#%s)" % (indent, self.text, sys.argv[1], fragment)) self.inside = None - self.text = '' + self.text = "" def handle_data(self, data): if self.inside: @@ -28,4 +27,4 @@ class HeadersToMarkdownParser(HTMLParser): HeadersToMarkdownParser().feed(sys.stdin.read()) -print('') +print("") diff --git a/scripts/test_python3.sh b/scripts/test_python3.sh deleted file mode 100755 index 455d5b7..0000000 --- a/scripts/test_python3.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -cd /tmp -rm -rf /tmp/orm_env* -virtualenv -p python3 /tmp/orm_env -cd /tmp/orm_env -source bin/activate -pip install infi.projector -git clone https://github.com/Infinidat/clickhouse_orm.git -cd clickhouse_orm -projector devenv build -bin/nosetests diff --git a/setup.cfg b/setup.cfg index 48f3a58..ab5c925 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,4 +18,3 @@ ignore = exclude = tests/sample_migrations examples - scripts