Chore: update scripts/docs

This commit is contained in:
olliemath 2021-08-16 09:35:59 +01:00
parent 19f89e0efe
commit 0e9dea5bcb
10 changed files with 409 additions and 112 deletions

View File

@ -2,7 +2,7 @@ Class Reference
===============
clickhouse_orm.database
----------------------------
-----------------------
### Database
@ -153,7 +153,7 @@ Extends Exception
Raised when a database operation fails.
clickhouse_orm.models
--------------------------
---------------------
### Model
@ -812,7 +812,7 @@ separated by non-alphanumeric characters.
clickhouse_orm.fields
--------------------------
---------------------
### ArrayField
@ -1047,7 +1047,7 @@ Extends Field
clickhouse_orm.engines
---------------------------
----------------------
### Engine
@ -1141,7 +1141,7 @@ Extends MergeTree
clickhouse_orm.query
-------------------------
--------------------
### QuerySet
@ -1444,7 +1444,7 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier
clickhouse_orm.funcs
-------------------------
--------------------
### F
@ -2012,7 +2012,7 @@ Initializer.
#### floor(n=None)
#### formatDateTime(format, timezone="")
#### formatDateTime(format, timezone=NO_VALUE)
#### gcd(b)
@ -2804,13 +2804,13 @@ Initializer.
#### toDateTimeOrZero()
#### toDayOfMonth()
#### toDayOfMonth(timezone=NO_VALUE)
#### toDayOfWeek()
#### toDayOfWeek(timezone=NO_VALUE)
#### toDayOfYear()
#### toDayOfYear(timezone=NO_VALUE)
#### toDecimal128(**kwargs)
@ -2861,7 +2861,7 @@ Initializer.
#### toFloat64OrZero()
#### toHour()
#### toHour(timezone=NO_VALUE)
#### toIPv4()
@ -2870,10 +2870,10 @@ Initializer.
#### toIPv6()
#### toISOWeek(timezone="")
#### toISOWeek(timezone=NO_VALUE)
#### toISOYear(timezone="")
#### toISOYear(timezone=NO_VALUE)
#### toInt16(**kwargs)
@ -2936,73 +2936,73 @@ Initializer.
#### toIntervalYear()
#### toMinute()
#### toMinute(timezone=NO_VALUE)
#### toMonday()
#### toMonday(timezone=NO_VALUE)
#### toMonth()
#### toMonth(timezone=NO_VALUE)
#### toQuarter(timezone="")
#### toQuarter(timezone=NO_VALUE)
#### toRelativeDayNum(timezone="")
#### toRelativeDayNum(timezone=NO_VALUE)
#### toRelativeHourNum(timezone="")
#### toRelativeHourNum(timezone=NO_VALUE)
#### toRelativeMinuteNum(timezone="")
#### toRelativeMinuteNum(timezone=NO_VALUE)
#### toRelativeMonthNum(timezone="")
#### toRelativeMonthNum(timezone=NO_VALUE)
#### toRelativeSecondNum(timezone="")
#### toRelativeSecondNum(timezone=NO_VALUE)
#### toRelativeWeekNum(timezone="")
#### toRelativeWeekNum(timezone=NO_VALUE)
#### toRelativeYearNum(timezone="")
#### toRelativeYearNum(timezone=NO_VALUE)
#### toSecond()
#### toSecond(timezone=NO_VALUE)
#### toStartOfDay()
#### toStartOfDay(timezone=NO_VALUE)
#### toStartOfFifteenMinutes()
#### toStartOfFifteenMinutes(timezone=NO_VALUE)
#### toStartOfFiveMinute()
#### toStartOfFiveMinute(timezone=NO_VALUE)
#### toStartOfHour()
#### toStartOfHour(timezone=NO_VALUE)
#### toStartOfISOYear()
#### toStartOfISOYear(timezone=NO_VALUE)
#### toStartOfMinute()
#### toStartOfMinute(timezone=NO_VALUE)
#### toStartOfMonth()
#### toStartOfMonth(timezone=NO_VALUE)
#### toStartOfQuarter()
#### toStartOfQuarter(timezone=NO_VALUE)
#### toStartOfTenMinutes()
#### toStartOfTenMinutes(timezone=NO_VALUE)
#### toStartOfWeek(mode=0)
#### toStartOfWeek(timezone=NO_VALUE)
#### toStartOfYear()
#### toStartOfYear(timezone=NO_VALUE)
#### toString()
@ -3011,7 +3011,7 @@ Initializer.
#### toStringCutToZero()
#### toTime(timezone="")
#### toTime(timezone=NO_VALUE)
#### toTimeZone(timezone)
@ -3056,22 +3056,22 @@ Initializer.
#### toUUID()
#### toUnixTimestamp(timezone="")
#### toUnixTimestamp(timezone=NO_VALUE)
#### toWeek(mode=0, timezone="")
#### toWeek(mode=0, timezone=NO_VALUE)
#### toYYYYMM(timezone="")
#### toYYYYMM(timezone=NO_VALUE)
#### toYYYYMMDD(timezone="")
#### toYYYYMMDD(timezone=NO_VALUE)
#### toYYYYMMDDhhmmss(timezone="")
#### toYYYYMMDDhhmmss(timezone=NO_VALUE)
#### toYear()
#### toYear(timezone=NO_VALUE)
#### to_sql(*args)
@ -3144,3 +3144,308 @@ For other functions:
#### uniqExact(**kwargs)
#### uniqExactIf(*args)
#### uniqExactOrDefault()
#### uniqExactOrDefaultIf(*args)
#### uniqExactOrNull()
#### uniqExactOrNullIf(*args)
#### uniqHLL12(**kwargs)
#### uniqHLL12If(*args)
#### uniqHLL12OrDefault()
#### uniqHLL12OrDefaultIf(*args)
#### uniqHLL12OrNull()
#### uniqHLL12OrNullIf(*args)
#### uniqIf(*args)
#### uniqOrDefault()
#### uniqOrDefaultIf(*args)
#### uniqOrNull()
#### uniqOrNullIf(*args)
#### upper(**kwargs)
#### upperUTF8()
#### varPop(**kwargs)
#### varPopIf(cond)
#### varPopOrDefault()
#### varPopOrDefaultIf(cond)
#### varPopOrNull()
#### varPopOrNullIf(cond)
#### varSamp(**kwargs)
#### varSampIf(cond)
#### varSampOrDefault()
#### varSampOrDefaultIf(cond)
#### varSampOrNull()
#### varSampOrNullIf(cond)
#### xxHash32()
#### xxHash64()
#### yesterday()
clickhouse_orm.system_models
----------------------------
### SystemPart
Extends Model
Contains information about parts of a table in the MergeTree family.
This model operates only fields, described in the reference. Other fields are ignored.
https://clickhouse.tech/docs/en/system_tables/system.parts/
#### SystemPart(**kwargs)
Creates a model instance, using keyword arguments as field values.
Since values are immediately converted to their Pythonic type,
invalid values will cause a `ValueError` to be raised.
Unrecognized field names will cause an `AttributeError`.
#### attach(settings=None)
Add a new part or partition from the 'detached' directory to the table.
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.create_table_sql(db)
Returns the SQL statement for creating a table for this model.
#### detach(settings=None)
Move a partition to the 'detached' directory and forget it.
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### drop(settings=None)
Delete a partition
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.drop_table_sql(db)
Returns the SQL command for deleting this model's table.
#### fetch(zookeeper_path, settings=None)
Download a partition from another server.
- `zookeeper_path`: Path in zookeeper to fetch from
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.fields(writable=False)
Returns an `OrderedDict` of the model's fields (from name to `Field` instance).
If `writable` is true, only writable fields are included.
Callers should not modify the dictionary.
#### freeze(settings=None)
Create a backup of a partition.
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None)
Create a model instance from a tab-separated line. The line may or may not include a newline.
The `field_names` list must match the fields defined in the model, but does not have to include all of them.
- `line`: the TSV-formatted data.
- `field_names`: names of the model fields in the data.
- `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones.
- `database`: if given, sets the database that this instance belongs to.
#### SystemPart.get(database, conditions="")
Get all data from system.parts table
- `database`: A database object to fetch data from.
- `conditions`: WHERE clause conditions. Database condition is added automatically
Returns: A list of SystemPart objects
#### SystemPart.get_active(database, conditions="")
Gets active data from system.parts table
- `database`: A database object to fetch data from.
- `conditions`: WHERE clause conditions. Database and active conditions are added automatically
Returns: A list of SystemPart objects
#### get_database()
Gets the `Database` that this model instance belongs to.
Returns `None` unless the instance was read from the database or written to it.
#### get_field(name)
Gets a `Field` instance given its name, or `None` if not found.
#### SystemPart.has_funcs_as_defaults()
Return True if some of the model's fields use a function expression
as a default value. This requires special handling when inserting instances.
#### SystemPart.is_read_only()
Returns true if the model is marked as read only.
#### SystemPart.is_system_model()
Returns true if the model represents a system table.
#### SystemPart.objects_in(database)
Returns a `QuerySet` for selecting instances of this model class.
#### set_database(db)
Sets the `Database` that this model instance belongs to.
This is done automatically when the instance is read from the database or written to it.
#### SystemPart.table_name()
#### to_db_string()
Returns the instance as a bytestring ready to be inserted into the database.
#### to_dict(include_readonly=True, field_names=None)
Returns the instance's column values as a dict.
- `include_readonly`: if false, returns only fields that can be inserted into database.
- `field_names`: an iterable of field names to return (optional)
#### to_tskv(include_readonly=True)
Returns the instance's column keys and values as a tab-separated line. A newline is not included.
Fields that were not assigned a value are omitted.
- `include_readonly`: if false, returns only fields that can be inserted into database.
#### to_tsv(include_readonly=True)
Returns the instance's column values as a tab-separated line. A newline is not included.
- `include_readonly`: if false, returns only fields that can be inserted into database.

View File

@ -78,17 +78,17 @@
* [Tests](contributing.md#tests)
* [Class Reference](class_reference.md#class-reference)
* [clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase)
* [clickhouse_orm.database](class_reference.md#clickhouse_ormdatabase)
* [Database](class_reference.md#database)
* [DatabaseException](class_reference.md#databaseexception)
* [clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels)
* [clickhouse_orm.models](class_reference.md#clickhouse_ormmodels)
* [Model](class_reference.md#model)
* [BufferModel](class_reference.md#buffermodel)
* [MergeModel](class_reference.md#mergemodel)
* [DistributedModel](class_reference.md#distributedmodel)
* [Constraint](class_reference.md#constraint)
* [Index](class_reference.md#index)
* [clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields)
* [clickhouse_orm.fields](class_reference.md#clickhouse_ormfields)
* [ArrayField](class_reference.md#arrayfield)
* [BaseEnumField](class_reference.md#baseenumfield)
* [BaseFloatField](class_reference.md#basefloatfield)
@ -120,7 +120,7 @@
* [UInt64Field](class_reference.md#uint64field)
* [UInt8Field](class_reference.md#uint8field)
* [UUIDField](class_reference.md#uuidfield)
* [clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines)
* [clickhouse_orm.engines](class_reference.md#clickhouse_ormengines)
* [Engine](class_reference.md#engine)
* [TinyLog](class_reference.md#tinylog)
* [Log](class_reference.md#log)
@ -132,10 +132,12 @@
* [CollapsingMergeTree](class_reference.md#collapsingmergetree)
* [SummingMergeTree](class_reference.md#summingmergetree)
* [ReplacingMergeTree](class_reference.md#replacingmergetree)
* [clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery)
* [clickhouse_orm.query](class_reference.md#clickhouse_ormquery)
* [QuerySet](class_reference.md#queryset)
* [AggregateQuerySet](class_reference.md#aggregatequeryset)
* [Q](class_reference.md#q)
* [clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs)
* [clickhouse_orm.funcs](class_reference.md#clickhouse_ormfuncs)
* [F](class_reference.md#f)
* [clickhouse_orm.system_models](class_reference.md#clickhouse_ormsystem_models)
* [SystemPart](class_reference.md#systempart)

View File

@ -3,7 +3,6 @@ line-length = 120
extend-exclude = '''
/(
| examples
| scripts
)/
'''

View File

@ -1,4 +1,4 @@
#!/bin/bash
mkdir -p ../htmldocs
find ./ -iname "*.md" -type f -exec sh -c 'echo "Converting ${0}"; pandoc "${0}" -s -o "../htmldocs/${0%.md}.html"' {} \;

View File

@ -1,5 +1,6 @@
#!/bin/bash
# Class reference
../bin/python ../scripts/generate_ref.py > class_reference.md
poetry run python ../scripts/generate_ref.py > class_reference.md
# Table of contents
../scripts/generate_toc.sh

View File

@ -1,11 +1,11 @@
import inspect
from collections import namedtuple
DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value')
DefaultArgSpec = namedtuple("DefaultArgSpec", "has_default default_value")
def _get_default_arg(args, defaults, arg_index):
""" Method that determines if an argument has default value or not,
"""Method that determines if an argument has default value or not,
and if yes what is the default value for the argument
:param args: array of arguments, eg: ['first_arg', 'second_arg', 'third_arg']
@ -25,12 +25,13 @@ def _get_default_arg(args, defaults, arg_index):
return DefaultArgSpec(False, None)
else:
value = defaults[arg_index - args_with_no_defaults]
if (type(value) is str):
if type(value) is str:
value = '"%s"' % value
return DefaultArgSpec(True, value)
def get_method_sig(method):
""" Given a function, it returns a string that pretty much looks how the
"""Given a function, it returns a string that pretty much looks how the
function signature would be written in python.
:param method: a python method
@ -42,31 +43,37 @@ def get_method_sig(method):
# list of defaults are returned in separate array.
# eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'],
# varargs=None, keywords=None, defaults=(42, 'something'))
argspec = inspect.getargspec(method)
arg_index=0
argspec = inspect.getfullargspec(method)
args = []
# Use the args and defaults array returned by argspec and find out
# which arguments has default
for arg in argspec.args:
default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index)
for idx, arg in enumerate(argspec.args):
default_arg = _get_default_arg(argspec.args, argspec.defaults, idx)
if default_arg.has_default:
val = default_arg.default_value
args.append("%s=%s" % (arg, val))
else:
args.append(arg)
for idx, arg in enumerate(argspec.kwonlyargs):
default_arg = _get_default_arg(argspec.kwonlyargs, argspec.kwonlydefaults, idx)
if default_arg.has_default:
val = default_arg.default_value
args.append("%s=%s" % (arg, val))
else:
args.append(arg)
arg_index += 1
if argspec.varargs:
args.append('*' + argspec.varargs)
if argspec.keywords:
args.append('**' + argspec.keywords)
args.append("*" + argspec.varargs)
if argspec.varkw:
args.append("**" + argspec.varkw)
return "%s(%s)" % (method.__name__, ", ".join(args[1:]))
def docstring(obj):
doc = (obj.__doc__ or '').rstrip()
doc = (obj.__doc__ or "").rstrip()
if doc:
lines = doc.split('\n')
lines = doc.split("\n")
# Find the length of the whitespace prefix common to all non-empty lines
indentation = min(len(line) - len(line.lstrip()) for line in lines if line.strip())
# Output the lines without the indentation
@ -76,30 +83,30 @@ def docstring(obj):
def class_doc(cls, list_methods=True):
bases = ', '.join([b.__name__ for b in cls.__bases__])
print('###', cls.__name__)
bases = ", ".join([b.__name__ for b in cls.__bases__])
print("###", cls.__name__)
print()
if bases != 'object':
print('Extends', bases)
if bases != "object":
print("Extends", bases)
print()
docstring(cls)
for name, method in inspect.getmembers(cls, lambda m: inspect.ismethod(m) or inspect.isfunction(m)):
if name == '__init__':
if name == "__init__":
# Initializer
print('####', get_method_sig(method).replace(name, cls.__name__))
elif name[0] == '_':
print("####", get_method_sig(method).replace(name, cls.__name__))
elif name[0] == "_":
# Private method
continue
elif hasattr(method, '__self__') and method.__self__ == cls:
elif hasattr(method, "__self__") and method.__self__ == cls:
# Class method
if not list_methods:
continue
print('#### %s.%s' % (cls.__name__, get_method_sig(method)))
print("#### %s.%s" % (cls.__name__, get_method_sig(method)))
else:
# Regular method
if not list_methods:
continue
print('####', get_method_sig(method))
print("####", get_method_sig(method))
print()
docstring(method)
print()
@ -108,7 +115,7 @@ def class_doc(cls, list_methods=True):
def module_doc(classes, list_methods=True):
mdl = classes[0].__module__
print(mdl)
print('-' * len(mdl))
print("-" * len(mdl))
print()
for cls in classes:
class_doc(cls, list_methods)
@ -118,21 +125,17 @@ def all_subclasses(cls):
return cls.__subclasses__() + [g for s in cls.__subclasses__() for g in all_subclasses(s)]
if __name__ == '__main__':
if __name__ == "__main__":
from clickhouse_orm import database
from clickhouse_orm import fields
from clickhouse_orm import engines
from clickhouse_orm import models
from clickhouse_orm import query
from clickhouse_orm import funcs
from clickhouse_orm import system_models
from clickhouse_orm import database, engines, fields, funcs, models, query, system_models
print('Class Reference')
print('===============')
print("Class Reference")
print("===============")
print()
module_doc([database.Database, database.DatabaseException])
module_doc([models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index])
module_doc(
[models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index]
)
module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False)
module_doc([engines.Engine] + all_subclasses(engines.Engine), False)
module_doc([query.QuerySet, query.AggregateQuerySet, query.Q])

View File

@ -1,7 +1,7 @@
#!/bin/bash
generate_one() {
# Converts Markdown to HTML using Pandoc, and then extracts the header tags
pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md
pandoc "$1" | poetry run python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md
}
printf "# Table of Contents\n\n" > toc.md

View File

@ -1,14 +1,13 @@
from html.parser import HTMLParser
import sys
from html.parser import HTMLParser
HEADER_TAGS = ('h1', 'h2', 'h3')
HEADER_TAGS = ("h1", "h2", "h3")
class HeadersToMarkdownParser(HTMLParser):
inside = None
text = ''
text = ""
def handle_starttag(self, tag, attrs):
if tag.lower() in HEADER_TAGS:
@ -16,11 +15,11 @@ class HeadersToMarkdownParser(HTMLParser):
def handle_endtag(self, tag):
if tag.lower() in HEADER_TAGS:
indent = ' ' * int(self.inside[1])
fragment = self.text.lower().replace(' ', '-').replace('.', '')
print('%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment))
indent = " " * int(self.inside[1])
fragment = self.text.lower().replace(" ", "-").replace(".", "")
print("%s* [%s](%s#%s)" % (indent, self.text, sys.argv[1], fragment))
self.inside = None
self.text = ''
self.text = ""
def handle_data(self, data):
if self.inside:
@ -28,4 +27,4 @@ class HeadersToMarkdownParser(HTMLParser):
HeadersToMarkdownParser().feed(sys.stdin.read())
print('')
print("")

View File

@ -1,11 +0,0 @@
#!/bin/bash
cd /tmp
rm -rf /tmp/orm_env*
virtualenv -p python3 /tmp/orm_env
cd /tmp/orm_env
source bin/activate
pip install infi.projector
git clone https://github.com/Infinidat/clickhouse_orm.git
cd clickhouse_orm
projector devenv build
bin/nosetests

View File

@ -18,4 +18,3 @@ ignore =
exclude =
tests/sample_migrations
examples
scripts