Chore: update scripts/docs

This commit is contained in:
olliemath 2021-08-16 09:35:59 +01:00
parent 19f89e0efe
commit 0e9dea5bcb
10 changed files with 409 additions and 112 deletions

View File

@ -2,7 +2,7 @@ Class Reference
===============
clickhouse_orm.database
----------------------------
-----------------------
### Database
@ -153,7 +153,7 @@ Extends Exception
Raised when a database operation fails.
clickhouse_orm.models
--------------------------
---------------------
### Model
@ -812,7 +812,7 @@ separated by non-alphanumeric characters.
clickhouse_orm.fields
--------------------------
---------------------
### ArrayField
@ -1047,7 +1047,7 @@ Extends Field
clickhouse_orm.engines
---------------------------
----------------------
### Engine
@ -1141,7 +1141,7 @@ Extends MergeTree
clickhouse_orm.query
-------------------------
--------------------
### QuerySet
@ -1444,7 +1444,7 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier
clickhouse_orm.funcs
-------------------------
--------------------
### F
@ -2012,7 +2012,7 @@ Initializer.
#### floor(n=None)
#### formatDateTime(format, timezone="")
#### formatDateTime(format, timezone=NO_VALUE)
#### gcd(b)
@ -2804,13 +2804,13 @@ Initializer.
#### toDateTimeOrZero()
#### toDayOfMonth()
#### toDayOfMonth(timezone=NO_VALUE)
#### toDayOfWeek()
#### toDayOfWeek(timezone=NO_VALUE)
#### toDayOfYear()
#### toDayOfYear(timezone=NO_VALUE)
#### toDecimal128(**kwargs)
@ -2861,7 +2861,7 @@ Initializer.
#### toFloat64OrZero()
#### toHour()
#### toHour(timezone=NO_VALUE)
#### toIPv4()
@ -2870,10 +2870,10 @@ Initializer.
#### toIPv6()
#### toISOWeek(timezone="")
#### toISOWeek(timezone=NO_VALUE)
#### toISOYear(timezone="")
#### toISOYear(timezone=NO_VALUE)
#### toInt16(**kwargs)
@ -2936,73 +2936,73 @@ Initializer.
#### toIntervalYear()
#### toMinute()
#### toMinute(timezone=NO_VALUE)
#### toMonday()
#### toMonday(timezone=NO_VALUE)
#### toMonth()
#### toMonth(timezone=NO_VALUE)
#### toQuarter(timezone="")
#### toQuarter(timezone=NO_VALUE)
#### toRelativeDayNum(timezone="")
#### toRelativeDayNum(timezone=NO_VALUE)
#### toRelativeHourNum(timezone="")
#### toRelativeHourNum(timezone=NO_VALUE)
#### toRelativeMinuteNum(timezone="")
#### toRelativeMinuteNum(timezone=NO_VALUE)
#### toRelativeMonthNum(timezone="")
#### toRelativeMonthNum(timezone=NO_VALUE)
#### toRelativeSecondNum(timezone="")
#### toRelativeSecondNum(timezone=NO_VALUE)
#### toRelativeWeekNum(timezone="")
#### toRelativeWeekNum(timezone=NO_VALUE)
#### toRelativeYearNum(timezone="")
#### toRelativeYearNum(timezone=NO_VALUE)
#### toSecond()
#### toSecond(timezone=NO_VALUE)
#### toStartOfDay()
#### toStartOfDay(timezone=NO_VALUE)
#### toStartOfFifteenMinutes()
#### toStartOfFifteenMinutes(timezone=NO_VALUE)
#### toStartOfFiveMinute()
#### toStartOfFiveMinute(timezone=NO_VALUE)
#### toStartOfHour()
#### toStartOfHour(timezone=NO_VALUE)
#### toStartOfISOYear()
#### toStartOfISOYear(timezone=NO_VALUE)
#### toStartOfMinute()
#### toStartOfMinute(timezone=NO_VALUE)
#### toStartOfMonth()
#### toStartOfMonth(timezone=NO_VALUE)
#### toStartOfQuarter()
#### toStartOfQuarter(timezone=NO_VALUE)
#### toStartOfTenMinutes()
#### toStartOfTenMinutes(timezone=NO_VALUE)
#### toStartOfWeek(mode=0)
#### toStartOfWeek(timezone=NO_VALUE)
#### toStartOfYear()
#### toStartOfYear(timezone=NO_VALUE)
#### toString()
@ -3011,7 +3011,7 @@ Initializer.
#### toStringCutToZero()
#### toTime(timezone="")
#### toTime(timezone=NO_VALUE)
#### toTimeZone(timezone)
@ -3056,22 +3056,22 @@ Initializer.
#### toUUID()
#### toUnixTimestamp(timezone="")
#### toUnixTimestamp(timezone=NO_VALUE)
#### toWeek(mode=0, timezone="")
#### toWeek(mode=0, timezone=NO_VALUE)
#### toYYYYMM(timezone="")
#### toYYYYMM(timezone=NO_VALUE)
#### toYYYYMMDD(timezone="")
#### toYYYYMMDD(timezone=NO_VALUE)
#### toYYYYMMDDhhmmss(timezone="")
#### toYYYYMMDDhhmmss(timezone=NO_VALUE)
#### toYear()
#### toYear(timezone=NO_VALUE)
#### to_sql(*args)
@ -3144,3 +3144,308 @@ For other functions:
#### uniqExact(**kwargs)
#### uniqExactIf(*args)
#### uniqExactOrDefault()
#### uniqExactOrDefaultIf(*args)
#### uniqExactOrNull()
#### uniqExactOrNullIf(*args)
#### uniqHLL12(**kwargs)
#### uniqHLL12If(*args)
#### uniqHLL12OrDefault()
#### uniqHLL12OrDefaultIf(*args)
#### uniqHLL12OrNull()
#### uniqHLL12OrNullIf(*args)
#### uniqIf(*args)
#### uniqOrDefault()
#### uniqOrDefaultIf(*args)
#### uniqOrNull()
#### uniqOrNullIf(*args)
#### upper(**kwargs)
#### upperUTF8()
#### varPop(**kwargs)
#### varPopIf(cond)
#### varPopOrDefault()
#### varPopOrDefaultIf(cond)
#### varPopOrNull()
#### varPopOrNullIf(cond)
#### varSamp(**kwargs)
#### varSampIf(cond)
#### varSampOrDefault()
#### varSampOrDefaultIf(cond)
#### varSampOrNull()
#### varSampOrNullIf(cond)
#### xxHash32()
#### xxHash64()
#### yesterday()
clickhouse_orm.system_models
----------------------------
### SystemPart
Extends Model
Contains information about parts of a table in the MergeTree family.
This model operates only fields, described in the reference. Other fields are ignored.
https://clickhouse.tech/docs/en/system_tables/system.parts/
#### SystemPart(**kwargs)
Creates a model instance, using keyword arguments as field values.
Since values are immediately converted to their Pythonic type,
invalid values will cause a `ValueError` to be raised.
Unrecognized field names will cause an `AttributeError`.
#### attach(settings=None)
Add a new part or partition from the 'detached' directory to the table.
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.create_table_sql(db)
Returns the SQL statement for creating a table for this model.
#### detach(settings=None)
Move a partition to the 'detached' directory and forget it.
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### drop(settings=None)
Delete a partition
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.drop_table_sql(db)
Returns the SQL command for deleting this model's table.
#### fetch(zookeeper_path, settings=None)
Download a partition from another server.
- `zookeeper_path`: Path in zookeeper to fetch from
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.fields(writable=False)
Returns an `OrderedDict` of the model's fields (from name to `Field` instance).
If `writable` is true, only writable fields are included.
Callers should not modify the dictionary.
#### freeze(settings=None)
Create a backup of a partition.
- `settings`: Settings for executing request to ClickHouse over db.raw() method
Returns: SQL Query
#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None)
Create a model instance from a tab-separated line. The line may or may not include a newline.
The `field_names` list must match the fields defined in the model, but does not have to include all of them.
- `line`: the TSV-formatted data.
- `field_names`: names of the model fields in the data.
- `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones.
- `database`: if given, sets the database that this instance belongs to.
#### SystemPart.get(database, conditions="")
Get all data from system.parts table
- `database`: A database object to fetch data from.
- `conditions`: WHERE clause conditions. Database condition is added automatically
Returns: A list of SystemPart objects
#### SystemPart.get_active(database, conditions="")
Gets active data from system.parts table
- `database`: A database object to fetch data from.
- `conditions`: WHERE clause conditions. Database and active conditions are added automatically
Returns: A list of SystemPart objects
#### get_database()
Gets the `Database` that this model instance belongs to.
Returns `None` unless the instance was read from the database or written to it.
#### get_field(name)
Gets a `Field` instance given its name, or `None` if not found.
#### SystemPart.has_funcs_as_defaults()
Return True if some of the model's fields use a function expression
as a default value. This requires special handling when inserting instances.
#### SystemPart.is_read_only()
Returns true if the model is marked as read only.
#### SystemPart.is_system_model()
Returns true if the model represents a system table.
#### SystemPart.objects_in(database)
Returns a `QuerySet` for selecting instances of this model class.
#### set_database(db)
Sets the `Database` that this model instance belongs to.
This is done automatically when the instance is read from the database or written to it.
#### SystemPart.table_name()
#### to_db_string()
Returns the instance as a bytestring ready to be inserted into the database.
#### to_dict(include_readonly=True, field_names=None)
Returns the instance's column values as a dict.
- `include_readonly`: if false, returns only fields that can be inserted into database.
- `field_names`: an iterable of field names to return (optional)
#### to_tskv(include_readonly=True)
Returns the instance's column keys and values as a tab-separated line. A newline is not included.
Fields that were not assigned a value are omitted.
- `include_readonly`: if false, returns only fields that can be inserted into database.
#### to_tsv(include_readonly=True)
Returns the instance's column values as a tab-separated line. A newline is not included.
- `include_readonly`: if false, returns only fields that can be inserted into database.

View File

@ -78,17 +78,17 @@
* [Tests](contributing.md#tests)
* [Class Reference](class_reference.md#class-reference)
* [clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase)
* [clickhouse_orm.database](class_reference.md#clickhouse_ormdatabase)
* [Database](class_reference.md#database)
* [DatabaseException](class_reference.md#databaseexception)
* [clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels)
* [clickhouse_orm.models](class_reference.md#clickhouse_ormmodels)
* [Model](class_reference.md#model)
* [BufferModel](class_reference.md#buffermodel)
* [MergeModel](class_reference.md#mergemodel)
* [DistributedModel](class_reference.md#distributedmodel)
* [Constraint](class_reference.md#constraint)
* [Index](class_reference.md#index)
* [clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields)
* [clickhouse_orm.fields](class_reference.md#clickhouse_ormfields)
* [ArrayField](class_reference.md#arrayfield)
* [BaseEnumField](class_reference.md#baseenumfield)
* [BaseFloatField](class_reference.md#basefloatfield)
@ -120,7 +120,7 @@
* [UInt64Field](class_reference.md#uint64field)
* [UInt8Field](class_reference.md#uint8field)
* [UUIDField](class_reference.md#uuidfield)
* [clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines)
* [clickhouse_orm.engines](class_reference.md#clickhouse_ormengines)
* [Engine](class_reference.md#engine)
* [TinyLog](class_reference.md#tinylog)
* [Log](class_reference.md#log)
@ -132,10 +132,12 @@
* [CollapsingMergeTree](class_reference.md#collapsingmergetree)
* [SummingMergeTree](class_reference.md#summingmergetree)
* [ReplacingMergeTree](class_reference.md#replacingmergetree)
* [clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery)
* [clickhouse_orm.query](class_reference.md#clickhouse_ormquery)
* [QuerySet](class_reference.md#queryset)
* [AggregateQuerySet](class_reference.md#aggregatequeryset)
* [Q](class_reference.md#q)
* [clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs)
* [clickhouse_orm.funcs](class_reference.md#clickhouse_ormfuncs)
* [F](class_reference.md#f)
* [clickhouse_orm.system_models](class_reference.md#clickhouse_ormsystem_models)
* [SystemPart](class_reference.md#systempart)

View File

@ -3,7 +3,6 @@ line-length = 120
extend-exclude = '''
/(
| examples
| scripts
)/
'''

View File

@ -1,4 +1,4 @@
#!/bin/bash
mkdir -p ../htmldocs
find ./ -iname "*.md" -type f -exec sh -c 'echo "Converting ${0}"; pandoc "${0}" -s -o "../htmldocs/${0%.md}.html"' {} \;

View File

@ -1,5 +1,6 @@
#!/bin/bash
# Class reference
../bin/python ../scripts/generate_ref.py > class_reference.md
poetry run python ../scripts/generate_ref.py > class_reference.md
# Table of contents
../scripts/generate_toc.sh

View File

@ -1,8 +1,8 @@
import inspect
from collections import namedtuple
DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value')
DefaultArgSpec = namedtuple("DefaultArgSpec", "has_default default_value")
def _get_default_arg(args, defaults, arg_index):
"""Method that determines if an argument has default value or not,
@ -25,10 +25,11 @@ def _get_default_arg(args, defaults, arg_index):
return DefaultArgSpec(False, None)
else:
value = defaults[arg_index - args_with_no_defaults]
if (type(value) is str):
if type(value) is str:
value = '"%s"' % value
return DefaultArgSpec(True, value)
def get_method_sig(method):
"""Given a function, it returns a string that pretty much looks how the
function signature would be written in python.
@ -42,31 +43,37 @@ def get_method_sig(method):
# list of defaults are returned in separate array.
# eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'],
# varargs=None, keywords=None, defaults=(42, 'something'))
argspec = inspect.getargspec(method)
arg_index=0
argspec = inspect.getfullargspec(method)
args = []
# Use the args and defaults array returned by argspec and find out
# which arguments has default
for arg in argspec.args:
default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index)
for idx, arg in enumerate(argspec.args):
default_arg = _get_default_arg(argspec.args, argspec.defaults, idx)
if default_arg.has_default:
val = default_arg.default_value
args.append("%s=%s" % (arg, val))
else:
args.append(arg)
for idx, arg in enumerate(argspec.kwonlyargs):
default_arg = _get_default_arg(argspec.kwonlyargs, argspec.kwonlydefaults, idx)
if default_arg.has_default:
val = default_arg.default_value
args.append("%s=%s" % (arg, val))
else:
args.append(arg)
arg_index += 1
if argspec.varargs:
args.append('*' + argspec.varargs)
if argspec.keywords:
args.append('**' + argspec.keywords)
args.append("*" + argspec.varargs)
if argspec.varkw:
args.append("**" + argspec.varkw)
return "%s(%s)" % (method.__name__, ", ".join(args[1:]))
def docstring(obj):
doc = (obj.__doc__ or '').rstrip()
doc = (obj.__doc__ or "").rstrip()
if doc:
lines = doc.split('\n')
lines = doc.split("\n")
# Find the length of the whitespace prefix common to all non-empty lines
indentation = min(len(line) - len(line.lstrip()) for line in lines if line.strip())
# Output the lines without the indentation
@ -76,30 +83,30 @@ def docstring(obj):
def class_doc(cls, list_methods=True):
bases = ', '.join([b.__name__ for b in cls.__bases__])
print('###', cls.__name__)
bases = ", ".join([b.__name__ for b in cls.__bases__])
print("###", cls.__name__)
print()
if bases != 'object':
print('Extends', bases)
if bases != "object":
print("Extends", bases)
print()
docstring(cls)
for name, method in inspect.getmembers(cls, lambda m: inspect.ismethod(m) or inspect.isfunction(m)):
if name == '__init__':
if name == "__init__":
# Initializer
print('####', get_method_sig(method).replace(name, cls.__name__))
elif name[0] == '_':
print("####", get_method_sig(method).replace(name, cls.__name__))
elif name[0] == "_":
# Private method
continue
elif hasattr(method, '__self__') and method.__self__ == cls:
elif hasattr(method, "__self__") and method.__self__ == cls:
# Class method
if not list_methods:
continue
print('#### %s.%s' % (cls.__name__, get_method_sig(method)))
print("#### %s.%s" % (cls.__name__, get_method_sig(method)))
else:
# Regular method
if not list_methods:
continue
print('####', get_method_sig(method))
print("####", get_method_sig(method))
print()
docstring(method)
print()
@ -108,7 +115,7 @@ def class_doc(cls, list_methods=True):
def module_doc(classes, list_methods=True):
mdl = classes[0].__module__
print(mdl)
print('-' * len(mdl))
print("-" * len(mdl))
print()
for cls in classes:
class_doc(cls, list_methods)
@ -118,21 +125,17 @@ def all_subclasses(cls):
return cls.__subclasses__() + [g for s in cls.__subclasses__() for g in all_subclasses(s)]
if __name__ == '__main__':
if __name__ == "__main__":
from clickhouse_orm import database
from clickhouse_orm import fields
from clickhouse_orm import engines
from clickhouse_orm import models
from clickhouse_orm import query
from clickhouse_orm import funcs
from clickhouse_orm import system_models
from clickhouse_orm import database, engines, fields, funcs, models, query, system_models
print('Class Reference')
print('===============')
print("Class Reference")
print("===============")
print()
module_doc([database.Database, database.DatabaseException])
module_doc([models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index])
module_doc(
[models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index]
)
module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False)
module_doc([engines.Engine] + all_subclasses(engines.Engine), False)
module_doc([query.QuerySet, query.AggregateQuerySet, query.Q])

View File

@ -1,7 +1,7 @@
#!/bin/bash
generate_one() {
# Converts Markdown to HTML using Pandoc, and then extracts the header tags
pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md
pandoc "$1" | poetry run python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md
}
printf "# Table of Contents\n\n" > toc.md

View File

@ -1,14 +1,13 @@
from html.parser import HTMLParser
import sys
from html.parser import HTMLParser
HEADER_TAGS = ('h1', 'h2', 'h3')
HEADER_TAGS = ("h1", "h2", "h3")
class HeadersToMarkdownParser(HTMLParser):
inside = None
text = ''
text = ""
def handle_starttag(self, tag, attrs):
if tag.lower() in HEADER_TAGS:
@ -16,11 +15,11 @@ class HeadersToMarkdownParser(HTMLParser):
def handle_endtag(self, tag):
if tag.lower() in HEADER_TAGS:
indent = ' ' * int(self.inside[1])
fragment = self.text.lower().replace(' ', '-').replace('.', '')
print('%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment))
indent = " " * int(self.inside[1])
fragment = self.text.lower().replace(" ", "-").replace(".", "")
print("%s* [%s](%s#%s)" % (indent, self.text, sys.argv[1], fragment))
self.inside = None
self.text = ''
self.text = ""
def handle_data(self, data):
if self.inside:
@ -28,4 +27,4 @@ class HeadersToMarkdownParser(HTMLParser):
HeadersToMarkdownParser().feed(sys.stdin.read())
print('')
print("")

View File

@ -1,11 +0,0 @@
#!/bin/bash
cd /tmp
rm -rf /tmp/orm_env*
virtualenv -p python3 /tmp/orm_env
cd /tmp/orm_env
source bin/activate
pip install infi.projector
git clone https://github.com/Infinidat/clickhouse_orm.git
cd clickhouse_orm
projector devenv build
bin/nosetests

View File

@ -18,4 +18,3 @@ ignore =
exclude =
tests/sample_migrations
examples
scripts