Chore: update scripts/docs

docs/class_reference.md

@@ -2,7 +2,7 @@ Class Reference
 ===============
 
 clickhouse_orm.database
-----------------------------
+-----------------------
 
 ### Database
 
@@ -153,7 +153,7 @@ Extends Exception
 Raised when a database operation fails.
 
 clickhouse_orm.models
---------------------------
+---------------------
 
 ### Model
 
@@ -812,7 +812,7 @@ separated by non-alphanumeric characters.
 
 
 clickhouse_orm.fields
---------------------------
+---------------------
 
 ### ArrayField
 
@@ -1047,7 +1047,7 @@ Extends Field
 
 
 clickhouse_orm.engines
----------------------------
+----------------------
 
 ### Engine
 
@@ -1141,7 +1141,7 @@ Extends MergeTree
 
 
 clickhouse_orm.query
--------------------------
+--------------------
 
 ### QuerySet
 
@@ -1444,7 +1444,7 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier
 
 
 clickhouse_orm.funcs
--------------------------
+--------------------
 
 ### F
 
@@ -2012,7 +2012,7 @@ Initializer.
 #### floor(n=None)
 
 
-#### formatDateTime(format, timezone="")
+#### formatDateTime(format, timezone=NO_VALUE)
 
 
 #### gcd(b)
@@ -2804,13 +2804,13 @@ Initializer.
 #### toDateTimeOrZero()
 
 
-#### toDayOfMonth()
+#### toDayOfMonth(timezone=NO_VALUE)
 
 
-#### toDayOfWeek()
+#### toDayOfWeek(timezone=NO_VALUE)
 
 
-#### toDayOfYear()
+#### toDayOfYear(timezone=NO_VALUE)
 
 
 #### toDecimal128(**kwargs)
@@ -2861,7 +2861,7 @@ Initializer.
 #### toFloat64OrZero()
 
 
-#### toHour()
+#### toHour(timezone=NO_VALUE)
 
 
 #### toIPv4()
@@ -2870,10 +2870,10 @@ Initializer.
 #### toIPv6()
 
 
-#### toISOWeek(timezone="")
+#### toISOWeek(timezone=NO_VALUE)
 
 
-#### toISOYear(timezone="")
+#### toISOYear(timezone=NO_VALUE)
 
 
 #### toInt16(**kwargs)
@@ -2936,73 +2936,73 @@ Initializer.
 #### toIntervalYear()
 
 
-#### toMinute()
+#### toMinute(timezone=NO_VALUE)
 
 
-#### toMonday()
+#### toMonday(timezone=NO_VALUE)
 
 
-#### toMonth()
+#### toMonth(timezone=NO_VALUE)
 
 
-#### toQuarter(timezone="")
+#### toQuarter(timezone=NO_VALUE)
 
 
-#### toRelativeDayNum(timezone="")
+#### toRelativeDayNum(timezone=NO_VALUE)
 
 
-#### toRelativeHourNum(timezone="")
+#### toRelativeHourNum(timezone=NO_VALUE)
 
 
-#### toRelativeMinuteNum(timezone="")
+#### toRelativeMinuteNum(timezone=NO_VALUE)
 
 
-#### toRelativeMonthNum(timezone="")
+#### toRelativeMonthNum(timezone=NO_VALUE)
 
 
-#### toRelativeSecondNum(timezone="")
+#### toRelativeSecondNum(timezone=NO_VALUE)
 
 
-#### toRelativeWeekNum(timezone="")
+#### toRelativeWeekNum(timezone=NO_VALUE)
 
 
-#### toRelativeYearNum(timezone="")
+#### toRelativeYearNum(timezone=NO_VALUE)
 
 
-#### toSecond()
+#### toSecond(timezone=NO_VALUE)
 
 
-#### toStartOfDay()
+#### toStartOfDay(timezone=NO_VALUE)
 
 
-#### toStartOfFifteenMinutes()
+#### toStartOfFifteenMinutes(timezone=NO_VALUE)
 
 
-#### toStartOfFiveMinute()
+#### toStartOfFiveMinute(timezone=NO_VALUE)
 
 
-#### toStartOfHour()
+#### toStartOfHour(timezone=NO_VALUE)
 
 
-#### toStartOfISOYear()
+#### toStartOfISOYear(timezone=NO_VALUE)
 
 
-#### toStartOfMinute()
+#### toStartOfMinute(timezone=NO_VALUE)
 
 
-#### toStartOfMonth()
+#### toStartOfMonth(timezone=NO_VALUE)
 
 
-#### toStartOfQuarter()
+#### toStartOfQuarter(timezone=NO_VALUE)
 
 
-#### toStartOfTenMinutes()
+#### toStartOfTenMinutes(timezone=NO_VALUE)
 
 
-#### toStartOfWeek(mode=0)
+#### toStartOfWeek(timezone=NO_VALUE)
 
 
-#### toStartOfYear()
+#### toStartOfYear(timezone=NO_VALUE)
 
 
 #### toString()
@@ -3011,7 +3011,7 @@ Initializer.
 #### toStringCutToZero()
 
 
-#### toTime(timezone="")
+#### toTime(timezone=NO_VALUE)
 
 
 #### toTimeZone(timezone)
@@ -3056,22 +3056,22 @@ Initializer.
 #### toUUID()
 
 
-#### toUnixTimestamp(timezone="")
+#### toUnixTimestamp(timezone=NO_VALUE)
 
 
-#### toWeek(mode=0, timezone="")
+#### toWeek(mode=0, timezone=NO_VALUE)
 
 
-#### toYYYYMM(timezone="")
+#### toYYYYMM(timezone=NO_VALUE)
 
 
-#### toYYYYMMDD(timezone="")
+#### toYYYYMMDD(timezone=NO_VALUE)
 
 
-#### toYYYYMMDDhhmmss(timezone="")
+#### toYYYYMMDDhhmmss(timezone=NO_VALUE)
 
 
-#### toYear()
+#### toYear(timezone=NO_VALUE)
 
 
 #### to_sql(*args)
@@ -3144,3 +3144,308 @@ For other functions:
 #### uniqExact(**kwargs)
 
 
+#### uniqExactIf(*args)
+
+
+#### uniqExactOrDefault()
+
+
+#### uniqExactOrDefaultIf(*args)
+
+
+#### uniqExactOrNull()
+
+
+#### uniqExactOrNullIf(*args)
+
+
+#### uniqHLL12(**kwargs)
+
+
+#### uniqHLL12If(*args)
+
+
+#### uniqHLL12OrDefault()
+
+
+#### uniqHLL12OrDefaultIf(*args)
+
+
+#### uniqHLL12OrNull()
+
+
+#### uniqHLL12OrNullIf(*args)
+
+
+#### uniqIf(*args)
+
+
+#### uniqOrDefault()
+
+
+#### uniqOrDefaultIf(*args)
+
+
+#### uniqOrNull()
+
+
+#### uniqOrNullIf(*args)
+
+
+#### upper(**kwargs)
+
+
+#### upperUTF8()
+
+
+#### varPop(**kwargs)
+
+
+#### varPopIf(cond)
+
+
+#### varPopOrDefault()
+
+
+#### varPopOrDefaultIf(cond)
+
+
+#### varPopOrNull()
+
+
+#### varPopOrNullIf(cond)
+
+
+#### varSamp(**kwargs)
+
+
+#### varSampIf(cond)
+
+
+#### varSampOrDefault()
+
+
+#### varSampOrDefaultIf(cond)
+
+
+#### varSampOrNull()
+
+
+#### varSampOrNullIf(cond)
+
+
+#### xxHash32()
+
+
+#### xxHash64()
+
+
+#### yesterday()
+
+
+clickhouse_orm.system_models
+----------------------------
+
+### SystemPart
+
+Extends Model
+
+
+Contains information about parts of a table in the MergeTree family.
+This model operates only fields, described in the reference. Other fields are ignored.
+https://clickhouse.tech/docs/en/system_tables/system.parts/
+
+#### SystemPart(**kwargs)
+
+
+Creates a model instance, using keyword arguments as field values.
+Since values are immediately converted to their Pythonic type,
+invalid values will cause a `ValueError` to be raised.
+Unrecognized field names will cause an `AttributeError`.
+
+
+#### attach(settings=None)
+
+
+ Add a new part or partition from the 'detached' directory to the table.
+
+- `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+Returns: SQL Query
+
+
+#### SystemPart.create_table_sql(db)
+
+
+Returns the SQL statement for creating a table for this model.
+
+
+#### detach(settings=None)
+
+
+Move a partition to the 'detached' directory and forget it.
+
+- `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+Returns: SQL Query
+
+
+#### drop(settings=None)
+
+
+Delete a partition
+
+- `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+Returns: SQL Query
+
+
+#### SystemPart.drop_table_sql(db)
+
+
+Returns the SQL command for deleting this model's table.
+
+
+#### fetch(zookeeper_path, settings=None)
+
+
+Download a partition from another server.
+
+- `zookeeper_path`: Path in zookeeper to fetch from
+- `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+Returns: SQL Query
+
+
+#### SystemPart.fields(writable=False)
+
+
+Returns an `OrderedDict` of the model's fields (from name to `Field` instance).
+If `writable` is true, only writable fields are included.
+Callers should not modify the dictionary.
+
+
+#### freeze(settings=None)
+
+
+Create a backup of a partition.
+
+- `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+Returns: SQL Query
+
+
+#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None)
+
+
+Create a model instance from a tab-separated line. The line may or may not include a newline.
+The `field_names` list must match the fields defined in the model, but does not have to include all of them.
+
+- `line`: the TSV-formatted data.
+- `field_names`: names of the model fields in the data.
+- `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones.
+- `database`: if given, sets the database that this instance belongs to.
+
+
+#### SystemPart.get(database, conditions="")
+
+
+Get all data from system.parts table
+
+- `database`: A database object to fetch data from.
+- `conditions`: WHERE clause conditions. Database condition is added automatically
+
+Returns: A list of SystemPart objects
+
+
+#### SystemPart.get_active(database, conditions="")
+
+
+Gets active data from system.parts table
+
+- `database`: A database object to fetch data from.
+- `conditions`: WHERE clause conditions. Database and active conditions are added automatically
+
+Returns: A list of SystemPart objects
+
+
+#### get_database()
+
+
+Gets the `Database` that this model instance belongs to.
+Returns `None` unless the instance was read from the database or written to it.
+
+
+#### get_field(name)
+
+
+Gets a `Field` instance given its name, or `None` if not found.
+
+
+#### SystemPart.has_funcs_as_defaults()
+
+
+Return True if some of the model's fields use a function expression
+as a default value. This requires special handling when inserting instances.
+
+
+#### SystemPart.is_read_only()
+
+
+Returns true if the model is marked as read only.
+
+
+#### SystemPart.is_system_model()
+
+
+Returns true if the model represents a system table.
+
+
+#### SystemPart.objects_in(database)
+
+
+Returns a `QuerySet` for selecting instances of this model class.
+
+
+#### set_database(db)
+
+
+Sets the `Database` that this model instance belongs to.
+This is done automatically when the instance is read from the database or written to it.
+
+
+#### SystemPart.table_name()
+
+
+#### to_db_string()
+
+
+Returns the instance as a bytestring ready to be inserted into the database.
+
+
+#### to_dict(include_readonly=True, field_names=None)
+
+
+Returns the instance's column values as a dict.
+
+- `include_readonly`: if false, returns only fields that can be inserted into database.
+- `field_names`: an iterable of field names to return (optional)
+
+
+#### to_tskv(include_readonly=True)
+
+
+Returns the instance's column keys and values as a tab-separated line. A newline is not included.
+Fields that were not assigned a value are omitted.
+
+- `include_readonly`: if false, returns only fields that can be inserted into database.
+
+
+#### to_tsv(include_readonly=True)
+
+
+Returns the instance's column values as a tab-separated line. A newline is not included.
+
+- `include_readonly`: if false, returns only fields that can be inserted into database.
+
+

docs/toc.md

@@ -78,17 +78,17 @@
       * [Tests](contributing.md#tests)
 
    * [Class Reference](class_reference.md#class-reference)
-      * [clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase)
+      * [clickhouse_orm.database](class_reference.md#clickhouse_ormdatabase)
          * [Database](class_reference.md#database)
          * [DatabaseException](class_reference.md#databaseexception)
-      * [clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels)
+      * [clickhouse_orm.models](class_reference.md#clickhouse_ormmodels)
          * [Model](class_reference.md#model)
          * [BufferModel](class_reference.md#buffermodel)
          * [MergeModel](class_reference.md#mergemodel)
          * [DistributedModel](class_reference.md#distributedmodel)
          * [Constraint](class_reference.md#constraint)
          * [Index](class_reference.md#index)
-      * [clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields)
+      * [clickhouse_orm.fields](class_reference.md#clickhouse_ormfields)
          * [ArrayField](class_reference.md#arrayfield)
          * [BaseEnumField](class_reference.md#baseenumfield)
          * [BaseFloatField](class_reference.md#basefloatfield)
@@ -120,7 +120,7 @@
          * [UInt64Field](class_reference.md#uint64field)
          * [UInt8Field](class_reference.md#uint8field)
          * [UUIDField](class_reference.md#uuidfield)
-      * [clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines)
+      * [clickhouse_orm.engines](class_reference.md#clickhouse_ormengines)
          * [Engine](class_reference.md#engine)
          * [TinyLog](class_reference.md#tinylog)
          * [Log](class_reference.md#log)
@@ -132,10 +132,12 @@
          * [CollapsingMergeTree](class_reference.md#collapsingmergetree)
          * [SummingMergeTree](class_reference.md#summingmergetree)
          * [ReplacingMergeTree](class_reference.md#replacingmergetree)
-      * [clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery)
+      * [clickhouse_orm.query](class_reference.md#clickhouse_ormquery)
          * [QuerySet](class_reference.md#queryset)
          * [AggregateQuerySet](class_reference.md#aggregatequeryset)
          * [Q](class_reference.md#q)
-      * [clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs)
+      * [clickhouse_orm.funcs](class_reference.md#clickhouse_ormfuncs)
          * [F](class_reference.md#f)
+      * [clickhouse_orm.system_models](class_reference.md#clickhouse_ormsystem_models)
+         * [SystemPart](class_reference.md#systempart)
 

pyproject.toml

@@ -3,7 +3,6 @@ line-length = 120
 extend-exclude = '''
 /(
   | examples
-  | scripts
 )/
 '''
 

scripts/docs2html.sh

@@ -1,4 +1,4 @@
-
+#!/bin/bash
 mkdir -p ../htmldocs
 
 find ./ -iname "*.md" -type f -exec sh -c 'echo "Converting ${0}"; pandoc "${0}" -s -o "../htmldocs/${0%.md}.html"' {} \;

scripts/generate_all.sh

@@ -1,5 +1,6 @@
+#!/bin/bash
 # Class reference
-../bin/python ../scripts/generate_ref.py > class_reference.md
+poetry run python ../scripts/generate_ref.py > class_reference.md
 
 # Table of contents
 ../scripts/generate_toc.sh

scripts/generate_ref.py

@@ -1,11 +1,11 @@
-
 import inspect
 from collections import namedtuple
 
-DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value')
+DefaultArgSpec = namedtuple("DefaultArgSpec", "has_default default_value")
+
 
 def _get_default_arg(args, defaults, arg_index):
-    """ Method that determines if an argument has default value or not,
+    """Method that determines if an argument has default value or not,
     and if yes what is the default value for the argument
 
     :param args: array of arguments, eg: ['first_arg', 'second_arg', 'third_arg']
@@ -25,12 +25,13 @@ def _get_default_arg(args, defaults, arg_index):
         return DefaultArgSpec(False, None)
     else:
         value = defaults[arg_index - args_with_no_defaults]
-        if (type(value) is str):
+        if type(value) is str:
             value = '"%s"' % value
         return DefaultArgSpec(True, value)
 
+
 def get_method_sig(method):
-    """ Given a function, it returns a string that pretty much looks how the
+    """Given a function, it returns a string that pretty much looks how the
     function signature would be written in python.
 
     :param method: a python method
@@ -42,31 +43,37 @@ def get_method_sig(method):
     # list of defaults are returned in separate array.
     # eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'],
     # varargs=None, keywords=None, defaults=(42, 'something'))
-    argspec = inspect.getargspec(method)
+    argspec = inspect.getfullargspec(method)
-    arg_index=0
     args = []
 
     # Use the args and defaults array returned by argspec and find out
     # which arguments has default
-    for arg in argspec.args:
+    for idx, arg in enumerate(argspec.args):
-        default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index)
+        default_arg = _get_default_arg(argspec.args, argspec.defaults, idx)
+        if default_arg.has_default:
+            val = default_arg.default_value
+            args.append("%s=%s" % (arg, val))
+        else:
+            args.append(arg)
+
+    for idx, arg in enumerate(argspec.kwonlyargs):
+        default_arg = _get_default_arg(argspec.kwonlyargs, argspec.kwonlydefaults, idx)
         if default_arg.has_default:
             val = default_arg.default_value
             args.append("%s=%s" % (arg, val))
         else:
             args.append(arg)
-        arg_index += 1
     if argspec.varargs:
-        args.append('*' + argspec.varargs)
+        args.append("*" + argspec.varargs)
-    if argspec.keywords:
+    if argspec.varkw:
-        args.append('**' + argspec.keywords)
+        args.append("**" + argspec.varkw)
     return "%s(%s)" % (method.__name__, ", ".join(args[1:]))
 
 
 def docstring(obj):
-    doc = (obj.__doc__ or '').rstrip()
+    doc = (obj.__doc__ or "").rstrip()
     if doc:
-        lines = doc.split('\n')
+        lines = doc.split("\n")
         # Find the length of the whitespace prefix common to all non-empty lines
         indentation = min(len(line) - len(line.lstrip()) for line in lines if line.strip())
         # Output the lines without the indentation
@@ -76,30 +83,30 @@ def docstring(obj):
 
 
 def class_doc(cls, list_methods=True):
-    bases = ', '.join([b.__name__ for b in cls.__bases__])
+    bases = ", ".join([b.__name__ for b in cls.__bases__])
-    print('###', cls.__name__)
+    print("###", cls.__name__)
     print()
-    if bases != 'object':
+    if bases != "object":
-        print('Extends', bases)
+        print("Extends", bases)
         print()
     docstring(cls)
     for name, method in inspect.getmembers(cls, lambda m: inspect.ismethod(m) or inspect.isfunction(m)):
-        if name == '__init__':
+        if name == "__init__":
             # Initializer
-            print('####', get_method_sig(method).replace(name, cls.__name__))
+            print("####", get_method_sig(method).replace(name, cls.__name__))
-        elif name[0] == '_':
+        elif name[0] == "_":
             # Private method
             continue
-        elif hasattr(method, '__self__') and method.__self__ == cls:
+        elif hasattr(method, "__self__") and method.__self__ == cls:
             # Class method
             if not list_methods:
                 continue
-            print('#### %s.%s' % (cls.__name__, get_method_sig(method)))
+            print("#### %s.%s" % (cls.__name__, get_method_sig(method)))
         else:
             # Regular method
             if not list_methods:
                 continue
-            print('####', get_method_sig(method))
+            print("####", get_method_sig(method))
         print()
         docstring(method)
         print()
@@ -108,7 +115,7 @@ def class_doc(cls, list_methods=True):
 def module_doc(classes, list_methods=True):
     mdl = classes[0].__module__
     print(mdl)
-    print('-' * len(mdl))
+    print("-" * len(mdl))
     print()
     for cls in classes:
         class_doc(cls, list_methods)
@@ -118,21 +125,17 @@ def all_subclasses(cls):
     return cls.__subclasses__() + [g for s in cls.__subclasses__() for g in all_subclasses(s)]
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
 
-    from clickhouse_orm import database
+    from clickhouse_orm import database, engines, fields, funcs, models, query, system_models
-    from clickhouse_orm import fields
-    from clickhouse_orm import engines
-    from clickhouse_orm import models
-    from clickhouse_orm import query
-    from clickhouse_orm import funcs
-    from clickhouse_orm import system_models
 
-    print('Class Reference')
+    print("Class Reference")
-    print('===============')
+    print("===============")
     print()
     module_doc([database.Database, database.DatabaseException])
-    module_doc([models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index])
+    module_doc(
+        [models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index]
+    )
     module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False)
     module_doc([engines.Engine] + all_subclasses(engines.Engine), False)
     module_doc([query.QuerySet, query.AggregateQuerySet, query.Q])

scripts/generate_toc.sh

@@ -1,7 +1,7 @@
-
+#!/bin/bash
 generate_one() {
     # Converts Markdown to HTML using Pandoc, and then extracts the header tags
-    pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md
+    pandoc "$1" | poetry run python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md
 }
 
 printf "# Table of Contents\n\n" > toc.md

scripts/html_to_markdown_toc.py

@@ -1,14 +1,13 @@
-from html.parser import HTMLParser
 import sys
+from html.parser import HTMLParser
 
-
+HEADER_TAGS = ("h1", "h2", "h3")
-HEADER_TAGS = ('h1', 'h2', 'h3')
 
 
 class HeadersToMarkdownParser(HTMLParser):
 
     inside = None
-    text = ''
+    text = ""
 
     def handle_starttag(self, tag, attrs):
         if tag.lower() in HEADER_TAGS:
@@ -16,11 +15,11 @@ class HeadersToMarkdownParser(HTMLParser):
 
     def handle_endtag(self, tag):
         if tag.lower() in HEADER_TAGS:
-            indent = '   ' * int(self.inside[1])
+            indent = "   " * int(self.inside[1])
-            fragment = self.text.lower().replace(' ', '-').replace('.', '')
+            fragment = self.text.lower().replace(" ", "-").replace(".", "")
-            print('%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment))
+            print("%s* [%s](%s#%s)" % (indent, self.text, sys.argv[1], fragment))
             self.inside = None
-            self.text = ''
+            self.text = ""
 
     def handle_data(self, data):
         if self.inside:
@@ -28,4 +27,4 @@ class HeadersToMarkdownParser(HTMLParser):
 
 
 HeadersToMarkdownParser().feed(sys.stdin.read())
-print('')
+print("")

scripts/test_python3.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-cd /tmp
-rm -rf /tmp/orm_env*
-virtualenv -p python3 /tmp/orm_env
-cd /tmp/orm_env
-source bin/activate
-pip install infi.projector
-git clone https://github.com/Infinidat/clickhouse_orm.git
-cd clickhouse_orm
-projector devenv build
-bin/nosetests

setup.cfg

@@ -18,4 +18,3 @@ ignore =
 exclude =
     tests/sample_migrations
     examples
-    scripts