mirror of
https://github.com/carrotquest/django-clickhouse.git
synced 2024-11-22 00:56:37 +03:00
Converted all typing hints to python 3 style
This commit is contained in:
parent
210c36a127
commit
5cb43ca6cd
|
@ -39,20 +39,27 @@ env:
|
||||||
- PG=9.6 DJANGO=2.1
|
- PG=9.6 DJANGO=2.1
|
||||||
- PG=10 DJANGO=2.1
|
- PG=10 DJANGO=2.1
|
||||||
- PG=11 DJANGO=2.1
|
- PG=11 DJANGO=2.1
|
||||||
|
- PG=12 DJANGO=2.1
|
||||||
- PG=9.6 DJANGO=2.2
|
- PG=9.6 DJANGO=2.2
|
||||||
- PG=10 DJANGO=2.2
|
- PG=10 DJANGO=2.2
|
||||||
- PG=11 DJANGO=2.2
|
- PG=11 DJANGO=2.2
|
||||||
|
- PG=12 DJANGO=2.2
|
||||||
- PG=9.6 DJANGO=3.0
|
- PG=9.6 DJANGO=3.0
|
||||||
- PG=10 DJANGO=3.0
|
- PG=10 DJANGO=3.0
|
||||||
- PG=11 DJANGO=3.0
|
- PG=11 DJANGO=3.0
|
||||||
|
- PG=12 DJANGO=3.0
|
||||||
|
|
||||||
before_install:
|
before_install:
|
||||||
# Use default PostgreSQL 11 port
|
# Use default PostgreSQL 11 port
|
||||||
- sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf
|
- sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf
|
||||||
- sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf
|
- sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf
|
||||||
|
|
||||||
|
- sudo sed -i 's/port = 5434/port = 5432/' /etc/postgresql/12/main/postgresql.conf
|
||||||
|
- sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf
|
||||||
|
|
||||||
# Start PostgreSQL version we need
|
# Start PostgreSQL version we need
|
||||||
- sudo systemctl stop postgresql && sudo systemctl start postgresql@$PG-main
|
- sudo systemctl stop postgresql
|
||||||
|
- sudo systemctl start postgresql@$PG-main
|
||||||
|
|
||||||
# ClickHouse sources
|
# ClickHouse sources
|
||||||
- sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
|
- sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
|
||||||
|
|
|
@ -94,12 +94,11 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values)
|
return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def objects_in(cls, database): # type: (Database) -> QuerySet
|
def objects_in(cls, database: Database)-> QuerySet:
|
||||||
return QuerySet(cls, database)
|
return QuerySet(cls, database)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_database_alias(cls, for_write=False):
|
def get_database_alias(cls, for_write: bool = False) -> str:
|
||||||
# type: (bool) -> str
|
|
||||||
"""
|
"""
|
||||||
Gets database alias for read or write purposes
|
Gets database alias for read or write purposes
|
||||||
:param for_write: Boolean flag if database is neede for read or for write
|
:param for_write: Boolean flag if database is neede for read or for write
|
||||||
|
@ -112,8 +111,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
return db_router.db_for_read(cls)
|
return db_router.db_for_read(cls)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_database(cls, for_write=False):
|
def get_database(cls, for_write: bool = False) -> Database:
|
||||||
# type: (bool) -> Database
|
|
||||||
"""
|
"""
|
||||||
Gets database alias for read or write purposes
|
Gets database alias for read or write purposes
|
||||||
:param for_write: Boolean flag if database is neede for read or for write
|
:param for_write: Boolean flag if database is neede for read or for write
|
||||||
|
@ -123,8 +121,8 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
return connections[db_alias]
|
return connections[db_alias]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_django_model_serializer(cls, writable=False, defaults=None):
|
def get_django_model_serializer(cls, writable: bool= False, defaults: Optional[dict] = None
|
||||||
# type: (bool, Optional[dict]) -> Django2ClickHouseModelSerializer
|
) -> Django2ClickHouseModelSerializer:
|
||||||
serializer_cls = lazy_class_import(cls.django_model_serializer)
|
serializer_cls = lazy_class_import(cls.django_model_serializer)
|
||||||
return serializer_cls(cls, writable=writable, defaults=defaults)
|
return serializer_cls(cls, writable=writable, defaults=defaults)
|
||||||
|
|
||||||
|
@ -171,7 +169,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_sync_query_set(cls, using, pk_set): # type: (str, Set[Any]) -> DjangoQuerySet
|
def get_sync_query_set(cls, using: str, pk_set: Set[Any]) -> DjangoQuerySet:
|
||||||
"""
|
"""
|
||||||
Forms django queryset to fetch for sync
|
Forms django queryset to fetch for sync
|
||||||
:param using: Database to fetch from
|
:param using: Database to fetch from
|
||||||
|
@ -181,7 +179,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
return cls.django_model.objects.filter(pk__in=pk_set).using(using)
|
return cls.django_model.objects.filter(pk__in=pk_set).using(using)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_sync_objects(cls, operations): # type: (List[Tuple[str, str]]) -> List[DjangoModel]
|
def get_sync_objects(cls, operations: List[Tuple[str, str]]) -> List[DjangoModel]:
|
||||||
"""
|
"""
|
||||||
Returns objects from main database to sync
|
Returns objects from main database to sync
|
||||||
:param operations: A list of operations to perform
|
:param operations: A list of operations to perform
|
||||||
|
@ -203,7 +201,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
return list(chain(*objs))
|
return list(chain(*objs))
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_insert_batch(cls, import_objects): # type: (Iterable[DjangoModel]) -> List[ClickHouseModel]
|
def get_insert_batch(cls, import_objects: Iterable[DjangoModel]) -> List['ClickHouseModel']:
|
||||||
"""
|
"""
|
||||||
Formats django model objects to batch of ClickHouse objects
|
Formats django model objects to batch of ClickHouse objects
|
||||||
:param import_objects: DjangoModel objects to import
|
:param import_objects: DjangoModel objects to import
|
||||||
|
@ -267,7 +265,7 @@ class ClickHouseModel(with_metaclass(ClickHouseModelMeta, InfiModel)):
|
||||||
raise ex
|
raise ex
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def need_sync(cls): # type: () -> bool
|
def need_sync(cls) -> bool:
|
||||||
"""
|
"""
|
||||||
Checks if this model needs synchronization: sync is enabled and delay has passed
|
Checks if this model needs synchronization: sync is enabled and delay has passed
|
||||||
:return: Boolean
|
:return: Boolean
|
||||||
|
|
|
@ -28,7 +28,7 @@ DEFAULTS = {
|
||||||
|
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
def __getattr__(self, item): # type: (str) -> Any
|
def __getattr__(self, item: str) -> Any:
|
||||||
if item not in DEFAULTS:
|
if item not in DEFAULTS:
|
||||||
raise AttributeError('Unknown config parameter `%s`' % item)
|
raise AttributeError('Unknown config parameter `%s`' % item)
|
||||||
|
|
||||||
|
|
|
@ -35,8 +35,8 @@ class Database(InfiDatabase):
|
||||||
def _get_applied_migrations(self, migrations_package_name):
|
def _get_applied_migrations(self, migrations_package_name):
|
||||||
raise NotImplementedError("This method is not supported by django_clickhouse.")
|
raise NotImplementedError("This method is not supported by django_clickhouse.")
|
||||||
|
|
||||||
def select_tuples(self, query, model_class, settings=None):
|
def select_tuples(self, query: str, model_class: Type['ClickHouseModel'], settings: Optional[dict] = None
|
||||||
# type: (str, Type['ClickHouseModel'], Optional[dict], Optional[dict]) -> Generator[tuple]
|
) -> Iterable[tuple]:
|
||||||
"""
|
"""
|
||||||
This method selects model_class namedtuples, instead of class instances.
|
This method selects model_class namedtuples, instead of class instances.
|
||||||
Less memory consumption, greater speed
|
Less memory consumption, greater speed
|
||||||
|
@ -67,11 +67,11 @@ class Database(InfiDatabase):
|
||||||
|
|
||||||
yield item
|
yield item
|
||||||
|
|
||||||
def insert_tuples(self, model_class, model_tuples, batch_size=None, formatted=False):
|
def insert_tuples(self, model_class: Type['ClickHouseModel'], model_tuples: Iterable[tuple],
|
||||||
# type: (Type['ClickHouseModel'], Iterable[tuple], Optional[int], bool) -> None
|
batch_size: Optional[int] = None, formatted: bool = False) -> None:
|
||||||
"""
|
"""
|
||||||
Inserts model_class namedtuples
|
Inserts model_class namedtuples
|
||||||
:param model_class: Clickhouse model, namedtuples are made from
|
:param model_class: ClickHouse model, namedtuples are made from
|
||||||
:param model_tuples: An iterable of tuples to insert
|
:param model_tuples: An iterable of tuples to insert
|
||||||
:param batch_size: Size of batch
|
:param batch_size: Size of batch
|
||||||
:param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string
|
:param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
This file contains wrappers for infi.clckhouse_orm engines to use in django-clickhouse
|
This file contains wrappers for infi.clckhouse_orm engines to use in django-clickhouse
|
||||||
"""
|
"""
|
||||||
import datetime
|
import datetime
|
||||||
from typing import List, Type, Union, Iterable, Generator
|
from typing import List, Type, Union, Iterable, Generator, Optional
|
||||||
|
|
||||||
from django.db.models import Model as DjangoModel
|
from django.db.models import Model as DjangoModel
|
||||||
from infi.clickhouse_orm import engines as infi_engines
|
from infi.clickhouse_orm import engines as infi_engines
|
||||||
|
@ -14,8 +14,7 @@ from .utils import format_datetime
|
||||||
|
|
||||||
|
|
||||||
class InsertOnlyEngineMixin:
|
class InsertOnlyEngineMixin:
|
||||||
def get_insert_batch(self, model_cls, objects):
|
def get_insert_batch(self, model_cls: Type['ClickHouseModel'], objects: List[DjangoModel]) -> Iterable[tuple]:
|
||||||
# type: (Type['ClickHouseModel'], List[DjangoModel]) -> Generator[tuple]
|
|
||||||
"""
|
"""
|
||||||
Gets a list of model_cls instances to insert into database
|
Gets a list of model_cls instances to insert into database
|
||||||
:param model_cls: ClickHouseModel subclass to import
|
:param model_cls: ClickHouseModel subclass to import
|
||||||
|
@ -69,8 +68,8 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
|
||||||
max_date=max_date, object_pks=','.join(object_pks))
|
max_date=max_date, object_pks=','.join(object_pks))
|
||||||
return connections[db_alias].select_tuples(query, model_cls)
|
return connections[db_alias].select_tuples(query, model_cls)
|
||||||
|
|
||||||
def get_final_versions(self, model_cls, objects, date_col=None):
|
def get_final_versions(self, model_cls: Type['ClickHouseModel'], objects: Iterable[DjangoModel],
|
||||||
# type: (Type['ClickHouseModel'], Iterable[DjangoModel], str) -> Generator[tuple]
|
date_col: Optional[str] = None) -> Iterable[tuple]:
|
||||||
"""
|
"""
|
||||||
Get objects, that are currently stored in ClickHouse.
|
Get objects, that are currently stored in ClickHouse.
|
||||||
Depending on the partition key this can be different for different models.
|
Depending on the partition key this can be different for different models.
|
||||||
|
@ -82,7 +81,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
|
||||||
:return: A generator of named tuples, representing previous state
|
:return: A generator of named tuples, representing previous state
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def _dt_to_str(dt): # type: (Union[datetime.date, datetime.datetime]) -> str
|
def _dt_to_str(dt: Union[datetime.date, datetime.datetime]) -> str:
|
||||||
if isinstance(dt, datetime.datetime):
|
if isinstance(dt, datetime.datetime):
|
||||||
return format_datetime(dt, 0, db_alias=db_alias)
|
return format_datetime(dt, 0, db_alias=db_alias)
|
||||||
elif isinstance(dt, datetime.date):
|
elif isinstance(dt, datetime.date):
|
||||||
|
@ -123,8 +122,7 @@ class CollapsingMergeTree(InsertOnlyEngineMixin, infi_engines.CollapsingMergeTre
|
||||||
else:
|
else:
|
||||||
return self._get_final_versions_by_final(*params)
|
return self._get_final_versions_by_final(*params)
|
||||||
|
|
||||||
def get_insert_batch(self, model_cls, objects):
|
def get_insert_batch(self, model_cls: Type['ClickHouseModel'], objects: List[DjangoModel]) -> Iterable[tuple]:
|
||||||
# type: (Type['ClickHouseModel'], List[DjangoModel]) -> Generator[tuple]
|
|
||||||
"""
|
"""
|
||||||
Gets a list of model_cls instances to insert into database
|
Gets a list of model_cls instances to insert into database
|
||||||
:param model_cls: ClickHouseModel subclass to import
|
:param model_cls: ClickHouseModel subclass to import
|
||||||
|
|
|
@ -23,7 +23,7 @@ class Migration:
|
||||||
"""
|
"""
|
||||||
operations = []
|
operations = []
|
||||||
|
|
||||||
def apply(self, db_alias, database=None): # type: (str, Optional[Database]) -> None
|
def apply(self, db_alias: str, database: Optional[Database] = None) -> None:
|
||||||
"""
|
"""
|
||||||
Applies migration to given database
|
Applies migration to given database
|
||||||
:param db_alias: Database alias to apply migration to
|
:param db_alias: Database alias to apply migration to
|
||||||
|
@ -41,8 +41,7 @@ class Migration:
|
||||||
op.apply(database)
|
op.apply(database)
|
||||||
|
|
||||||
|
|
||||||
def migrate_app(app_label, db_alias, up_to=9999, database=None):
|
def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None) -> None:
|
||||||
# type: (str, str, int, Optional[Database]) -> None
|
|
||||||
"""
|
"""
|
||||||
Migrates given django app
|
Migrates given django app
|
||||||
:param app_label: App label to migrate
|
:param app_label: App label to migrate
|
||||||
|
@ -110,7 +109,7 @@ class MigrationHistory(ClickHouseModel):
|
||||||
engine = MergeTree('applied', ('db_alias', 'package_name', 'module_name'))
|
engine = MergeTree('applied', ('db_alias', 'package_name', 'module_name'))
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def set_migration_applied(cls, db_alias, migrations_package, name): # type: (str, str, str) -> None
|
def set_migration_applied(cls, db_alias: str, migrations_package: str, name: str) -> None:
|
||||||
"""
|
"""
|
||||||
Sets migration apply status
|
Sets migration apply status
|
||||||
:param db_alias: Database alias migration is applied to
|
:param db_alias: Database alias migration is applied to
|
||||||
|
@ -126,7 +125,7 @@ class MigrationHistory(ClickHouseModel):
|
||||||
applied=datetime.date.today())
|
applied=datetime.date.today())
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_applied_migrations(cls, db_alias, migrations_package): # type: (str, str) -> Set[str]
|
def get_applied_migrations(cls, db_alias: str, migrations_package: str) -> Set[str]:
|
||||||
"""
|
"""
|
||||||
Returns applied migrations names
|
Returns applied migrations names
|
||||||
:param db_alias: Database alias, to check
|
:param db_alias: Database alias, to check
|
||||||
|
|
|
@ -5,7 +5,6 @@ It saves all operations to storage in order to write them to ClickHouse later.
|
||||||
|
|
||||||
from typing import Optional, Any, Type, Set
|
from typing import Optional, Any, Type, Set
|
||||||
|
|
||||||
import functools
|
|
||||||
import six
|
import six
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
from django.db.models import QuerySet as DjangoQuerySet, Model as DjangoModel, Manager as DjangoManager
|
from django.db.models import QuerySet as DjangoQuerySet, Model as DjangoModel, Manager as DjangoManager
|
||||||
|
@ -147,7 +146,7 @@ class ClickHouseSyncModel(DjangoModel):
|
||||||
abstract = True
|
abstract = True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_clickhouse_storage(cls): # type: () -> Storage
|
def get_clickhouse_storage(cls) -> Storage:
|
||||||
"""
|
"""
|
||||||
Returns Storage instance to save clickhouse sync data to
|
Returns Storage instance to save clickhouse sync data to
|
||||||
:return:
|
:return:
|
||||||
|
@ -156,8 +155,7 @@ class ClickHouseSyncModel(DjangoModel):
|
||||||
return storage_cls()
|
return storage_cls()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def register_clickhouse_sync_model(cls, model_cls):
|
def register_clickhouse_sync_model(cls, model_cls: Type['ClickHouseModel']) -> None:
|
||||||
# type: (Type['django_clickhouse.clickhouse_models.ClickHouseModel']) -> None
|
|
||||||
"""
|
"""
|
||||||
Registers ClickHouse model to listen to this model updates
|
Registers ClickHouse model to listen to this model updates
|
||||||
:param model_cls: Model class to register
|
:param model_cls: Model class to register
|
||||||
|
@ -169,7 +167,7 @@ class ClickHouseSyncModel(DjangoModel):
|
||||||
cls._clickhouse_sync_models.add(model_cls)
|
cls._clickhouse_sync_models.add(model_cls)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_clickhouse_sync_models(cls): # type: () -> Set['django_clickhouse.clickhouse_models.ClickHouseModel']
|
def get_clickhouse_sync_models(cls) -> Set['ClickHouseModel']:
|
||||||
"""
|
"""
|
||||||
Returns all clickhouse models, listening to this class
|
Returns all clickhouse models, listening to this class
|
||||||
:return: A set of model classes to sync
|
:return: A set of model classes to sync
|
||||||
|
@ -177,8 +175,7 @@ class ClickHouseSyncModel(DjangoModel):
|
||||||
return getattr(cls, '_clickhouse_sync_models', set())
|
return getattr(cls, '_clickhouse_sync_models', set())
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def register_clickhouse_operations(cls, operation, *model_pks, using=None):
|
def register_clickhouse_operations(cls, operation: str, *model_pks: Any, using: Optional[str] = None) -> None:
|
||||||
# type: (str, *Any, Optional[str]) -> None
|
|
||||||
"""
|
"""
|
||||||
Registers model operation in storage
|
Registers model operation in storage
|
||||||
:param operation: Operation type - one of [insert, update, delete)
|
:param operation: Operation type - one of [insert, update, delete)
|
||||||
|
@ -197,10 +194,10 @@ class ClickHouseSyncModel(DjangoModel):
|
||||||
storage = cls.get_clickhouse_storage()
|
storage = cls.get_clickhouse_storage()
|
||||||
transaction.on_commit(_on_commit, using=using)
|
transaction.on_commit(_on_commit, using=using)
|
||||||
|
|
||||||
def post_save(self, created, using=None): # type: (bool, Optional[str]) -> None
|
def post_save(self, created: bool, using: Optional[str] = None) -> None:
|
||||||
self.register_clickhouse_operations('insert' if created else 'update', self.pk, using=using)
|
self.register_clickhouse_operations('insert' if created else 'update', self.pk, using=using)
|
||||||
|
|
||||||
def post_delete(self, using=None): # type: (Optional[str]) -> None
|
def post_delete(self, using: Optional[str] = None) -> None:
|
||||||
self.register_clickhouse_operations('delete', self.pk, using=using)
|
self.register_clickhouse_operations('delete', self.pk, using=using)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from typing import Optional, Iterable, List
|
from typing import Optional, Iterable, List, Type
|
||||||
|
|
||||||
from copy import copy
|
from copy import copy
|
||||||
from infi.clickhouse_orm.database import Database
|
from infi.clickhouse_orm.database import Database
|
||||||
|
@ -13,22 +13,22 @@ class QuerySet(InfiQuerySet):
|
||||||
Basic QuerySet to use
|
Basic QuerySet to use
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, model_cls, database=None): # type: (Type[InfiModel], Optional[Database]) -> None
|
def __init__(self, model_cls: Type[InfiModel], database: Optional[Database] = None) -> None:
|
||||||
super(QuerySet, self).__init__(model_cls, database)
|
super(QuerySet, self).__init__(model_cls, database)
|
||||||
self._db_alias = None
|
self._db_alias = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def _database(self): # type: () -> Database
|
def _database(self) -> Database:
|
||||||
# HACK for correct work of all infi.clickhouse-orm methods
|
# HACK for correct work of all infi.clickhouse-orm methods
|
||||||
# There are no write QuerySet methods now, so I use for_write=False by default
|
# There are no write QuerySet methods now, so I use for_write=False by default
|
||||||
return self.get_database(for_write=False)
|
return self.get_database(for_write=False)
|
||||||
|
|
||||||
@_database.setter
|
@_database.setter
|
||||||
def _database(self, database): # type: (Database) -> None
|
def _database(self, database: Database) -> None:
|
||||||
# HACK for correct work of all infi.clickhouse-orm methods
|
# HACK for correct work of all infi.clickhouse-orm methods
|
||||||
self._db = database
|
self._db = database
|
||||||
|
|
||||||
def get_database(self, for_write=False): # type: (bool) -> Database
|
def get_database(self, for_write: bool = False) -> Database:
|
||||||
"""
|
"""
|
||||||
Gets database to execute query on. Looks for constructor or using() method.
|
Gets database to execute query on. Looks for constructor or using() method.
|
||||||
If nothing was set tries to get database from model class using router.
|
If nothing was set tries to get database from model class using router.
|
||||||
|
@ -43,7 +43,7 @@ class QuerySet(InfiQuerySet):
|
||||||
|
|
||||||
return self._db
|
return self._db
|
||||||
|
|
||||||
def using(self, db_alias): # type: (str) -> QuerySet
|
def using(self, db_alias: str) -> 'QuerySet':
|
||||||
"""
|
"""
|
||||||
Sets database alias to use for this query
|
Sets database alias to use for this query
|
||||||
:param db_alias: Database alias name from CLICKHOUSE_DATABASES config option
|
:param db_alias: Database alias name from CLICKHOUSE_DATABASES config option
|
||||||
|
@ -54,7 +54,7 @@ class QuerySet(InfiQuerySet):
|
||||||
qs._db = None # Previous database should be forgotten
|
qs._db = None # Previous database should be forgotten
|
||||||
return qs
|
return qs
|
||||||
|
|
||||||
def all(self): # type: () -> QuerySet
|
def all(self) -> 'QuerySet':
|
||||||
"""
|
"""
|
||||||
Returns all items of queryset
|
Returns all items of queryset
|
||||||
:return: QuerySet
|
:return: QuerySet
|
||||||
|
@ -70,7 +70,7 @@ class QuerySet(InfiQuerySet):
|
||||||
self.get_database(for_write=True).insert([instance])
|
self.get_database(for_write=True).insert([instance])
|
||||||
return instance
|
return instance
|
||||||
|
|
||||||
def bulk_create(self, model_instances, batch_size=1000): # type: (Iterable[InfiModel], int) -> List[InfiModel]
|
def bulk_create(self, model_instances: Iterable[InfiModel], batch_size: int = 1000) -> List[InfiModel]:
|
||||||
self.get_database(for_write=True).insert(model_instances=model_instances, batch_size=batch_size)
|
self.get_database(for_write=True).insert(model_instances=model_instances, batch_size=batch_size)
|
||||||
return list(model_instances)
|
return list(model_instances)
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
"""
|
"""
|
||||||
This file defines router to find appropriate database
|
This file defines router to find appropriate database
|
||||||
"""
|
"""
|
||||||
from typing import Optional
|
from typing import Type
|
||||||
|
|
||||||
import random
|
import random
|
||||||
import six
|
import six
|
||||||
|
@ -13,8 +13,7 @@ from .utils import lazy_class_import
|
||||||
|
|
||||||
|
|
||||||
class DefaultRouter:
|
class DefaultRouter:
|
||||||
def db_for_read(self, model, **hints):
|
def db_for_read(self, model: Type[ClickHouseModel], **hints) -> str:
|
||||||
# type: (ClickHouseModel, **dict) -> str
|
|
||||||
"""
|
"""
|
||||||
Gets database to read from for model
|
Gets database to read from for model
|
||||||
:param model: Model to decide for
|
:param model: Model to decide for
|
||||||
|
@ -23,8 +22,7 @@ class DefaultRouter:
|
||||||
"""
|
"""
|
||||||
return random.choice(model.read_db_aliases)
|
return random.choice(model.read_db_aliases)
|
||||||
|
|
||||||
def db_for_write(self, model, **hints):
|
def db_for_write(self, model: Type[ClickHouseModel], **hints) -> str:
|
||||||
# type: (ClickHouseModel, **dict) -> str
|
|
||||||
"""
|
"""
|
||||||
Gets database to write to for model
|
Gets database to write to for model
|
||||||
:param model: Model to decide for
|
:param model: Model to decide for
|
||||||
|
@ -33,8 +31,8 @@ class DefaultRouter:
|
||||||
"""
|
"""
|
||||||
return random.choice(model.write_db_aliases)
|
return random.choice(model.write_db_aliases)
|
||||||
|
|
||||||
def allow_migrate(self, db_alias, app_label, operation, model=None, **hints):
|
def allow_migrate(self, db_alias: str, app_label: str, operation: Operation,
|
||||||
# type: (str, str, Operation, Optional[ClickHouseModel], **dict) -> bool
|
model=None, **hints) -> bool:
|
||||||
"""
|
"""
|
||||||
Checks if migration can be applied to given database
|
Checks if migration can be applied to given database
|
||||||
:param db_alias: Database alias to check
|
:param db_alias: Database alias to check
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from typing import NamedTuple
|
from typing import NamedTuple, Optional, Iterable, Type
|
||||||
|
|
||||||
import pytz
|
import pytz
|
||||||
from django.db.models import Model as DjangoModel
|
from django.db.models import Model as DjangoModel
|
||||||
|
@ -7,7 +7,19 @@ from .utils import model_to_dict
|
||||||
|
|
||||||
|
|
||||||
class Django2ClickHouseModelSerializer:
|
class Django2ClickHouseModelSerializer:
|
||||||
def __init__(self, model_cls, fields=None, exclude_fields=None, writable=False, defaults=None):
|
def __init__(self, model_cls: Type['ClickHouseModel'], fields: Optional[Iterable[str]] = None,
|
||||||
|
exclude_fields: Optional[Iterable[str]] = None, writable: bool = False,
|
||||||
|
defaults: Optional[dict] = None) -> None:
|
||||||
|
"""
|
||||||
|
Initializes serializer
|
||||||
|
:param model_cls: ClickHouseModel subclass to serialize to
|
||||||
|
:param fields: Optional. A list of fields to add into result tuple
|
||||||
|
:param exclude_fields: Fields to exclude from result tuple
|
||||||
|
:param writable: If fields parameter is not set directly,
|
||||||
|
this flags determines if only writable or all fields should be taken from model_cls
|
||||||
|
:param defaults: A dictionary of field: value which are taken as default values for model_cls instances
|
||||||
|
:return: None
|
||||||
|
"""
|
||||||
self._model_cls = model_cls
|
self._model_cls = model_cls
|
||||||
if fields is not None:
|
if fields is not None:
|
||||||
self.serialize_fields = fields
|
self.serialize_fields = fields
|
||||||
|
@ -18,7 +30,7 @@ class Django2ClickHouseModelSerializer:
|
||||||
self._result_class = self._model_cls.get_tuple_class(defaults=defaults)
|
self._result_class = self._model_cls.get_tuple_class(defaults=defaults)
|
||||||
self._fields = self._model_cls.fields(writable=False)
|
self._fields = self._model_cls.fields(writable=False)
|
||||||
|
|
||||||
def _get_serialize_kwargs(self, obj):
|
def _get_serialize_kwargs(self, obj: DjangoModel) -> dict:
|
||||||
data = model_to_dict(obj, fields=self.serialize_fields, exclude_fields=self.exclude_serialize_fields)
|
data = model_to_dict(obj, fields=self.serialize_fields, exclude_fields=self.exclude_serialize_fields)
|
||||||
|
|
||||||
# Remove None values, they should be initialized as defaults
|
# Remove None values, they should be initialized as defaults
|
||||||
|
@ -29,5 +41,5 @@ class Django2ClickHouseModelSerializer:
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def serialize(self, obj): # type: (DjangoModel) -> NamedTuple
|
def serialize(self, obj: DjangoModel) -> NamedTuple:
|
||||||
return self._result_class(**self._get_serialize_kwargs(obj))
|
return self._result_class(**self._get_serialize_kwargs(obj))
|
||||||
|
|
|
@ -39,7 +39,7 @@ class Storage:
|
||||||
But ClickHouse is idempotent to duplicate inserts. So we can insert one batch twice correctly.
|
But ClickHouse is idempotent to duplicate inserts. So we can insert one batch twice correctly.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def pre_sync(self, import_key, **kwargs): # type: (str, **dict) -> None
|
def pre_sync(self, import_key: str, **kwargs) -> None:
|
||||||
"""
|
"""
|
||||||
This method is called before import process starts
|
This method is called before import process starts
|
||||||
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
||||||
|
@ -48,7 +48,7 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def post_sync(self, import_key, **kwargs): # type: (str, **dict) -> None
|
def post_sync(self, import_key: str, **kwargs) -> None:
|
||||||
"""
|
"""
|
||||||
This method is called after import process has finished.
|
This method is called after import process has finished.
|
||||||
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
||||||
|
@ -57,7 +57,7 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def post_sync_failed(self, import_key, **kwargs): # type: (str, **dict) -> None
|
def post_sync_failed(self, import_key: str, **kwargs) -> None:
|
||||||
"""
|
"""
|
||||||
This method is called after import process has finished with exception.
|
This method is called after import process has finished with exception.
|
||||||
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
||||||
|
@ -66,7 +66,7 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def post_batch_removed(self, import_key, batch_size): # type: (str, int) -> None
|
def post_batch_removed(self, import_key: str, batch_size: int) -> None:
|
||||||
"""
|
"""
|
||||||
This method marks that batch has been removed in statsd
|
This method marks that batch has been removed in statsd
|
||||||
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
||||||
|
@ -76,8 +76,7 @@ class Storage:
|
||||||
key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key)
|
key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key)
|
||||||
statsd.gauge(key, self.operations_count(import_key))
|
statsd.gauge(key, self.operations_count(import_key))
|
||||||
|
|
||||||
def operations_count(self, import_key, **kwargs):
|
def operations_count(self, import_key: str, **kwargs) -> int:
|
||||||
# type: (str, **dict) -> int
|
|
||||||
"""
|
"""
|
||||||
Returns sync queue size
|
Returns sync queue size
|
||||||
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
||||||
|
@ -86,8 +85,7 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
raise NotImplemented()
|
raise NotImplemented()
|
||||||
|
|
||||||
def get_operations(self, import_key, count, **kwargs):
|
def get_operations(self, import_key: str, count: int, **kwargs) -> List[Tuple[str, str]]:
|
||||||
# type: (str, int, **dict) -> List[Tuple[str, str]]
|
|
||||||
"""
|
"""
|
||||||
Must return a list of operations on the model.
|
Must return a list of operations on the model.
|
||||||
Method should be error safe - if something goes wrong, import data should not be lost.
|
Method should be error safe - if something goes wrong, import data should not be lost.
|
||||||
|
@ -98,7 +96,7 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
raise NotImplemented()
|
raise NotImplemented()
|
||||||
|
|
||||||
def register_operations(self, import_key, operation, *pks): # type: (str, str, *Any) -> int
|
def register_operations(self, import_key: str, operation: str, *pks: Any) -> int:
|
||||||
"""
|
"""
|
||||||
Registers new incoming operation
|
Registers new incoming operation
|
||||||
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
|
||||||
|
@ -108,8 +106,7 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
def register_operations_wrapped(self, import_key, operation, *pks):
|
def register_operations_wrapped(self, import_key: str, operation: str, *pks: Any) -> int:
|
||||||
# type: (str, str, *Any) -> int
|
|
||||||
"""
|
"""
|
||||||
This is a wrapper for register_operation method, checking main parameters.
|
This is a wrapper for register_operation method, checking main parameters.
|
||||||
This method should be called from inner functions.
|
This method should be called from inner functions.
|
||||||
|
@ -140,14 +137,14 @@ class Storage:
|
||||||
"""
|
"""
|
||||||
raise NotImplemented()
|
raise NotImplemented()
|
||||||
|
|
||||||
def get_last_sync_time(self, import_key): # type: (str) -> Optional[datetime.datetime]
|
def get_last_sync_time(self, import_key: str) -> Optional[datetime.datetime]:
|
||||||
"""
|
"""
|
||||||
Gets the last time, sync has been executed
|
Gets the last time, sync has been executed
|
||||||
:return: datetime.datetime if last sync has been. Otherwise - None.
|
:return: datetime.datetime if last sync has been. Otherwise - None.
|
||||||
"""
|
"""
|
||||||
raise NotImplemented()
|
raise NotImplemented()
|
||||||
|
|
||||||
def set_last_sync_time(self, import_key, dt): # type: (str, datetime.datetime) -> None
|
def set_last_sync_time(self, import_key: str, dt: datetime.datetime) -> None:
|
||||||
"""
|
"""
|
||||||
Sets successful sync time
|
Sets successful sync time
|
||||||
:return: None
|
:return: None
|
||||||
|
|
|
@ -11,14 +11,14 @@ from .utils import get_subclasses
|
||||||
|
|
||||||
|
|
||||||
@shared_task(queue=config.CELERY_QUEUE)
|
@shared_task(queue=config.CELERY_QUEUE)
|
||||||
def sync_clickhouse_model(cls): # type: (ClickHouseModel) -> None
|
def sync_clickhouse_model(model_cls) -> None:
|
||||||
"""
|
"""
|
||||||
Syncs one batch of given ClickHouseModel
|
Syncs one batch of given ClickHouseModel
|
||||||
:param cls: ClickHouseModel subclass
|
:param model_cls: ClickHouseModel subclass
|
||||||
:return: None
|
:return: None
|
||||||
"""
|
"""
|
||||||
cls.get_storage().set_last_sync_time(cls.get_import_key(), datetime.datetime.now())
|
model_cls.get_storage().set_last_sync_time(model_cls.get_import_key(), datetime.datetime.now())
|
||||||
cls.sync_batch_from_storage()
|
model_cls.sync_batch_from_storage()
|
||||||
|
|
||||||
|
|
||||||
@shared_task(queue=config.CELERY_QUEUE)
|
@shared_task(queue=config.CELERY_QUEUE)
|
||||||
|
|
|
@ -18,7 +18,7 @@ from .database import connections
|
||||||
T = TypeVar('T')
|
T = TypeVar('T')
|
||||||
|
|
||||||
|
|
||||||
def get_tz_offset(db_alias=None): # type: (Optional[str]) -> int
|
def get_tz_offset(db_alias: Optional[str] = None) -> int:
|
||||||
"""
|
"""
|
||||||
Returns ClickHouse server timezone offset in minutes
|
Returns ClickHouse server timezone offset in minutes
|
||||||
:param db_alias: The database alias used
|
:param db_alias: The database alias used
|
||||||
|
@ -28,8 +28,8 @@ def get_tz_offset(db_alias=None): # type: (Optional[str]) -> int
|
||||||
return int(db.server_timezone.utcoffset(datetime.datetime.utcnow()).total_seconds() / 60)
|
return int(db.server_timezone.utcoffset(datetime.datetime.utcnow()).total_seconds() / 60)
|
||||||
|
|
||||||
|
|
||||||
def format_datetime(dt, timezone_offset=0, day_end=False, db_alias=None):
|
def format_datetime(dt: Union[datetime.date, datetime.datetime], timezone_offset: int = 0, day_end: bool = False,
|
||||||
# type: (Union[datetime.date, datetime.datetime], int, bool, Optional[str]) -> str
|
db_alias: Optional[str] = None) -> str:
|
||||||
"""
|
"""
|
||||||
Formats datetime and date objects to format that can be used in WHERE conditions of query
|
Formats datetime and date objects to format that can be used in WHERE conditions of query
|
||||||
:param dt: datetime.datetime or datetime.date object
|
:param dt: datetime.datetime or datetime.date object
|
||||||
|
@ -58,9 +58,9 @@ def format_datetime(dt, timezone_offset=0, day_end=False, db_alias=None):
|
||||||
return server_dt.strftime("%Y-%m-%d %H:%M:%S")
|
return server_dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
def module_exists(module_name): # type: (str) -> bool
|
def module_exists(module_name: str) -> bool:
|
||||||
"""
|
"""
|
||||||
Checks if moudle exists
|
Checks if module exists
|
||||||
:param module_name: Dot-separated module name
|
:param module_name: Dot-separated module name
|
||||||
:return: Boolean
|
:return: Boolean
|
||||||
"""
|
"""
|
||||||
|
@ -69,7 +69,7 @@ def module_exists(module_name): # type: (str) -> bool
|
||||||
return spam_spec is not None
|
return spam_spec is not None
|
||||||
|
|
||||||
|
|
||||||
def lazy_class_import(obj): # type: (Union[str, Any]) -> Any
|
def lazy_class_import(obj: Union[str, Any]) -> Any:
|
||||||
"""
|
"""
|
||||||
If string is given, imports object by given module path.
|
If string is given, imports object by given module path.
|
||||||
Otherwise returns the object
|
Otherwise returns the object
|
||||||
|
@ -88,7 +88,7 @@ def lazy_class_import(obj): # type: (Union[str, Any]) -> Any
|
||||||
return obj
|
return obj
|
||||||
|
|
||||||
|
|
||||||
def get_subclasses(cls, recursive=False): # type: (T, bool) -> Set[T]
|
def get_subclasses(cls: T, recursive: bool = False) -> Set[T]:
|
||||||
"""
|
"""
|
||||||
Gets all subclasses of given class
|
Gets all subclasses of given class
|
||||||
Attention!!! Classes would be found only if they were imported before using this function
|
Attention!!! Classes would be found only if they were imported before using this function
|
||||||
|
@ -105,8 +105,8 @@ def get_subclasses(cls, recursive=False): # type: (T, bool) -> Set[T]
|
||||||
return subclasses
|
return subclasses
|
||||||
|
|
||||||
|
|
||||||
def model_to_dict(instance, fields=None, exclude_fields=None):
|
def model_to_dict(instance: DjangoModel, fields: Optional[Iterable[str]] = None,
|
||||||
# type: (DjangoModel, Optional[Iterable[str]], Optional[Iterable[str]]) -> Dict[str, Any]
|
exclude_fields: Optional[Iterable[str]] = None) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Standard model_to_dict ignores some fields if they have invalid naming
|
Standard model_to_dict ignores some fields if they have invalid naming
|
||||||
:param instance: Object to convert to dictionary
|
:param instance: Object to convert to dictionary
|
||||||
|
|
Loading…
Reference in New Issue
Block a user