ets-labs/python-dependency-injector
1
1
Fork 0
mirror of https://github.com/ets-labs/python-dependency-injector.git synced 2024-11-24 10:34:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Optimization r1 (#571)

Browse Source 
* Add isfuture() and iscoroutine() optimization

* Apply async mode optimization

* Wiring changes

* Add optimization for wiring of async coroutines

* Remove unused imports

* Update changelog

* Refactor async mode checks
This commit is contained in:
Roman Mogylatov 2022-03-27 22:20:05 -04:00 committed by GitHub
parent f0c55cda22
commit 77b5cdebd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 19897 additions and 9950 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/main/changelog.rst
View File

@ -9,6 +9,7 @@ follows `Semantic versioning`_
Develop
-------
- Optimize synchronous and asynchronous injections and wiring from x1.5 to x7 times depending on the use case.
- Fix bug `#569 <https://github.com/ets-labs/python-dependency-injector/issues/569>`_:
"numpy.typing.NDArray breaks wiring". Thanks to
`@VKFisher (Vlad Fisher) <https://github.com/VKFisher>`_ for reporting the issue and providing a fix.

4
setup.py
View File

@ -58,6 +58,10 @@ setup(name="dependency-injector",
["src/dependency_injector/providers.c"],
define_macros=list(defined_macros.items()),
extra_compile_args=["-O2"]),
Extension("dependency_injector._cwiring",
["src/dependency_injector/_cwiring.c"],
define_macros=list(defined_macros.items()),
extra_compile_args=["-O2"]),
],
install_requires=requirements,
extras_require={

9381
src/dependency_injector/_cwiring.c Normal file
View File

File diff suppressed because it is too large Load Diff

88
src/dependency_injector/_cwiring.pyx Normal file
View File

@ -0,0 +1,88 @@
"""Wiring optimizations module."""
import asyncio
import collections.abc
import functools
import inspect
import types
from . import providers
from .wiring import _Marker
def _get_sync_patched(fn):
@functools.wraps(fn)
def _patched(*args, **kwargs):
cdef object result
cdef dict to_inject
to_inject = kwargs.copy()
for injection, provider in _patched.__injections__.items():
if injection not in kwargs or isinstance(kwargs[injection], _Marker):
to_inject[injection] = provider()
result = fn(*args, **to_inject)
if _patched.__closing__:
for injection, provider in _patched.__closing__.items():
if injection in kwargs and not isinstance(kwargs[injection], _Marker):
continue
if not isinstance(provider, providers.Resource):
continue
provider.shutdown()
return result
return _patched
def _get_async_patched(fn):
@functools.wraps(fn)
async def _patched(*args, **kwargs):
cdef object result
cdef dict to_inject
cdef list to_inject_await = []
cdef list to_close_await = []
to_inject = kwargs.copy()
for injection, provider in _patched.__injections__.items():
if injection not in kwargs or isinstance(kwargs[injection], _Marker):
provide = provider()
if _isawaitable(provide):
to_inject_await.append((injection, provide))
else:
to_inject[injection] = provide
if to_inject_await:
async_to_inject = await asyncio.gather(*(provide for _, provide in to_inject_await))
for provide, (injection, _) in zip(async_to_inject, to_inject_await):
to_inject[injection] = provide
result = await fn(*args, **to_inject)
if _patched.__closing__:
for injection, provider in _patched.__closing__.items():
if injection in kwargs \
and isinstance(kwargs[injection], _Marker):
continue
if not isinstance(provider, providers.Resource):
continue
shutdown = provider.shutdown()
if _isawaitable(shutdown):
to_close_await.append(shutdown)
await asyncio.gather(*to_close_await)
return result
# Hotfix for iscoroutinefunction() for Cython < 3.0.0; can be removed after migration to Cython 3.0.0+
_patched._is_coroutine = asyncio.coroutines._is_coroutine
return _patched
cdef bint _isawaitable(object instance):
"""Return true if object can be passed to an ``await`` expression."""
return (isinstance(instance, types.CoroutineType) or
isinstance(instance, types.GeneratorType) and
bool(instance.gi_code.co_flags & inspect.CO_ITERABLE_COROUTINE) or
isinstance(instance, collections.abc.Awaitable))

2600
src/dependency_injector/containers.c
View File

File diff suppressed because it is too large Load Diff

17602
src/dependency_injector/providers.c
View File

File diff suppressed because it is too large Load Diff

66
src/dependency_injector/providers.pxd
View File

@ -10,6 +10,14 @@ import functools
cimport cython
cdef int ASYNC_MODE_UNDEFINED
cdef int ASYNC_MODE_ENABLED
cdef int ASYNC_MODE_DISABLED
cdef set __iscoroutine_typecache
cdef tuple __COROUTINE_TYPES
# Base providers
cdef class Provider(object):
cdef tuple __overridden
@ -383,11 +391,13 @@ cdef inline object __provide_positional_args(
tuple args,
tuple inj_args,
int inj_args_len,
int async_mode,
):
cdef int index
cdef list positional_args = []
cdef list future_args = []
cdef PositionalInjection injection
cdef object value
if inj_args_len == 0:
return args
@ -397,7 +407,7 @@ cdef inline object __provide_positional_args(
value = __get_value(injection)
positional_args.append(value)
if __is_future_or_coroutine(value):
if async_mode != ASYNC_MODE_DISABLED and __is_future_or_coroutine(value):
future_args.append((index, value))
positional_args.extend(args)
@ -414,6 +424,7 @@ cdef inline object __provide_keyword_args(
dict kwargs,
tuple inj_kwargs,
int inj_kwargs_len,
int async_mode,
):
cdef int index
cdef object name
@ -428,7 +439,7 @@ cdef inline object __provide_keyword_args(
name = __get_name(kw_injection)
value = __get_value(kw_injection)
kwargs[name] = value
if __is_future_or_coroutine(value):
if async_mode != ASYNC_MODE_DISABLED and __is_future_or_coroutine(value):
future_kwargs.append((name, value))
else:
kwargs, prefixed = __separate_prefixed_kwargs(kwargs)
@ -447,7 +458,7 @@ cdef inline object __provide_keyword_args(
value = __get_value(kw_injection)
kwargs[name] = value
if __is_future_or_coroutine(value):
if async_mode != ASYNC_MODE_DISABLED and __is_future_or_coroutine(value):
future_kwargs.append((name, value))
if future_kwargs:
@ -550,20 +561,26 @@ cdef inline object __call(
dict context_kwargs,
tuple injection_kwargs,
int injection_kwargs_len,
int async_mode,
):
args = __provide_positional_args(
cdef object args = __provide_positional_args(
context_args,
injection_args,
injection_args_len,
async_mode,
)
kwargs = __provide_keyword_args(
cdef object kwargs = __provide_keyword_args(
context_kwargs,
injection_kwargs,
injection_kwargs_len,
async_mode,
)
is_future_args = __is_future_or_coroutine(args)
is_future_kwargs = __is_future_or_coroutine(kwargs)
if async_mode == ASYNC_MODE_DISABLED:
return call(*args, **kwargs)
cdef bint is_future_args = __is_future_or_coroutine(args)
cdef bint is_future_kwargs = __is_future_or_coroutine(kwargs)
if is_future_args or is_future_kwargs:
future_args = args if is_future_args else __future_result(args)
@ -609,7 +626,7 @@ cdef inline object __async_result_callback(object future_result, object future):
future_result.set_result(result)
cdef inline object __callable_call(Callable self, tuple args, dict kwargs):
cdef inline object __callable_call(Callable self, tuple args, dict kwargs, ):
return __call(
self.__provides,
args,
@ -618,13 +635,23 @@ cdef inline object __callable_call(Callable self, tuple args, dict kwargs):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
cdef inline object __factory_call(Factory self, tuple args, dict kwargs):
cdef object instance
instance = __callable_call(self.__instantiator, args, kwargs)
instance = __call(
self.__instantiator.__provides,
args,
self.__instantiator.__args,
self.__instantiator.__args_len,
kwargs,
self.__instantiator.__kwargs,
self.__instantiator.__kwargs_len,
self.__async_mode,
)
if self.__attributes_len > 0:
attributes = __provide_attributes(self.__attributes, self.__attributes_len)
@ -643,9 +670,26 @@ cdef inline object __factory_call(Factory self, tuple args, dict kwargs):
cdef inline bint __is_future_or_coroutine(object instance):
if asyncio is None:
return __isfuture(instance) or __iscoroutine(instance)
cdef inline bint __isfuture(object obj):
return hasattr(obj.__class__, "_asyncio_future_blocking") and obj._asyncio_future_blocking is not None
cdef inline bint __iscoroutine(object obj):
if type(obj) in __iscoroutine_typecache:
return True
if isinstance(obj, __COROUTINE_TYPES):
# Just in case we don't want to cache more than 100
# positive types. That shouldn't ever happen, unless
# someone stressing the system on purpose.
if len(__iscoroutine_typecache) < 100:
__iscoroutine_typecache.add(type(obj))
return True
else:
return False
return asyncio.isfuture(instance) or asyncio.iscoroutine(instance)
cdef inline object __future_result(object instance):

31
src/dependency_injector/providers.pyx
View File

@ -145,6 +145,9 @@ cdef int ASYNC_MODE_UNDEFINED = 0
cdef int ASYNC_MODE_ENABLED = 1
cdef int ASYNC_MODE_DISABLED = 2
cdef set __iscoroutine_typecache = set()
cdef tuple __COROUTINE_TYPES = asyncio.coroutines._COROUTINE_TYPES if asyncio else tuple()
cdef class Provider(object):
"""Base provider class.
@ -220,13 +223,13 @@ cdef class Provider(object):
else:
result = self._provide(args, kwargs)
if self.is_async_mode_disabled():
if self.__async_mode == ASYNC_MODE_DISABLED:
return result
elif self.is_async_mode_enabled():
elif self.__async_mode == ASYNC_MODE_ENABLED:
if __is_future_or_coroutine(result):
return result
return __future_result(result)
elif self.is_async_mode_undefined():
elif self.__async_mode == ASYNC_MODE_UNDEFINED:
if __is_future_or_coroutine(result):
self.enable_async_mode()
else:
@ -810,10 +813,10 @@ cdef class Dependency(Provider):
else:
self._raise_undefined_error()
if self.is_async_mode_disabled():
if self.__async_mode == ASYNC_MODE_DISABLED:
self._check_instance_type(result)
return result
elif self.is_async_mode_enabled():
elif self.__async_mode == ASYNC_MODE_ENABLED:
if __is_future_or_coroutine(result):
future_result = asyncio.Future()
result = asyncio.ensure_future(result)
@ -822,7 +825,7 @@ cdef class Dependency(Provider):
else:
self._check_instance_type(result)
return __future_result(result)
elif self.is_async_mode_undefined():
elif self.__async_mode == ASYNC_MODE_UNDEFINED:
if __is_future_or_coroutine(result):
self.enable_async_mode()
@ -3407,7 +3410,7 @@ cdef class List(Provider):
cpdef object _provide(self, tuple args, dict kwargs):
"""Return result of provided callable call."""
return __provide_positional_args(args, self.__args, self.__args_len)
return __provide_positional_args(args, self.__args, self.__args_len, self.__async_mode)
cdef class Dict(Provider):
@ -3533,7 +3536,7 @@ cdef class Dict(Provider):
cpdef object _provide(self, tuple args, dict kwargs):
"""Return result of provided callable call."""
return __provide_keyword_args(kwargs, self.__kwargs, self.__kwargs_len)
return __provide_keyword_args(kwargs, self.__kwargs, self.__kwargs_len, self.__async_mode)
@ -3690,7 +3693,7 @@ cdef class Resource(Provider):
def shutdown(self):
"""Shutdown resource."""
if not self.__initialized:
if self.is_async_mode_enabled():
if self.__async_mode == ASYNC_MODE_ENABLED:
result = asyncio.Future()
result.set_result(None)
return result
@ -3709,7 +3712,7 @@ cdef class Resource(Provider):
self.__initialized = False
self.__shutdowner = None
if self.is_async_mode_enabled():
if self.__async_mode == ASYNC_MODE_ENABLED:
result = asyncio.Future()
result.set_result(None)
return result
@ -3736,6 +3739,7 @@ cdef class Resource(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
self.__shutdowner = initializer.shutdown
elif self._is_async_resource_subclass(self.__provides):
@ -3748,6 +3752,7 @@ cdef class Resource(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
self.__initialized = True
return self._create_init_future(async_init, initializer.shutdown)
@ -3760,6 +3765,7 @@ cdef class Resource(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
self.__resource = next(initializer)
self.__shutdowner = initializer.send
@ -3772,6 +3778,7 @@ cdef class Resource(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
self.__initialized = True
return self._create_init_future(initializer)
@ -3784,6 +3791,7 @@ cdef class Resource(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
self.__initialized = True
return self._create_async_gen_init_future(initializer)
@ -3796,6 +3804,7 @@ cdef class Resource(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
else:
raise Error("Unknown type of resource initializer")
@ -4510,6 +4519,7 @@ cdef class MethodCaller(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
def _async_provide(self, future_result, args, kwargs, future):
@ -4523,6 +4533,7 @@ cdef class MethodCaller(Provider):
kwargs,
self.__kwargs,
self.__kwargs_len,
self.__async_mode,
)
except Exception as exception:
future_result.set_exception(exception)

74
src/dependency_injector/wiring.py
View File

@ -1,7 +1,5 @@
"""Wiring module."""
import asyncio
import functools
import inspect
import importlib
import importlib.machinery
@ -58,7 +56,6 @@ except ImportError:
from . import providers
if sys.version_info[:2] == (3, 5):
warnings.warn(
"Dependency Injector will drop support of Python 3.5 after Jan 1st of 2022. "
@ -600,71 +597,6 @@ def _get_patched(fn, reference_injections, reference_closing):
return patched
def _get_sync_patched(fn):
@functools.wraps(fn)
def _patched(*args, **kwargs):
to_inject = kwargs.copy()
for injection, provider in _patched.__injections__.items():
if injection not in kwargs \
or _is_fastapi_default_arg_injection(injection, kwargs):
to_inject[injection] = provider()
result = fn(*args, **to_inject)
for injection, provider in _patched.__closing__.items():
if injection in kwargs \
and not _is_fastapi_default_arg_injection(injection, kwargs):
continue
if not isinstance(provider, providers.Resource):
continue
provider.shutdown()
return result
return _patched
def _get_async_patched(fn):
@functools.wraps(fn)
async def _patched(*args, **kwargs):
to_inject = kwargs.copy()
to_inject_await = []
to_close_await = []
for injection, provider in _patched.__injections__.items():
if injection not in kwargs \
or _is_fastapi_default_arg_injection(injection, kwargs):
provide = provider()
if inspect.isawaitable(provide):
to_inject_await.append((injection, provide))
else:
to_inject[injection] = provide
async_to_inject = await asyncio.gather(*[provide for _, provide in to_inject_await])
for provide, (injection, _) in zip(async_to_inject, to_inject_await):
to_inject[injection] = provide
result = await fn(*args, **to_inject)
for injection, provider in _patched.__closing__.items():
if injection in kwargs \
and not _is_fastapi_default_arg_injection(injection, kwargs):
continue
if not isinstance(provider, providers.Resource):
continue
shutdown = provider.shutdown()
if inspect.isawaitable(shutdown):
to_close_await.append(shutdown)
await asyncio.gather(*to_close_await)
return result
return _patched
def _is_fastapi_default_arg_injection(injection, kwargs):
"""Check if injection is FastAPI injection of the default argument."""
return injection in kwargs and isinstance(kwargs[injection], _Marker)
def _is_fastapi_depends(param: Any) -> bool:
return fastapi and isinstance(param, fastapi.params.Depends)
@ -828,6 +760,8 @@ class ClassGetItemMeta(GenericMeta):
class _Marker(Generic[T], metaclass=ClassGetItemMeta):
__IS_MARKER__ = True
def __init__(
self,
provider: Union[providers.Provider, Container, str],
@ -958,3 +892,7 @@ def is_loader_installed() -> bool:
_patched_registry = PatchedRegistry()
_inspect_filter = InspectFilter()
_loader = AutoLoader()
# Optimizations
from ._cwiring import _get_sync_patched # noqa
from ._cwiring import _get_async_patched # noqa