Merge branch 'release/4.34.0' into master

This commit is contained in:
Roman Mogylatov 2021-06-24 16:03:37 +03:00
commit 54de3a9d2c
10 changed files with 11997 additions and 11668 deletions

View File

@ -7,6 +7,19 @@ that were made in every particular version.
From version 0.7.6 *Dependency Injector* framework strictly
follows `Semantic versioning`_
4.34.0
------
- Add option ``envs_required`` for configuration provider ``.from_yaml()`` and ``.from_ini()``
methods. With ``envs_required=True`` methods ``.from_yaml()`` and ``.from_ini()`` raise
an exception when encounter an undefined environment variable in the configuration file.
By default this option is set to false for preserving previous behavior ``envs_required=False``.
- Add raising of an exception in configuration provider strict mode when provider encounters
an undefined environment variable in the configuration file.
- Update configuration provider environment variables interpolation to replace
undefined environment variables with an empty value.
- Update configuration provider to perform environment variables interpolation before passing
configuration file content to the parser.
4.33.0
------
- Add support of default value for environment variable in INI and YAML

View File

@ -5,11 +5,14 @@ Configuration provider
.. meta::
:keywords: Python,DI,Dependency injection,IoC,Inversion of Control,Configuration,Injection,
Option,Ini,Json,Yaml,Pydantic,Dict,Environment Variable,Default,Load,Read,Get
Option,Ini,Json,Yaml,Pydantic,Dict,Environment Variable Interpolation,
Environment Variable Substitution,Environment Variable in Config,
Environment Variable in YAML file,Environment Variable in INI file,Default,Load,Read
:description: Configuration provides configuration options to the other providers. This page
demonstrates how to use Configuration provider to inject the dependencies, load
a configuration from an ini or yaml file, a dictionary, an environment variable,
or a pydantic settings object.
or a pydantic settings object. This page also describes how to substitute (interpolate)
environment variables in YAML and INI configuration files.
.. currentmodule:: dependency_injector.providers
@ -42,12 +45,7 @@ where ``examples/providers/configuration/config.ini`` is:
.. literalinclude:: ../../examples/providers/configuration/config.ini
:language: ini
:py:meth:`Configuration.from_ini` method supports environment variables interpolation. Use
``${ENV_NAME}`` format in the configuration file to substitute value from ``ENV_NAME`` environment
variable.
You can also specify a default value using ``${ENV_NAME:default}`` format. If environment
variable ``ENV_NAME`` is undefined, configuration provider will substitute value ``default``.
:py:meth:`Configuration.from_ini` method supports environment variables interpolation.
.. code-block:: ini
@ -56,6 +54,8 @@ variable ``ENV_NAME`` is undefined, configuration provider will substitute value
option2 = {$ENV_VAR}/path
option3 = {$ENV_VAR:default}
See also: :ref:`configuration-envs-interpolation`.
Loading from a YAML file
------------------------
@ -72,12 +72,7 @@ where ``examples/providers/configuration/config.yml`` is:
.. literalinclude:: ../../examples/providers/configuration/config.yml
:language: ini
:py:meth:`Configuration.from_yaml` method supports environment variables interpolation. Use
``${ENV_NAME}`` format in the configuration file to substitute value from ``ENV_NAME`` environment
variable.
You can also specify a default value using ``${ENV_NAME:default}`` format. If environment
variable ``ENV_NAME`` is undefined, configuration provider will substitute value ``default``.
:py:meth:`Configuration.from_yaml` method supports environment variables interpolation.
.. code-block:: ini
@ -86,6 +81,8 @@ variable ``ENV_NAME`` is undefined, configuration provider will substitute value
option2: {$ENV_VAR}/path
option3: {$ENV_VAR:default}
See also: :ref:`configuration-envs-interpolation`.
:py:meth:`Configuration.from_yaml` method uses custom version of ``yaml.SafeLoader``.
To use another loader use ``loader`` argument:
@ -191,6 +188,73 @@ where ``examples/providers/configuration/config.local.yml`` is:
.. literalinclude:: ../../examples/providers/configuration/config.local.yml
:language: ini
.. _configuration-envs-interpolation:
Using environment variables in configuration files
--------------------------------------------------
``Configuration`` provider supports environment variables interpolation in configuration files.
Use ``${ENV_NAME}`` in the configuration file to substitute value from environment
variable ``ENV_NAME``.
.. code-block:: ini
section:
option: ${ENV_NAME}
You can also specify a default value using ``${ENV_NAME:default}`` format. If environment
variable ``ENV_NAME`` is undefined, configuration provider will substitute value ``default``.
.. code-block:: ini
[section]
option = {$ENV_NAME:default}
If you'd like to specify a default value for environment variable inside of the application you can use
``os.environ.setdefault()``.
.. literalinclude:: ../../examples/providers/configuration/configuration_env_interpolation_os_default.py
:language: python
:lines: 3-
:emphasize-lines: 12
If environment variable is undefined and doesn't have a default, ``Configuration`` provider
will replace it with an empty value. This is a default behavior. To raise an error on
undefined environment variable that doesn't have a default value, pass argument
``envs_required=True`` to a configuration reading method:
.. code-block:: python
container.config.from_yaml('config.yml', envs_required=True)
See also: :ref:`configuration-strict-mode`.
.. note::
``Configuration`` provider makes environment variables interpolation before parsing. This preserves
original parser behavior. For instance, undefined environment variable in YAML configuration file
will be replaced with an empty value and then YAML parser will load the file.
Original configuration file:
.. code-block:: ini
section:
option: ${ENV_NAME}
Configuration file after interpolation where ``ENV_NAME`` is undefined:
.. code-block:: ini
section:
option:
Configuration provider after parsing interpolated YAML file contains ``None`` in
option ``section.option``:
.. code-block:: python
assert container.config.section.option() is None
Mandatory and optional sources
------------------------------
@ -310,6 +374,21 @@ configuration data is undefined:
except ValueError:
...
Environment variables interpolation in strict mode raises an exception when encounters
an undefined environment variable without a default value.
.. code-block:: ini
section:
option: {$UNDEFINED}
.. code-block:: python
try:
container.config.from_yaml('undefined_env.yml') # raise exception
except ValueError:
...
You can override ``.from_*()`` methods behaviour in strict mode using ``required`` argument:
.. code-block:: python

View File

@ -0,0 +1,2 @@
section:
option: ${ENV_VAR}

View File

@ -0,0 +1,19 @@
"""`Configuration` provider values loading example."""
import os
from dependency_injector import containers, providers
class Container(containers.DeclarativeContainer):
config = providers.Configuration()
if __name__ == '__main__':
os.environ.setdefault('ENV_VAR', 'default value')
container = Container()
container.config.from_yaml('config-with-env-var.yml')
assert container.config.section.option() == 'default value'

View File

@ -1,6 +1,6 @@
"""Top-level package."""
__version__ = '4.33.0'
__version__ = '4.34.0'
"""Version number.
:type: str

File diff suppressed because it is too large Load Diff

View File

@ -200,8 +200,8 @@ class ConfigurationOption(Provider[Any]):
def required(self) -> ConfigurationOption: ...
def is_required(self) -> bool: ...
def update(self, value: Any) -> None: ...
def from_ini(self, filepath: Union[Path, str], required: bool = False) -> None: ...
def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any]=None) -> None: ...
def from_ini(self, filepath: Union[Path, str], required: bool = False, envs_required: bool = False) -> None: ...
def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any] = None, envs_required: bool = False) -> None: ...
def from_pydantic(self, settings: PydanticSettings, required: bool = False, **kwargs: Any) -> None: ...
def from_dict(self, options: _Dict[str, Any], required: bool = False) -> None: ...
def from_env(self, name: str, default: Optional[Any] = None, required: bool = False) -> None: ...
@ -237,8 +237,8 @@ class Configuration(Object[Any]):
def set(self, selector: str, value: Any) -> OverridingContext[P]: ...
def reset_cache(self) -> None: ...
def update(self, value: Any) -> None: ...
def from_ini(self, filepath: Union[Path, str], required: bool = False) -> None: ...
def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any]=None) -> None: ...
def from_ini(self, filepath: Union[Path, str], required: bool = False, envs_required: bool = False) -> None: ...
def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any] = None, envs_required: bool = False) -> None: ...
def from_pydantic(self, settings: PydanticSettings, required: bool = False, **kwargs: Any) -> None: ...
def from_dict(self, options: _Dict[str, Any], required: bool = False) -> None: ...
def from_env(self, name: str, default: Optional[Any] = None, required: bool = False) -> None: ...

View File

@ -68,68 +68,60 @@ config_env_marker_pattern = re.compile(
r'\${(?P<name>[^}^{:]+)(?P<separator>:?)(?P<default>.*?)}',
)
def _resolve_config_env_markers(config_value):
""""Replace environment variable markers with their values."""
for match in reversed(list(config_env_marker_pattern.finditer(config_value))):
def _resolve_config_env_markers(config_content, envs_required=False):
"""Replace environment variable markers with their values."""
findings = list(config_env_marker_pattern.finditer(config_content))
for match in reversed(findings):
env_name = match.group('name')
has_default = match.group('separator') == ':'
value = os.getenv(match.group('name'))
value = os.getenv(env_name)
if value is None:
if not has_default:
continue
if not has_default and envs_required:
raise ValueError(f'Missing required environment variable "{env_name}"')
value = match.group('default')
span_min, span_max = match.span()
config_value = f'{config_value[:span_min]}{value}{config_value[span_max:]}'
return config_value
config_content = f'{config_content[:span_min]}{value}{config_content[span_max:]}'
return config_content
if sys.version_info[0] == 3:
class EnvInterpolation(iniconfigparser.BasicInterpolation):
"""Interpolation which expands environment variables in values."""
def before_get(self, parser, section, option, value, defaults):
value = super().before_get(parser, section, option, value, defaults)
return _resolve_config_env_markers(value)
def _parse_ini_file(filepath):
parser = iniconfigparser.ConfigParser(interpolation=EnvInterpolation())
def _parse_ini_file(filepath, envs_required=False):
parser = iniconfigparser.ConfigParser()
with open(filepath) as config_file:
parser.read_file(config_file)
config_string = _resolve_config_env_markers(
config_file.read(),
envs_required=envs_required,
)
parser.read_string(config_string)
return parser
else:
import StringIO
def _parse_ini_file(filepath):
def _parse_ini_file(filepath, envs_required=False):
parser = iniconfigparser.ConfigParser()
with open(filepath) as config_file:
config_string = _resolve_config_env_markers(config_file.read())
config_string = _resolve_config_env_markers(
config_file.read(),
envs_required=envs_required,
)
parser.readfp(StringIO.StringIO(config_string))
return parser
if yaml:
# TODO: use SafeLoader without env interpolation by default in version 5.*
def yaml_env_marker_constructor(_, node):
""""Replace environment variable marker with its value."""
return _resolve_config_env_markers(node.value)
yaml.add_implicit_resolver('!path', config_env_marker_pattern)
yaml.add_constructor('!path', yaml_env_marker_constructor)
class YamlLoader(yaml.SafeLoader):
"""Custom YAML loader.
"""YAML loader.
Inherits ``yaml.SafeLoader`` and add environment variables interpolation.
This loader mimics ``yaml.SafeLoader``.
"""
YamlLoader.add_implicit_resolver('!path', config_env_marker_pattern, None)
YamlLoader.add_constructor('!path', yaml_env_marker_constructor)
else:
class YamlLoader:
"""Custom YAML loader.
"""YAML loader.
Inherits ``yaml.SafeLoader`` and add environment variables interpolation.
This loader mimics ``yaml.SafeLoader``.
"""
@ -1535,7 +1527,7 @@ cdef class ConfigurationOption(Provider):
"""
self.override(value)
def from_ini(self, filepath, required=UNDEFINED):
def from_ini(self, filepath, required=UNDEFINED, envs_required=False):
"""Load configuration from the ini file.
Loaded configuration is merged recursively over existing configuration.
@ -1546,10 +1538,16 @@ cdef class ConfigurationOption(Provider):
:param required: When required is True, raise an exception if file does not exist.
:type required: bool
:param envs_required: When True, raises an error on undefined environment variable.
:type envs_required: bool
:rtype: None
"""
try:
parser = _parse_ini_file(filepath)
parser = _parse_ini_file(
filepath,
envs_required=envs_required or self._is_strict_mode_enabled(),
)
except IOError as exception:
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
@ -1567,7 +1565,7 @@ cdef class ConfigurationOption(Provider):
current_config = {}
self.override(merge_dicts(current_config, config))
def from_yaml(self, filepath, required=UNDEFINED, loader=None):
def from_yaml(self, filepath, required=UNDEFINED, loader=None, envs_required=False):
"""Load configuration from the yaml file.
Loaded configuration is merged recursively over existing configuration.
@ -1581,6 +1579,9 @@ cdef class ConfigurationOption(Provider):
:param loader: YAML loader, :py:class:`YamlLoader` is used if not specified.
:type loader: ``yaml.Loader``
:param envs_required: When True, raises an error on undefined environment variable.
:type envs_required: bool
:rtype: None
"""
if yaml is None:
@ -1595,7 +1596,7 @@ cdef class ConfigurationOption(Provider):
try:
with open(filepath) as opened_file:
config = yaml.load(opened_file, loader)
config_content = opened_file.read()
except IOError as exception:
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
@ -1604,6 +1605,12 @@ cdef class ConfigurationOption(Provider):
raise
return
config_content = _resolve_config_env_markers(
config_content,
envs_required=envs_required or self._is_strict_mode_enabled(),
)
config = yaml.load(config_content, loader)
current_config = self.__call__()
if not current_config:
current_config = {}
@ -1956,7 +1963,7 @@ cdef class Configuration(Object):
"""
self.override(value)
def from_ini(self, filepath, required=UNDEFINED):
def from_ini(self, filepath, required=UNDEFINED, envs_required=False):
"""Load configuration from the ini file.
Loaded configuration is merged recursively over existing configuration.
@ -1967,10 +1974,16 @@ cdef class Configuration(Object):
:param required: When required is True, raise an exception if file does not exist.
:type required: bool
:param envs_required: When True, raises an error on undefined environment variable.
:type envs_required: bool
:rtype: None
"""
try:
parser = _parse_ini_file(filepath)
parser = _parse_ini_file(
filepath,
envs_required=envs_required or self._is_strict_mode_enabled(),
)
except IOError as exception:
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
@ -1988,7 +2001,7 @@ cdef class Configuration(Object):
current_config = {}
self.override(merge_dicts(current_config, config))
def from_yaml(self, filepath, required=UNDEFINED, loader=None):
def from_yaml(self, filepath, required=UNDEFINED, loader=None, envs_required=False):
"""Load configuration from the yaml file.
Loaded configuration is merged recursively over existing configuration.
@ -2002,6 +2015,9 @@ cdef class Configuration(Object):
:param loader: YAML loader, :py:class:`YamlLoader` is used if not specified.
:type loader: ``yaml.Loader``
:param envs_required: When True, raises an error on undefined environment variable.
:type envs_required: bool
:rtype: None
"""
if yaml is None:
@ -2016,7 +2032,7 @@ cdef class Configuration(Object):
try:
with open(filepath) as opened_file:
config = yaml.load(opened_file, loader)
config_content = opened_file.read()
except IOError as exception:
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
@ -2025,6 +2041,12 @@ cdef class Configuration(Object):
raise
return
config_content = _resolve_config_env_markers(
config_content,
envs_required=envs_required or self._is_strict_mode_enabled(),
)
config = yaml.load(config_content, loader)
current_config = self.__call__()
if not current_config:
current_config = {}

View File

@ -629,7 +629,7 @@ class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
self.assertEqual(self.config.section1.value1(), 'test-value')
self.assertEqual(self.config.section1.value2(), 'test-path/path')
def test_missing_envs(self):
def test_missing_envs_not_required(self):
del os.environ['CONFIG_TEST_ENV']
del os.environ['CONFIG_TEST_PATH']
@ -639,20 +639,107 @@ class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
self.config(),
{
'section1': {
'value1': '${CONFIG_TEST_ENV}',
'value2': '${CONFIG_TEST_PATH}/path',
'value1': '',
'value2': '/path',
},
},
)
self.assertEqual(
self.config.section1(),
{
'value1': '${CONFIG_TEST_ENV}',
'value2': '${CONFIG_TEST_PATH}/path',
'value1': '',
'value2': '/path',
},
)
self.assertEqual(self.config.section1.value1(), '${CONFIG_TEST_ENV}')
self.assertEqual(self.config.section1.value2(), '${CONFIG_TEST_PATH}/path')
self.assertEqual(self.config.section1.value1(), '')
self.assertEqual(self.config.section1.value2(), '/path')
def test_missing_envs_required(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'[section]\n'
'undefined=${UNDEFINED}\n'
)
with self.assertRaises(ValueError) as context:
self.config.from_ini(self.config_file, envs_required=True)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
def test_missing_envs_strict_mode(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'[section]\n'
'undefined=${UNDEFINED}\n'
)
self.config.set_strict(True)
with self.assertRaises(ValueError) as context:
self.config.from_ini(self.config_file)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
def test_option_missing_envs_not_required(self):
del os.environ['CONFIG_TEST_ENV']
del os.environ['CONFIG_TEST_PATH']
self.config.option.from_ini(self.config_file)
self.assertEqual(
self.config.option(),
{
'section1': {
'value1': '',
'value2': '/path',
},
},
)
self.assertEqual(
self.config.option.section1(),
{
'value1': '',
'value2': '/path',
},
)
self.assertEqual(self.config.option.section1.value1(), '')
self.assertEqual(self.config.option.section1.value2(), '/path')
def test_option_missing_envs_required(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'[section]\n'
'undefined=${UNDEFINED}\n'
)
with self.assertRaises(ValueError) as context:
self.config.option.from_ini(self.config_file, envs_required=True)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
def test_option_missing_envs_strict_mode(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'[section]\n'
'undefined=${UNDEFINED}\n'
)
self.config.set_strict(True)
with self.assertRaises(ValueError) as context:
self.config.option.from_ini(self.config_file)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
def test_default_values(self):
os.environ['DEFINED'] = 'defined'
@ -673,7 +760,7 @@ class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
{
'defined_with_default': 'defined',
'undefined_with_default': 'default',
'complex': 'defined/path/defined/${UNDEFINED}/default',
'complex': 'defined/path/defined//default',
},
)
@ -880,7 +967,7 @@ class ConfigFromYamlWithEnvInterpolationTests(unittest.TestCase):
self.assertEqual(self.config.section1.value2(), 'test-path/path')
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_missing_envs(self):
def test_missing_envs_not_required(self):
del os.environ['CONFIG_TEST_ENV']
del os.environ['CONFIG_TEST_PATH']
@ -890,20 +977,112 @@ class ConfigFromYamlWithEnvInterpolationTests(unittest.TestCase):
self.config(),
{
'section1': {
'value1': '${CONFIG_TEST_ENV}',
'value2': '${CONFIG_TEST_PATH}/path',
'value1': None,
'value2': '/path',
},
},
)
self.assertEqual(
self.config.section1(),
{
'value1': '${CONFIG_TEST_ENV}',
'value2': '${CONFIG_TEST_PATH}/path',
'value1': None,
'value2': '/path',
},
)
self.assertEqual(self.config.section1.value1(), '${CONFIG_TEST_ENV}')
self.assertEqual(self.config.section1.value2(), '${CONFIG_TEST_PATH}/path')
self.assertIsNone(self.config.section1.value1())
self.assertEqual(self.config.section1.value2(), '/path')
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_missing_envs_required(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'section:\n'
' undefined: ${UNDEFINED}\n'
)
with self.assertRaises(ValueError) as context:
self.config.from_yaml(self.config_file, envs_required=True)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_missing_envs_strict_mode(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'section:\n'
' undefined: ${UNDEFINED}\n'
)
self.config.set_strict(True)
with self.assertRaises(ValueError) as context:
self.config.from_yaml(self.config_file)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_option_missing_envs_not_required(self):
del os.environ['CONFIG_TEST_ENV']
del os.environ['CONFIG_TEST_PATH']
self.config.option.from_yaml(self.config_file)
self.assertEqual(
self.config.option(),
{
'section1': {
'value1': None,
'value2': '/path',
},
},
)
self.assertEqual(
self.config.option.section1(),
{
'value1': None,
'value2': '/path',
},
)
self.assertIsNone(self.config.option.section1.value1())
self.assertEqual(self.config.option.section1.value2(), '/path')
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_option_missing_envs_required(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'section:\n'
' undefined: ${UNDEFINED}\n'
)
with self.assertRaises(ValueError) as context:
self.config.option.from_yaml(self.config_file, envs_required=True)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_option_missing_envs_strict_mode(self):
with open(self.config_file, 'w') as config_file:
config_file.write(
'section:\n'
' undefined: ${UNDEFINED}\n'
)
self.config.set_strict(True)
with self.assertRaises(ValueError) as context:
self.config.option.from_yaml(self.config_file)
self.assertEqual(
str(context.exception),
'Missing required environment variable "UNDEFINED"',
)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
def test_default_values(self):
@ -925,7 +1104,7 @@ class ConfigFromYamlWithEnvInterpolationTests(unittest.TestCase):
{
'defined_with_default': 'defined',
'undefined_with_default': 'default',
'complex': 'defined/path/defined/${UNDEFINED}/default',
'complex': 'defined/path/defined//default',
},
)

View File

@ -277,6 +277,7 @@ class TestSchemaMultipleContainersWithInlineProviders(unittest.TestCase):
class TestSchemaBoto3Session(unittest.TestCase):
@unittest.skip('Boto3 tries to connect to the internet')
def test(self):
container = containers.DynamicContainer()
container.from_yaml_schema(f'{_SAMPLES_DIR}/schemasample/container-boto3-session.yml')