369 Add required argument to config from_* methods (#376)

* Update typing stubs

* Update from_yaml() method

* Update from_ini() method

* Update from_dict() method

* Update from_env() method

* Update documentation

* Update changelog

* Update changelog

* Make doc block fix

* Add extra test for from_ini()
This commit is contained in:
Roman Mogylatov 2021-01-24 10:27:45 -05:00 committed by GitHub
parent 2d49308c16
commit 4cc39fc6eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 5509 additions and 4540 deletions

View File

@ -9,12 +9,18 @@ follows `Semantic versioning`_
Development version
-------------------
- Add ``loader`` argument to the configuration provider ``Configuration.from_yaml(..., loader=...)`` to override the
default YAML loader.
- Add ``loader`` argument to the configuration provider ``Configuration.from_yaml(..., loader=...)``
to override the default YAML loader.
Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
- Make security improvement: change default YAML loader to the custom ``yaml.SafeLoader`` with a support
of environment variables interpolation.
Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
- Update configuration provider ``.from_*()`` methods to raise an exception in strict mode if
configuration file does not exist or configuration data is undefined.
Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
- Add ``required`` argument to the configuration provider ``.from_*()`` methods to specify
mandatory configuration sources.
Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
- Fix a bug with asynchronous injections: async providers do not work with async dependencies.
See issue: `#368 <https://github.com/ets-labs/python-dependency-injector/issues/368>`_.
Thanks `@kolypto <https://github.com/kolypto>`_ for the bug report.

View File

@ -127,6 +127,43 @@ where ``examples/providers/configuration/config.local.yml`` is:
.. literalinclude:: ../../examples/providers/configuration/config.local.yml
:language: ini
Mandatory and optional sources
------------------------------
By default, methods ``.from_yaml()`` and ``.from_ini()`` ignore errors if configuration file does not exist.
You can use this to specify optional configuration files.
If configuration file is mandatory, use ``required`` argument. Configuration provider will raise an error
if required file does not exist.
You can also use ``required`` argument when loading configuration from dictionaries and environment variables.
Mandatory YAML file:
.. code-block:: python
container.config.from_yaml('config.yaml', required=True)
Mandatory INI file:
.. code-block:: python
container.config.from_ini('config.ini', required=True)
Mandatory dictionary:
.. code-block:: python
container.config.from_dict(config_dict, required=True)
Mandatory environment variable:
.. code-block:: python
container.config.api_key.from_env('API_KEY', required=True)
See also: :ref:`configuration-strict-mode`.
Specifying the value type
-------------------------
@ -157,6 +194,8 @@ With the ``.as_(callback, *args, **kwargs)`` you can specify a function that wil
before the injection. The value from the config will be passed as a first argument. The returned
value will be injected. Parameters ``*args`` and ``**kwargs`` are handled as any other injections.
.. _configuration-strict-mode:
Strict mode and required options
--------------------------------
@ -183,12 +222,12 @@ configuration data is undefined:
container = Container()
try:
container.config.from_yaml('./does-not_exist.yml') # raise exception
container.config.from_yaml('does-not_exist.yml') # raise exception
except FileNotFoundError:
...
try:
container.config.from_ini('./does-not_exist.ini') # raise exception
container.config.from_ini('does-not_exist.ini') # raise exception
except FileNotFoundError:
...
@ -202,6 +241,21 @@ configuration data is undefined:
except ValueError:
...
You can override ``.from_*()`` methods behaviour in strict mode using ``required`` argument:
.. code-block:: python
class Container(containers.DeclarativeContainer):
config = providers.Configuration(strict=True)
if __name__ == '__main__':
container = Container()
container.config.from_yaml('config.yml')
container.config.from_yaml('config.local.yml', required=False)
You can also use ``.required()`` option modifier when making an injection. It does not require to switch
configuration provider to strict mode.

File diff suppressed because it is too large Load Diff

View File

@ -154,10 +154,10 @@ class ConfigurationOption(Provider[Any]):
def required(self) -> ConfigurationOption: ...
def is_required(self) -> bool: ...
def update(self, value: Any) -> None: ...
def from_ini(self, filepath: Union[Path, str]) -> None: ...
def from_yaml(self, filepath: Union[Path, str], loader: Optional[Any]=None) -> None: ...
def from_dict(self, options: _Dict[str, Any]) -> None: ...
def from_env(self, name: str, default: Optional[Any] = None) -> None: ...
def from_ini(self, filepath: Union[Path, str], required: bool = False) -> None: ...
def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any]=None) -> None: ...
def from_dict(self, options: _Dict[str, Any], required: bool = False) -> None: ...
def from_env(self, name: str, default: Optional[Any] = None, required: bool = False) -> None: ...
class TypedConfigurationOption(Callable[T]):
@ -175,10 +175,10 @@ class Configuration(Object[Any]):
def set(self, selector: str, value: Any) -> OverridingContext: ...
def reset_cache(self) -> None: ...
def update(self, value: Any) -> None: ...
def from_ini(self, filepath: Union[Path, str]) -> None: ...
def from_yaml(self, filepath: Union[Path, str], loader: Optional[Any]=None) -> None: ...
def from_dict(self, options: _Dict[str, Any]) -> None: ...
def from_env(self, name: str, default: Optional[Any] = None) -> None: ...
def from_ini(self, filepath: Union[Path, str], required: bool = False) -> None: ...
def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any]=None) -> None: ...
def from_dict(self, options: _Dict[str, Any], required: bool = False) -> None: ...
def from_env(self, name: str, default: Optional[Any] = None, required: bool = False) -> None: ...
class Factory(Provider[T]):

View File

@ -1312,7 +1312,7 @@ cdef class ConfigurationOption(Provider):
"""
self.override(value)
def from_ini(self, filepath):
def from_ini(self, filepath, required=UNDEFINED):
"""Load configuration from the ini file.
Loaded configuration is merged recursively over existing configuration.
@ -1320,12 +1320,17 @@ cdef class ConfigurationOption(Provider):
:param filepath: Path to the configuration file.
:type filepath: str
:param required: When required is True, raise an exception if file does not exist.
:type required: bool
:rtype: None
"""
try:
parser = _parse_ini_file(filepath)
except IOError as exception:
if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
and exception.errno in (errno.ENOENT, errno.EISDIR):
exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
raise
return
@ -1339,7 +1344,7 @@ cdef class ConfigurationOption(Provider):
current_config = {}
self.override(merge_dicts(current_config, config))
def from_yaml(self, filepath, loader=None):
def from_yaml(self, filepath, required=UNDEFINED, loader=None):
"""Load configuration from the yaml file.
Loaded configuration is merged recursively over existing configuration.
@ -1347,6 +1352,9 @@ cdef class ConfigurationOption(Provider):
:param filepath: Path to the configuration file.
:type filepath: str
:param required: When required is True, raise an exception if file does not exist.
:type required: bool
:param loader: YAML loader, :py:class:`YamlLoader` is used if not specified.
:type loader: ``yaml.Loader``
@ -1367,7 +1375,9 @@ cdef class ConfigurationOption(Provider):
with open(filepath) as opened_file:
config = yaml.load(opened_file, loader)
except IOError as exception:
if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
and exception.errno in (errno.ENOENT, errno.EISDIR):
exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
raise
return
@ -1377,7 +1387,7 @@ cdef class ConfigurationOption(Provider):
current_config = {}
self.override(merge_dicts(current_config, config))
def from_dict(self, options):
def from_dict(self, options, required=UNDEFINED):
"""Load configuration from the dictionary.
Loaded configuration is merged recursively over existing configuration.
@ -1385,17 +1395,27 @@ cdef class ConfigurationOption(Provider):
:param options: Configuration options.
:type options: dict
:param required: When required is True, raise an exception if dictionary is empty.
:type required: bool
:rtype: None
"""
if self._is_strict_mode_enabled() and not options:
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
and not options:
raise ValueError('Can not use empty dictionary')
current_config = self.__call__()
if not current_config:
try:
current_config = self.__call__()
except Error:
current_config = {}
else:
if not current_config:
current_config = {}
self.override(merge_dicts(current_config, options))
def from_env(self, name, default=UNDEFINED):
def from_env(self, name, default=UNDEFINED, required=UNDEFINED):
"""Load configuration value from the environment variable.
:param name: Name of the environment variable.
@ -1404,12 +1424,16 @@ cdef class ConfigurationOption(Provider):
:param default: Default value that is used if environment variable does not exist.
:type default: object
:param required: When required is True, raise an exception if environment variable is undefined.
:type required: bool
:rtype: None
"""
value = os.environ.get(name, default)
if value is UNDEFINED:
if self._is_strict_mode_enabled():
if required is not False \
and (self._is_strict_mode_enabled() or required is True):
raise ValueError('Environment variable "{0}" is undefined'.format(name))
value = None
@ -1621,7 +1645,7 @@ cdef class Configuration(Object):
"""
self.override(value)
def from_ini(self, filepath):
def from_ini(self, filepath, required=UNDEFINED):
"""Load configuration from the ini file.
Loaded configuration is merged recursively over existing configuration.
@ -1629,12 +1653,17 @@ cdef class Configuration(Object):
:param filepath: Path to the configuration file.
:type filepath: str
:param required: When required is True, raise an exception if file does not exist.
:type required: bool
:rtype: None
"""
try:
parser = _parse_ini_file(filepath)
except IOError as exception:
if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
and exception.errno in (errno.ENOENT, errno.EISDIR):
exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
raise
return
@ -1648,7 +1677,7 @@ cdef class Configuration(Object):
current_config = {}
self.override(merge_dicts(current_config, config))
def from_yaml(self, filepath, loader=None):
def from_yaml(self, filepath, required=UNDEFINED, loader=None):
"""Load configuration from the yaml file.
Loaded configuration is merged recursively over existing configuration.
@ -1656,6 +1685,9 @@ cdef class Configuration(Object):
:param filepath: Path to the configuration file.
:type filepath: str
:param required: When required is True, raise an exception if file does not exist.
:type required: bool
:param loader: YAML loader, :py:class:`YamlLoader` is used if not specified.
:type loader: ``yaml.Loader``
@ -1675,7 +1707,9 @@ cdef class Configuration(Object):
with open(filepath) as opened_file:
config = yaml.load(opened_file, loader)
except IOError as exception:
if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
and exception.errno in (errno.ENOENT, errno.EISDIR):
exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
raise
return
@ -1685,7 +1719,7 @@ cdef class Configuration(Object):
current_config = {}
self.override(merge_dicts(current_config, config))
def from_dict(self, options):
def from_dict(self, options, required=UNDEFINED):
"""Load configuration from the dictionary.
Loaded configuration is merged recursively over existing configuration.
@ -1693,9 +1727,14 @@ cdef class Configuration(Object):
:param options: Configuration options.
:type options: dict
:param required: When required is True, raise an exception if dictionary is empty.
:type required: bool
:rtype: None
"""
if self._is_strict_mode_enabled() and not options:
if required is not False \
and (self._is_strict_mode_enabled() or required is True) \
and not options:
raise ValueError('Can not use empty dictionary')
current_config = self.__call__()
@ -1703,7 +1742,7 @@ cdef class Configuration(Object):
current_config = {}
self.override(merge_dicts(current_config, options))
def from_env(self, name, default=UNDEFINED):
def from_env(self, name, default=UNDEFINED, required=UNDEFINED):
"""Load configuration value from the environment variable.
:param name: Name of the environment variable.
@ -1712,12 +1751,16 @@ cdef class Configuration(Object):
:param default: Default value that is used if environment variable does not exist.
:type default: object
:param required: When required is True, raise an exception if environment variable is undefined.
:type required: bool
:rtype: None
"""
value = os.environ.get(name, default)
if value is UNDEFINED:
if self._is_strict_mode_enabled():
if required is not False \
and (self._is_strict_mode_enabled() or required is True):
raise ValueError('Environment variable "{0}" is undefined'.format(name))
value = None

View File

@ -399,6 +399,16 @@ class ConfigFromIniTests(unittest.TestCase):
self.assertEqual(self.config.section2(), {'value2': '2'})
self.assertEqual(self.config.section2.value2(), '2')
def test_option(self):
self.config.option.from_ini(self.config_file_1)
self.assertEqual(self.config(), {'option': {'section1': {'value1': '1'}, 'section2': {'value2': '2'}}})
self.assertEqual(self.config.option(), {'section1': {'value1': '1'}, 'section2': {'value2': '2'}})
self.assertEqual(self.config.option.section1(), {'value1': '1'})
self.assertEqual(self.config.option.section1.value1(), '1')
self.assertEqual(self.config.option.section2(), {'value2': '2'})
self.assertEqual(self.config.option.section2.value2(), '2')
def test_merge(self):
self.config.from_ini(self.config_file_1)
self.config.from_ini(self.config_file_2)
@ -444,6 +454,25 @@ class ConfigFromIniTests(unittest.TestCase):
with self.assertRaises(IOError):
self.config.option.from_ini('./does_not_exist.ini')
def test_required_file_does_not_exist(self):
with self.assertRaises(IOError):
self.config.from_ini('./does_not_exist.ini', required=True)
def test_required_option_file_does_not_exist(self):
with self.assertRaises(IOError):
self.config.option.from_ini('./does_not_exist.ini', required=True)
def test_not_required_file_does_not_exist_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.from_ini('./does_not_exist.ini', required=False)
self.assertEqual(self.config(), {})
def test_not_required_option_file_does_not_exist_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.option.from_ini('./does_not_exist.ini', required=False)
with self.assertRaises(errors.Error):
self.config.option()
class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
@ -565,6 +594,25 @@ class ConfigFromYamlTests(unittest.TestCase):
with self.assertRaises(IOError):
self.config.option.from_yaml('./does_not_exist.yml')
def test_required_file_does_not_exist(self):
with self.assertRaises(IOError):
self.config.from_yaml('./does_not_exist.yml', required=True)
def test_required_option_file_does_not_exist(self):
with self.assertRaises(IOError):
self.config.option.from_yaml('./does_not_exist.yml', required=True)
def test_not_required_file_does_not_exist_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.from_yaml('./does_not_exist.yml', required=False)
self.assertEqual(self.config(), {})
def test_not_required_option_file_does_not_exist_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.option.from_yaml('./does_not_exist.yml', required=False)
with self.assertRaises(errors.Error):
self.config.option()
def test_no_yaml_installed(self):
@contextlib.contextmanager
def no_yaml_module():
@ -699,24 +747,6 @@ class ConfigFromDict(unittest.TestCase):
self.assertEqual(self.config.section2(), {'value2': '2'})
self.assertEqual(self.config.section2.value2(), '2')
def test_empty_dict(self):
self.config.from_dict({})
self.assertEqual(self.config(), {})
def test_option_empty_dict(self):
self.config.option.from_dict({})
self.assertEqual(self.config.option(), {})
def test_empty_dict_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
with self.assertRaises(ValueError):
self.config.from_dict({})
def test_option_empty_dict_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
with self.assertRaises(ValueError):
self.config.option.from_dict({})
def test_merge(self):
self.config.from_dict(self.config_options_1)
self.config.from_dict(self.config_options_2)
@ -744,6 +774,43 @@ class ConfigFromDict(unittest.TestCase):
self.assertEqual(self.config.section3(), {'value3': '3'})
self.assertEqual(self.config.section3.value3(), '3')
def test_empty_dict(self):
self.config.from_dict({})
self.assertEqual(self.config(), {})
def test_option_empty_dict(self):
self.config.option.from_dict({})
self.assertEqual(self.config.option(), {})
def test_empty_dict_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
with self.assertRaises(ValueError):
self.config.from_dict({})
def test_option_empty_dict_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
with self.assertRaises(ValueError):
self.config.option.from_dict({})
def test_required_empty_dict(self):
with self.assertRaises(ValueError):
self.config.from_dict({}, required=True)
def test_required_option_empty_dict(self):
with self.assertRaises(ValueError):
self.config.option.from_dict({}, required=True)
def test_not_required_empty_dict_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.from_dict({}, required=False)
self.assertEqual(self.config(), {})
def test_not_required_option_empty_dict_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.option.from_dict({}, required=False)
self.assertEqual(self.config.option(), {})
self.assertEqual(self.config(), {'option': {}})
class ConfigFromEnvTests(unittest.TestCase):
@ -759,10 +826,25 @@ class ConfigFromEnvTests(unittest.TestCase):
self.config.from_env('CONFIG_TEST_ENV')
self.assertEqual(self.config(), 'test-value')
def test_with_children(self):
self.config.section1.value1.from_env('CONFIG_TEST_ENV')
self.assertEqual(self.config(), {'section1': {'value1': 'test-value'}})
self.assertEqual(self.config.section1(), {'value1': 'test-value'})
self.assertEqual(self.config.section1.value1(), 'test-value')
def test_default(self):
self.config.from_env('UNDEFINED_ENV', 'default-value')
self.assertEqual(self.config(), 'default-value')
def test_default_none(self):
self.config.from_env('UNDEFINED_ENV')
self.assertIsNone(self.config())
def test_option_default_none(self):
self.config.option.from_env('UNDEFINED_ENV')
self.assertIsNone(self.config.option())
def test_undefined_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
with self.assertRaises(ValueError):
@ -783,17 +865,38 @@ class ConfigFromEnvTests(unittest.TestCase):
self.config.option.from_env('UNDEFINED_ENV', 'default-value')
self.assertEqual(self.config.option(), 'default-value')
def test_default_none(self):
self.config.from_env('UNDEFINED_ENV')
def test_required_undefined(self):
with self.assertRaises(ValueError):
self.config.from_env('UNDEFINED_ENV', required=True)
def test_required_undefined_with_default(self):
self.config.from_env('UNDEFINED_ENV', default='default-value', required=True)
self.assertEqual(self.config(), 'default-value')
def test_option_required_undefined(self):
with self.assertRaises(ValueError):
self.config.option.from_env('UNDEFINED_ENV', required=True)
def test_option_required_undefined_with_default(self):
self.config.option.from_env('UNDEFINED_ENV', default='default-value', required=True)
self.assertEqual(self.config.option(), 'default-value')
def test_not_required_undefined_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.from_env('UNDEFINED_ENV', required=False)
self.assertIsNone(self.config())
def test_option_default_none(self):
self.config.option.from_env('UNDEFINED_ENV')
def test_option_not_required_undefined_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.option.from_env('UNDEFINED_ENV', required=False)
self.assertIsNone(self.config.option())
def test_with_children(self):
self.config.section1.value1.from_env('CONFIG_TEST_ENV')
def test_not_required_undefined_with_default_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.from_env('UNDEFINED_ENV', default='default-value', required=False)
self.assertEqual(self.config(), 'default-value')
self.assertEqual(self.config(), {'section1': {'value1': 'test-value'}})
self.assertEqual(self.config.section1(), {'value1': 'test-value'})
self.assertEqual(self.config.section1.value1(), 'test-value')
def test_option_not_required_undefined_with_default_in_strict_mode(self):
self.config = providers.Configuration(strict=True)
self.config.option.from_env('UNDEFINED_ENV', default='default-value', required=False)
self.assertEqual(self.config.option(), 'default-value')