369 Add required argument to config from_* methods (#376)

* Update typing stubs

* Update from_yaml() method

* Update from_ini() method

* Update from_dict() method

* Update from_env() method

* Update documentation

* Update changelog

* Update changelog

* Make doc block fix

* Add extra test for from_ini()

docs/main/changelog.rst

@@ -9,12 +9,18 @@ follows `Semantic versioning`_
 
 Development version
 -------------------
-- Add ``loader`` argument to the configuration provider ``Configuration.from_yaml(..., loader=...)`` to override the
-  default YAML loader.
+- Add ``loader`` argument to the configuration provider ``Configuration.from_yaml(..., loader=...)``
+  to override the default YAML loader.
+  Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
 - Make security improvement: change default YAML loader to the custom ``yaml.SafeLoader`` with a support
   of environment variables interpolation.
+  Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
 - Update configuration provider ``.from_*()`` methods to raise an exception in strict mode if
   configuration file does not exist or configuration data is undefined.
+  Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
+- Add ``required`` argument to the configuration provider ``.from_*()`` methods to specify
+  mandatory configuration sources.
+  Many thanks to `Stefano Frazzetto <https://github.com/StefanoFrazzetto>`_ for suggesting an improvement.
 - Fix a bug with asynchronous injections: async providers do not work with async dependencies.
   See issue: `#368 <https://github.com/ets-labs/python-dependency-injector/issues/368>`_.
   Thanks `@kolypto <https://github.com/kolypto>`_ for the bug report.

docs/providers/configuration.rst

@@ -127,6 +127,43 @@ where ``examples/providers/configuration/config.local.yml`` is:
 .. literalinclude:: ../../examples/providers/configuration/config.local.yml
    :language: ini
 
+Mandatory and optional sources
+------------------------------
+
+By default, methods ``.from_yaml()`` and ``.from_ini()`` ignore errors if configuration file does not exist.
+You can use this to specify optional configuration files.
+
+If configuration file is mandatory, use ``required`` argument. Configuration provider will raise an error
+if required file does not exist.
+
+You can also use ``required`` argument when loading configuration from dictionaries and environment variables.
+
+Mandatory YAML file:
+
+.. code-block:: python
+
+   container.config.from_yaml('config.yaml', required=True)
+
+Mandatory INI file:
+
+.. code-block:: python
+
+   container.config.from_ini('config.ini', required=True)
+
+Mandatory dictionary:
+
+.. code-block:: python
+
+   container.config.from_dict(config_dict, required=True)
+
+Mandatory environment variable:
+
+.. code-block:: python
+
+   container.config.api_key.from_env('API_KEY', required=True)
+
+See also: :ref:`configuration-strict-mode`.
+
 Specifying the value type
 -------------------------
 
@@ -157,6 +194,8 @@ With the ``.as_(callback, *args, **kwargs)`` you can specify a function that wil
 before the injection. The value from the config will be passed as a first argument. The returned
 value will be injected. Parameters ``*args`` and ``**kwargs`` are handled as any other injections.
 
+.. _configuration-strict-mode:
+
 Strict mode and required options
 --------------------------------
 
@@ -183,12 +222,12 @@ configuration data is undefined:
        container = Container()
 
        try:
-           container.config.from_yaml('./does-not_exist.yml')  # raise exception
+           container.config.from_yaml('does-not_exist.yml')  # raise exception
        except FileNotFoundError:
            ...
 
        try:
-           container.config.from_ini('./does-not_exist.ini')  # raise exception
+           container.config.from_ini('does-not_exist.ini')  # raise exception
        except FileNotFoundError:
            ...
 
@@ -202,6 +241,21 @@ configuration data is undefined:
        except ValueError:
            ...
 
+You can override ``.from_*()`` methods behaviour in strict mode using ``required`` argument:
+
+.. code-block:: python
+
+   class Container(containers.DeclarativeContainer):
+
+       config = providers.Configuration(strict=True)
+
+
+   if __name__ == '__main__':
+       container = Container()
+
+       container.config.from_yaml('config.yml')
+       container.config.from_yaml('config.local.yml', required=False)
+
 You can also use ``.required()`` option modifier when making an injection. It does not require to switch
 configuration provider to strict mode.
 

src/dependency_injector/providers.pyi

@@ -154,10 +154,10 @@ class ConfigurationOption(Provider[Any]):
     def required(self) -> ConfigurationOption: ...
     def is_required(self) -> bool: ...
     def update(self, value: Any) -> None: ...
-    def from_ini(self, filepath: Union[Path, str]) -> None: ...
-    def from_yaml(self, filepath: Union[Path, str], loader: Optional[Any]=None) -> None: ...
-    def from_dict(self, options: _Dict[str, Any]) -> None: ...
-    def from_env(self, name: str, default: Optional[Any] = None) -> None: ...
+    def from_ini(self, filepath: Union[Path, str], required: bool = False) -> None: ...
+    def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any]=None) -> None: ...
+    def from_dict(self, options: _Dict[str, Any], required: bool = False) -> None: ...
+    def from_env(self, name: str, default: Optional[Any] = None, required: bool = False) -> None: ...
 
 
 class TypedConfigurationOption(Callable[T]):
@@ -175,10 +175,10 @@ class Configuration(Object[Any]):
     def set(self, selector: str, value: Any) -> OverridingContext: ...
     def reset_cache(self) -> None: ...
     def update(self, value: Any) -> None: ...
-    def from_ini(self, filepath: Union[Path, str]) -> None: ...
-    def from_yaml(self, filepath: Union[Path, str], loader: Optional[Any]=None) -> None: ...
-    def from_dict(self, options: _Dict[str, Any]) -> None: ...
-    def from_env(self, name: str, default: Optional[Any] = None) -> None: ...
+    def from_ini(self, filepath: Union[Path, str], required: bool = False) -> None: ...
+    def from_yaml(self, filepath: Union[Path, str], required: bool = False, loader: Optional[Any]=None) -> None: ...
+    def from_dict(self, options: _Dict[str, Any], required: bool = False) -> None: ...
+    def from_env(self, name: str, default: Optional[Any] = None, required: bool = False) -> None: ...
 
 
 class Factory(Provider[T]):

src/dependency_injector/providers.pyx

@@ -1312,7 +1312,7 @@ cdef class ConfigurationOption(Provider):
         """
         self.override(value)
 
-    def from_ini(self, filepath):
+    def from_ini(self, filepath, required=UNDEFINED):
         """Load configuration from the ini file.
 
         Loaded configuration is merged recursively over existing configuration.
@@ -1320,12 +1320,17 @@ cdef class ConfigurationOption(Provider):
         :param filepath: Path to the configuration file.
         :type filepath: str
 
+        :param required: When required is True, raise an exception if file does not exist.
+        :type required: bool
+
         :rtype: None
         """
         try:
             parser = _parse_ini_file(filepath)
         except IOError as exception:
-            if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
+            if required is not False \
+                    and (self._is_strict_mode_enabled() or required is True) \
+                    and exception.errno in (errno.ENOENT, errno.EISDIR):
                 exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
                 raise
             return
@@ -1339,7 +1344,7 @@ cdef class ConfigurationOption(Provider):
             current_config = {}
         self.override(merge_dicts(current_config, config))
 
-    def from_yaml(self, filepath, loader=None):
+    def from_yaml(self, filepath, required=UNDEFINED, loader=None):
         """Load configuration from the yaml file.
 
         Loaded configuration is merged recursively over existing configuration.
@@ -1347,6 +1352,9 @@ cdef class ConfigurationOption(Provider):
         :param filepath: Path to the configuration file.
         :type filepath: str
 
+        :param required: When required is True, raise an exception if file does not exist.
+        :type required: bool
+
         :param loader: YAML loader, :py:class:`YamlLoader` is used if not specified.
         :type loader: ``yaml.Loader``
 
@@ -1367,7 +1375,9 @@ cdef class ConfigurationOption(Provider):
             with open(filepath) as opened_file:
                 config = yaml.load(opened_file, loader)
         except IOError as exception:
-            if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
+            if required is not False \
+                    and (self._is_strict_mode_enabled() or required is True) \
+                    and exception.errno in (errno.ENOENT, errno.EISDIR):
                 exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
                 raise
             return
@@ -1377,7 +1387,7 @@ cdef class ConfigurationOption(Provider):
             current_config = {}
         self.override(merge_dicts(current_config, config))
 
-    def from_dict(self, options):
+    def from_dict(self, options, required=UNDEFINED):
         """Load configuration from the dictionary.
 
         Loaded configuration is merged recursively over existing configuration.
@@ -1385,17 +1395,27 @@ cdef class ConfigurationOption(Provider):
         :param options: Configuration options.
         :type options: dict
 
+        :param required: When required is True, raise an exception if dictionary is empty.
+        :type required: bool
+
         :rtype: None
         """
-        if self._is_strict_mode_enabled() and not options:
+        if required is not False \
+                and (self._is_strict_mode_enabled() or required is True) \
+                and not options:
             raise ValueError('Can not use empty dictionary')
 
+        try:
             current_config = self.__call__()
+        except Error:
+            current_config = {}
+        else:
             if not current_config:
                 current_config = {}
+
         self.override(merge_dicts(current_config, options))
 
-    def from_env(self, name, default=UNDEFINED):
+    def from_env(self, name, default=UNDEFINED, required=UNDEFINED):
         """Load configuration value from the environment variable.
 
         :param name: Name of the environment variable.
@@ -1404,12 +1424,16 @@ cdef class ConfigurationOption(Provider):
         :param default: Default value that is used if environment variable does not exist.
         :type default: object
 
+        :param required: When required is True, raise an exception if environment variable is undefined.
+        :type required: bool
+
         :rtype: None
         """
         value = os.environ.get(name, default)
 
         if value is UNDEFINED:
-            if self._is_strict_mode_enabled():
+            if required is not False \
+                    and (self._is_strict_mode_enabled() or required is True):
                 raise ValueError('Environment variable "{0}" is undefined'.format(name))
             value = None
 
@@ -1621,7 +1645,7 @@ cdef class Configuration(Object):
         """
         self.override(value)
 
-    def from_ini(self, filepath):
+    def from_ini(self, filepath, required=UNDEFINED):
         """Load configuration from the ini file.
 
         Loaded configuration is merged recursively over existing configuration.
@@ -1629,12 +1653,17 @@ cdef class Configuration(Object):
         :param filepath: Path to the configuration file.
         :type filepath: str
 
+        :param required: When required is True, raise an exception if file does not exist.
+        :type required: bool
+
         :rtype: None
         """
         try:
             parser = _parse_ini_file(filepath)
         except IOError as exception:
-            if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
+            if required is not False \
+                    and (self._is_strict_mode_enabled() or required is True) \
+                    and exception.errno in (errno.ENOENT, errno.EISDIR):
                 exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
                 raise
             return
@@ -1648,7 +1677,7 @@ cdef class Configuration(Object):
             current_config = {}
         self.override(merge_dicts(current_config, config))
 
-    def from_yaml(self, filepath, loader=None):
+    def from_yaml(self, filepath, required=UNDEFINED, loader=None):
         """Load configuration from the yaml file.
 
         Loaded configuration is merged recursively over existing configuration.
@@ -1656,6 +1685,9 @@ cdef class Configuration(Object):
         :param filepath: Path to the configuration file.
         :type filepath: str
 
+        :param required: When required is True, raise an exception if file does not exist.
+        :type required: bool
+
         :param loader: YAML loader, :py:class:`YamlLoader` is used if not specified.
         :type loader: ``yaml.Loader``
 
@@ -1675,7 +1707,9 @@ cdef class Configuration(Object):
             with open(filepath) as opened_file:
                 config = yaml.load(opened_file, loader)
         except IOError as exception:
-            if self._is_strict_mode_enabled() and exception.errno in (errno.ENOENT, errno.EISDIR):
+            if required is not False \
+                    and (self._is_strict_mode_enabled() or required is True) \
+                    and exception.errno in (errno.ENOENT, errno.EISDIR):
                 exception.strerror = 'Unable to load configuration file {0}'.format(exception.strerror)
                 raise
             return
@@ -1685,7 +1719,7 @@ cdef class Configuration(Object):
             current_config = {}
         self.override(merge_dicts(current_config, config))
 
-    def from_dict(self, options):
+    def from_dict(self, options, required=UNDEFINED):
         """Load configuration from the dictionary.
 
         Loaded configuration is merged recursively over existing configuration.
@@ -1693,9 +1727,14 @@ cdef class Configuration(Object):
         :param options: Configuration options.
         :type options: dict
 
+        :param required: When required is True, raise an exception if dictionary is empty.
+        :type required: bool
+
         :rtype: None
         """
-        if self._is_strict_mode_enabled() and not options:
+        if required is not False \
+                and (self._is_strict_mode_enabled() or required is True) \
+                and not options:
             raise ValueError('Can not use empty dictionary')
 
         current_config = self.__call__()
@@ -1703,7 +1742,7 @@ cdef class Configuration(Object):
             current_config = {}
         self.override(merge_dicts(current_config, options))
 
-    def from_env(self, name, default=UNDEFINED):
+    def from_env(self, name, default=UNDEFINED, required=UNDEFINED):
         """Load configuration value from the environment variable.
 
         :param name: Name of the environment variable.
@@ -1712,12 +1751,16 @@ cdef class Configuration(Object):
         :param default: Default value that is used if environment variable does not exist.
         :type default: object
 
+        :param required: When required is True, raise an exception if environment variable is undefined.
+        :type required: bool
+
         :rtype: None
         """
         value = os.environ.get(name, default)
 
         if value is UNDEFINED:
-            if self._is_strict_mode_enabled():
+            if required is not False \
+                    and (self._is_strict_mode_enabled() or required is True):
                 raise ValueError('Environment variable "{0}" is undefined'.format(name))
             value = None
 

tests/unit/providers/test_configuration_py2_py3.py

@@ -399,6 +399,16 @@ class ConfigFromIniTests(unittest.TestCase):
         self.assertEqual(self.config.section2(), {'value2': '2'})
         self.assertEqual(self.config.section2.value2(), '2')
 
+    def test_option(self):
+        self.config.option.from_ini(self.config_file_1)
+
+        self.assertEqual(self.config(), {'option': {'section1': {'value1': '1'}, 'section2': {'value2': '2'}}})
+        self.assertEqual(self.config.option(), {'section1': {'value1': '1'}, 'section2': {'value2': '2'}})
+        self.assertEqual(self.config.option.section1(), {'value1': '1'})
+        self.assertEqual(self.config.option.section1.value1(), '1')
+        self.assertEqual(self.config.option.section2(), {'value2': '2'})
+        self.assertEqual(self.config.option.section2.value2(), '2')
+
     def test_merge(self):
         self.config.from_ini(self.config_file_1)
         self.config.from_ini(self.config_file_2)
@@ -444,6 +454,25 @@ class ConfigFromIniTests(unittest.TestCase):
         with self.assertRaises(IOError):
             self.config.option.from_ini('./does_not_exist.ini')
 
+    def test_required_file_does_not_exist(self):
+        with self.assertRaises(IOError):
+            self.config.from_ini('./does_not_exist.ini', required=True)
+
+    def test_required_option_file_does_not_exist(self):
+        with self.assertRaises(IOError):
+            self.config.option.from_ini('./does_not_exist.ini', required=True)
+
+    def test_not_required_file_does_not_exist_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.from_ini('./does_not_exist.ini', required=False)
+        self.assertEqual(self.config(), {})
+
+    def test_not_required_option_file_does_not_exist_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.option.from_ini('./does_not_exist.ini', required=False)
+        with self.assertRaises(errors.Error):
+            self.config.option()
+
 
 class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
 
@@ -565,6 +594,25 @@ class ConfigFromYamlTests(unittest.TestCase):
         with self.assertRaises(IOError):
             self.config.option.from_yaml('./does_not_exist.yml')
 
+    def test_required_file_does_not_exist(self):
+        with self.assertRaises(IOError):
+            self.config.from_yaml('./does_not_exist.yml', required=True)
+
+    def test_required_option_file_does_not_exist(self):
+        with self.assertRaises(IOError):
+            self.config.option.from_yaml('./does_not_exist.yml', required=True)
+
+    def test_not_required_file_does_not_exist_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.from_yaml('./does_not_exist.yml', required=False)
+        self.assertEqual(self.config(), {})
+
+    def test_not_required_option_file_does_not_exist_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.option.from_yaml('./does_not_exist.yml', required=False)
+        with self.assertRaises(errors.Error):
+            self.config.option()
+
     def test_no_yaml_installed(self):
         @contextlib.contextmanager
         def no_yaml_module():
@@ -699,24 +747,6 @@ class ConfigFromDict(unittest.TestCase):
         self.assertEqual(self.config.section2(), {'value2': '2'})
         self.assertEqual(self.config.section2.value2(), '2')
 
-    def test_empty_dict(self):
-        self.config.from_dict({})
-        self.assertEqual(self.config(), {})
-
-    def test_option_empty_dict(self):
-        self.config.option.from_dict({})
-        self.assertEqual(self.config.option(), {})
-
-    def test_empty_dict_in_strict_mode(self):
-        self.config = providers.Configuration(strict=True)
-        with self.assertRaises(ValueError):
-            self.config.from_dict({})
-
-    def test_option_empty_dict_in_strict_mode(self):
-        self.config = providers.Configuration(strict=True)
-        with self.assertRaises(ValueError):
-            self.config.option.from_dict({})
-
     def test_merge(self):
         self.config.from_dict(self.config_options_1)
         self.config.from_dict(self.config_options_2)
@@ -744,6 +774,43 @@ class ConfigFromDict(unittest.TestCase):
         self.assertEqual(self.config.section3(), {'value3': '3'})
         self.assertEqual(self.config.section3.value3(), '3')
 
+    def test_empty_dict(self):
+        self.config.from_dict({})
+        self.assertEqual(self.config(), {})
+
+    def test_option_empty_dict(self):
+        self.config.option.from_dict({})
+        self.assertEqual(self.config.option(), {})
+
+    def test_empty_dict_in_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        with self.assertRaises(ValueError):
+            self.config.from_dict({})
+
+    def test_option_empty_dict_in_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        with self.assertRaises(ValueError):
+            self.config.option.from_dict({})
+
+    def test_required_empty_dict(self):
+        with self.assertRaises(ValueError):
+            self.config.from_dict({}, required=True)
+
+    def test_required_option_empty_dict(self):
+        with self.assertRaises(ValueError):
+            self.config.option.from_dict({}, required=True)
+
+    def test_not_required_empty_dict_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.from_dict({}, required=False)
+        self.assertEqual(self.config(), {})
+
+    def test_not_required_option_empty_dict_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.option.from_dict({}, required=False)
+        self.assertEqual(self.config.option(), {})
+        self.assertEqual(self.config(), {'option': {}})
+
 
 class ConfigFromEnvTests(unittest.TestCase):
 
@@ -759,10 +826,25 @@ class ConfigFromEnvTests(unittest.TestCase):
         self.config.from_env('CONFIG_TEST_ENV')
         self.assertEqual(self.config(), 'test-value')
 
+    def test_with_children(self):
+        self.config.section1.value1.from_env('CONFIG_TEST_ENV')
+
+        self.assertEqual(self.config(), {'section1': {'value1': 'test-value'}})
+        self.assertEqual(self.config.section1(), {'value1': 'test-value'})
+        self.assertEqual(self.config.section1.value1(), 'test-value')
+
     def test_default(self):
         self.config.from_env('UNDEFINED_ENV', 'default-value')
         self.assertEqual(self.config(), 'default-value')
 
+    def test_default_none(self):
+        self.config.from_env('UNDEFINED_ENV')
+        self.assertIsNone(self.config())
+
+    def test_option_default_none(self):
+        self.config.option.from_env('UNDEFINED_ENV')
+        self.assertIsNone(self.config.option())
+
     def test_undefined_in_strict_mode(self):
         self.config = providers.Configuration(strict=True)
         with self.assertRaises(ValueError):
@@ -783,17 +865,38 @@ class ConfigFromEnvTests(unittest.TestCase):
         self.config.option.from_env('UNDEFINED_ENV', 'default-value')
         self.assertEqual(self.config.option(), 'default-value')
 
-    def test_default_none(self):
-        self.config.from_env('UNDEFINED_ENV')
+    def test_required_undefined(self):
+        with self.assertRaises(ValueError):
+            self.config.from_env('UNDEFINED_ENV', required=True)
+
+    def test_required_undefined_with_default(self):
+        self.config.from_env('UNDEFINED_ENV', default='default-value', required=True)
+        self.assertEqual(self.config(), 'default-value')
+
+    def test_option_required_undefined(self):
+        with self.assertRaises(ValueError):
+            self.config.option.from_env('UNDEFINED_ENV', required=True)
+
+    def test_option_required_undefined_with_default(self):
+        self.config.option.from_env('UNDEFINED_ENV', default='default-value', required=True)
+        self.assertEqual(self.config.option(), 'default-value')
+
+    def test_not_required_undefined_in_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.from_env('UNDEFINED_ENV', required=False)
         self.assertIsNone(self.config())
 
-    def test_option_default_none(self):
-        self.config.option.from_env('UNDEFINED_ENV')
+    def test_option_not_required_undefined_in_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.option.from_env('UNDEFINED_ENV', required=False)
         self.assertIsNone(self.config.option())
 
-    def test_with_children(self):
-        self.config.section1.value1.from_env('CONFIG_TEST_ENV')
+    def test_not_required_undefined_with_default_in_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.from_env('UNDEFINED_ENV', default='default-value', required=False)
+        self.assertEqual(self.config(), 'default-value')
 
-        self.assertEqual(self.config(), {'section1': {'value1': 'test-value'}})
-        self.assertEqual(self.config.section1(), {'value1': 'test-value'})
-        self.assertEqual(self.config.section1.value1(), 'test-value')
+    def test_option_not_required_undefined_with_default_in_strict_mode(self):
+        self.config = providers.Configuration(strict=True)
+        self.config.option.from_env('UNDEFINED_ENV', default='default-value', required=False)
+        self.assertEqual(self.config.option(), 'default-value')