459 Add default value for environment variable for yaml and ini config files (#461)

* Add tests for partial yaml interpolation

* Add tests for partial ini interpolation

* Add yaml config env defaults parsing

* Implement default interpolation for ini files

* Add tests for ini files env interpolation

* Update docs

* Update docs

* Update config docs keywords

docs/main/changelog.rst

@@ -7,6 +7,13 @@ that were made in every particular version.
 From version 0.7.6 *Dependency Injector* framework strictly 
 follows `Semantic versioning`_
 
+Development version
+------
+- Add support of default value for environment variable in INI and YAML
+  configuration files with ``${ENV_NAME:default}`` format.
+  See issue `#459 <https://github.com/ets-labs/python-dependency-injector/issues/459>`_.
+  Thanks to `Maksym Shemet @hbmshemet <https://github.com/hbmshemet>`_ for suggesting the feature.
+
 4.32.3
 ------
 - This fix a typo in ``di_in_python.rst`` doc.

docs/providers/configuration.rst

@@ -5,7 +5,7 @@ Configuration provider
 
 .. meta::
    :keywords: Python,DI,Dependency injection,IoC,Inversion of Control,Configuration,Injection,
-              Option,Ini,Json,Yaml,Pydantic,Dict,Environment Variable,Load,Read,Get
+              Option,Ini,Json,Yaml,Pydantic,Dict,Environment Variable,Default,Load,Read,Get
    :description: Configuration provides configuration options to the other providers. This page
                  demonstrates how to use Configuration provider to inject the dependencies, load
                  a configuration from an ini or yaml file, a dictionary, an environment variable,
@@ -43,8 +43,18 @@ where ``examples/providers/configuration/config.ini`` is:
    :language: ini
 
 :py:meth:`Configuration.from_ini` method supports environment variables interpolation. Use
-``${ENV_NAME}`` format in the configuration file to substitute value of the environment
-variable ``ENV_NAME``.
+``${ENV_NAME}`` format in the configuration file to substitute value from ``ENV_NAME`` environment
+variable.
+
+You can also specify a default value using ``${ENV_NAME:default}`` format. If environment
+variable ``ENV_NAME`` is undefined, configuration provider will substitute value ``default``.
+
+.. code-block:: ini
+
+   [section]
+   option1 = {$ENV_VAR}
+   option2 = {$ENV_VAR}/path
+   option3 = {$ENV_VAR:default}
 
 Loading from a YAML file
 ------------------------
@@ -62,12 +72,22 @@ where ``examples/providers/configuration/config.yml`` is:
 .. literalinclude:: ../../examples/providers/configuration/config.yml
    :language: ini
 
+:py:meth:`Configuration.from_yaml` method supports environment variables interpolation. Use
+``${ENV_NAME}`` format in the configuration file to substitute value from ``ENV_NAME`` environment
+variable.
+
+You can also specify a default value using ``${ENV_NAME:default}`` format. If environment
+variable ``ENV_NAME`` is undefined, configuration provider will substitute value ``default``.
+
+.. code-block:: ini
+
+   section:
+     option1: {$ENV_VAR}
+     option2: {$ENV_VAR}/path
+     option3: {$ENV_VAR:default}
+
 :py:meth:`Configuration.from_yaml` method uses custom version of ``yaml.SafeLoader``.
-
-The loader supports environment variables interpolation. Use ``${ENV_NAME}`` format
-in the configuration file to substitute value of the environment variable ``ENV_NAME``.
-
-You can also specify a YAML loader as an argument:
+To use another loader use ``loader`` argument:
 
 .. code-block:: python
 

src/dependency_injector/providers.pyx

@@ -64,6 +64,25 @@ else:  # pragma: no cover
                                     copy.deepcopy(obj.im_self, memo),
                                     obj.im_class)
 
+config_env_marker_pattern = re.compile(
+    r'\${(?P<name>[^}^{:]+)(?P<separator>:?)(?P<default>.*?)}',
+)
+
+def _resolve_config_env_markers(config_value):
+    """"Replace environment variable markers with their values."""
+    for match in reversed(list(config_env_marker_pattern.finditer(config_value))):
+        has_default = match.group('separator') == ':'
+
+        value = os.getenv(match.group('name'))
+        if value is None:
+            if not has_default:
+                continue
+            value = match.group('default')
+
+        span_min, span_max = match.span()
+        config_value = f'{config_value[:span_min]}{value}{config_value[span_max:]}'
+    return config_value
+
 
 if sys.version_info[0] == 3:
     class EnvInterpolation(iniconfigparser.BasicInterpolation):
@@ -71,7 +90,7 @@ if sys.version_info[0] == 3:
 
         def before_get(self, parser, section, option, value, defaults):
             value = super().before_get(parser, section, option, value, defaults)
-            return os.path.expandvars(value)
+            return _resolve_config_env_markers(value)
 
     def _parse_ini_file(filepath):
         parser = iniconfigparser.ConfigParser(interpolation=EnvInterpolation())
@@ -84,19 +103,18 @@ else:
     def _parse_ini_file(filepath):
         parser = iniconfigparser.ConfigParser()
         with open(filepath) as config_file:
-            config_string = os.path.expandvars(config_file.read())
+            config_string = _resolve_config_env_markers(config_file.read())
         parser.readfp(StringIO.StringIO(config_string))
         return parser
 
 
 if yaml:
     # TODO: use SafeLoader without env interpolation by default in version 5.*
-    yaml_env_marker_pattern = re.compile(r'\$\{([^}^{]+)\}')
     def yaml_env_marker_constructor(_, node):
         """"Replace environment variable marker with its value."""
-        return os.path.expandvars(node.value)
+        return _resolve_config_env_markers(node.value)
 
-    yaml.add_implicit_resolver('!path', yaml_env_marker_pattern)
+    yaml.add_implicit_resolver('!path', config_env_marker_pattern)
     yaml.add_constructor('!path', yaml_env_marker_constructor)
 
     class YamlLoader(yaml.SafeLoader):
@@ -105,7 +123,7 @@ if yaml:
         Inherits ``yaml.SafeLoader`` and add environment variables interpolation.
         """
 
-    YamlLoader.add_implicit_resolver('!path', yaml_env_marker_pattern, None)
+    YamlLoader.add_implicit_resolver('!path', config_env_marker_pattern, None)
     YamlLoader.add_constructor('!path', yaml_env_marker_constructor)
 else:
     class YamlLoader:

tests/unit/providers/test_configuration_py2_py3.py

@@ -591,17 +591,20 @@ class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
         self.config = providers.Configuration(name='config')
 
         os.environ['CONFIG_TEST_ENV'] = 'test-value'
+        os.environ['CONFIG_TEST_PATH'] = 'test-path'
 
         _, self.config_file = tempfile.mkstemp()
         with open(self.config_file, 'w') as config_file:
             config_file.write(
                 '[section1]\n'
                 'value1=${CONFIG_TEST_ENV}\n'
+                'value2=${CONFIG_TEST_PATH}/path\n'
             )
 
     def tearDown(self):
         del self.config
-        del os.environ['CONFIG_TEST_ENV']
+        os.environ.pop('CONFIG_TEST_ENV', None)
+        os.environ.pop('CONFIG_TEST_PATH', None)
         os.unlink(self.config_file)
 
     def test_env_variable_interpolation(self):
@@ -612,11 +615,67 @@ class ConfigFromIniWithEnvInterpolationTests(unittest.TestCase):
             {
                 'section1': {
                     'value1': 'test-value',
+                    'value2': 'test-path/path',
                 },
             },
         )
-        self.assertEqual(self.config.section1(), {'value1': 'test-value'})
+        self.assertEqual(
+            self.config.section1(),
+            {
+                'value1': 'test-value',
+                'value2': 'test-path/path',
+            },
+        )
         self.assertEqual(self.config.section1.value1(), 'test-value')
+        self.assertEqual(self.config.section1.value2(), 'test-path/path')
+
+    def test_missing_envs(self):
+        del os.environ['CONFIG_TEST_ENV']
+        del os.environ['CONFIG_TEST_PATH']
+
+        self.config.from_ini(self.config_file)
+
+        self.assertEqual(
+            self.config(),
+            {
+                'section1': {
+                    'value1': '${CONFIG_TEST_ENV}',
+                    'value2': '${CONFIG_TEST_PATH}/path',
+                },
+            },
+        )
+        self.assertEqual(
+            self.config.section1(),
+            {
+                'value1': '${CONFIG_TEST_ENV}',
+                'value2': '${CONFIG_TEST_PATH}/path',
+            },
+        )
+        self.assertEqual(self.config.section1.value1(), '${CONFIG_TEST_ENV}')
+        self.assertEqual(self.config.section1.value2(), '${CONFIG_TEST_PATH}/path')
+
+    def test_default_values(self):
+        os.environ['DEFINED'] = 'defined'
+        self.addCleanup(os.environ.pop, 'DEFINED')
+
+        with open(self.config_file, 'w') as config_file:
+            config_file.write(
+                '[section]\n'
+                'defined_with_default=${DEFINED:default}\n'
+                'undefined_with_default=${UNDEFINED:default}\n'
+                'complex=${DEFINED}/path/${DEFINED:default}/${UNDEFINED}/${UNDEFINED:default}\n'
+            )
+
+        self.config.from_ini(self.config_file)
+
+        self.assertEqual(
+            self.config.section(),
+            {
+                'defined_with_default': 'defined',
+                'undefined_with_default': 'default',
+                'complex': 'defined/path/defined/${UNDEFINED}/default',
+            },
+        )
 
 
 class ConfigFromYamlTests(unittest.TestCase):
@@ -781,17 +840,20 @@ class ConfigFromYamlWithEnvInterpolationTests(unittest.TestCase):
         self.config = providers.Configuration(name='config')
 
         os.environ['CONFIG_TEST_ENV'] = 'test-value'
+        os.environ['CONFIG_TEST_PATH'] = 'test-path'
 
         _, self.config_file = tempfile.mkstemp()
         with open(self.config_file, 'w') as config_file:
             config_file.write(
                 'section1:\n'
                 '  value1: ${CONFIG_TEST_ENV}\n'
+                '  value2: ${CONFIG_TEST_PATH}/path\n'
             )
 
     def tearDown(self):
         del self.config
-        del os.environ['CONFIG_TEST_ENV']
+        os.environ.pop('CONFIG_TEST_ENV', None)
+        os.environ.pop('CONFIG_TEST_PATH', None)
         os.unlink(self.config_file)
 
     @unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
@@ -803,11 +865,69 @@ class ConfigFromYamlWithEnvInterpolationTests(unittest.TestCase):
             {
                 'section1': {
                     'value1': 'test-value',
+                    'value2': 'test-path/path',
                 },
             },
         )
-        self.assertEqual(self.config.section1(), {'value1': 'test-value'})
+        self.assertEqual(
+            self.config.section1(),
+            {
+                'value1': 'test-value',
+                'value2': 'test-path/path',
+            },
+        )
         self.assertEqual(self.config.section1.value1(), 'test-value')
+        self.assertEqual(self.config.section1.value2(), 'test-path/path')
+
+    @unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
+    def test_missing_envs(self):
+        del os.environ['CONFIG_TEST_ENV']
+        del os.environ['CONFIG_TEST_PATH']
+
+        self.config.from_yaml(self.config_file)
+
+        self.assertEqual(
+            self.config(),
+            {
+                'section1': {
+                    'value1': '${CONFIG_TEST_ENV}',
+                    'value2': '${CONFIG_TEST_PATH}/path',
+                },
+            },
+        )
+        self.assertEqual(
+            self.config.section1(),
+            {
+                'value1': '${CONFIG_TEST_ENV}',
+                'value2': '${CONFIG_TEST_PATH}/path',
+            },
+        )
+        self.assertEqual(self.config.section1.value1(), '${CONFIG_TEST_ENV}')
+        self.assertEqual(self.config.section1.value2(), '${CONFIG_TEST_PATH}/path')
+
+    @unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
+    def test_default_values(self):
+        os.environ['DEFINED'] = 'defined'
+        self.addCleanup(os.environ.pop, 'DEFINED')
+
+        with open(self.config_file, 'w') as config_file:
+            config_file.write(
+                'section:\n'
+                '  defined_with_default: ${DEFINED:default}\n'
+                '  undefined_with_default: ${UNDEFINED:default}\n'
+                '  complex: ${DEFINED}/path/${DEFINED:default}/${UNDEFINED}/${UNDEFINED:default}\n'
+            )
+
+        self.config.from_yaml(self.config_file)
+
+        self.assertEqual(
+            self.config.section(),
+            {
+                'defined_with_default': 'defined',
+                'undefined_with_default': 'default',
+                'complex': 'defined/path/defined/${UNDEFINED}/default',
+            },
+        )
 
     @unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
     def test_option_env_variable_interpolation(self):
@@ -818,41 +938,47 @@ class ConfigFromYamlWithEnvInterpolationTests(unittest.TestCase):
             {
                 'section1': {
                     'value1': 'test-value',
+                    'value2': 'test-path/path',
                 },
             },
         )
-        self.assertEqual(self.config.option.section1(), {'value1': 'test-value'})
+        self.assertEqual(
+            self.config.option.section1(),
+            {
+                'value1': 'test-value',
+                'value2': 'test-path/path',
+            },
+        )
         self.assertEqual(self.config.option.section1.value1(), 'test-value')
+        self.assertEqual(self.config.option.section1.value2(), 'test-path/path')
 
     @unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
     def test_env_variable_interpolation_custom_loader(self):
         self.config.from_yaml(self.config_file, loader=yaml.UnsafeLoader)
 
         self.assertEqual(
-            self.config(),
+            self.config.section1(),
             {
-                'section1': {
                 'value1': 'test-value',
-                },
+                'value2': 'test-path/path',
             },
         )
-        self.assertEqual(self.config.section1(), {'value1': 'test-value'})
         self.assertEqual(self.config.section1.value1(), 'test-value')
+        self.assertEqual(self.config.section1.value2(), 'test-path/path')
 
     @unittest.skipIf(sys.version_info[:2] == (3, 4), 'PyYAML does not support Python 3.4')
     def test_option_env_variable_interpolation_custom_loader(self):
         self.config.option.from_yaml(self.config_file, loader=yaml.UnsafeLoader)
 
         self.assertEqual(
-            self.config.option(),
+            self.config.option.section1(),
             {
-                'section1': {
                 'value1': 'test-value',
-                },
+                'value2': 'test-path/path',
             },
         )
-        self.assertEqual(self.config.option.section1(), {'value1': 'test-value'})
         self.assertEqual(self.config.option.section1.value1(), 'test-value')
+        self.assertEqual(self.config.option.section1.value2(), 'test-path/path')
 
 
 class ConfigFromPydanticTests(unittest.TestCase):