Pull YAML configuration values from file pointed to by environment

Add logic so that a YAML tag of the form !ENVFILE indicates that a value
of the form $(SOME_ENVIRONMENT_VARIABLE_FILE) should be replaced with the
value obtained by reading the process environment variable of the same
name to get a file path and then reading the file contents.

Author: skoranda

example/plugins/microservices/ldap_attribute_store.yaml.example

@@ -10,6 +10,9 @@ config:
       bind_dn: cn=admin,dc=example,dc=org
       # Obtain bind password from environment variable LDAP_BIND_PASSWORD.
       bind_password: !ENV ${LDAP_BIND_PASSWORD}
+      # Obtain bind password from file pointed to by
+      # environment variable LDAP_BIND_PASSWORD_FILE.
+      # bind_password: !ENVFILE $(LDAP_BIND_PASSWORD)
       search_base: ou=People,dc=example,dc=org
       read_only: true
       auto_bind: true

src/satosa/satosa_config.py

@@ -3,6 +3,7 @@
 """
 import logging
 import os
+import os.path
 import re
 
 import yaml
@@ -145,12 +146,12 @@ def _load_yaml(self, config_file):
         :return: Loaded config
         """
         # Tag to indicate environment variable: !ENV
-        tag = '!ENV'
+        tag_env = '!ENV'
 
         # Pattern for environment variable: ${word}
-        pattern = re.compile('.*?\${(\w+)}.*?')
+        pattern_env = re.compile('.*?\${(\w+)}.*?')
 
-        yaml.SafeLoader.add_implicit_resolver(tag, pattern, None)
+        yaml.SafeLoader.add_implicit_resolver(tag_env, pattern_env, None)
 
         def constructor_env_variables(loader, node):
             """
@@ -160,7 +161,7 @@ def constructor_env_variables(loader, node):
             :return: value of the environment variable
             """
             value = loader.construct_scalar(node)
-            match = pattern.findall(value)
+            match = pattern_env.findall(value)
             if match:
                 new_value = value
                 for m in match:
@@ -169,7 +170,39 @@ def constructor_env_variables(loader, node):
                 return new_value
             return value
 
-        yaml.SafeLoader.add_constructor(tag, constructor_env_variables)
+        yaml.SafeLoader.add_constructor(tag_env, constructor_env_variables)
+
+        # Tag to indicate file pointed to by environment variable: !ENVFILE
+        tag_env_file = '!ENVFILE'
+
+        # Pattern for environment variable: $(word)
+        pattern_env_file = re.compile('.*?\$\((\w+)\).*?')
+
+        yaml.SafeLoader.add_implicit_resolver(tag_env_file,
+                                              pattern_env_file, None)
+
+        def constructor_envfile_variables(loader, node):
+            """
+            Extracts the environment variable from the node's value.
+            :param yaml.Loader loader: the yaml loader
+            :param node: the current node in the yaml
+            :return: value read from file pointed to by environment variable
+            """
+            value = loader.construct_scalar(node)
+            match = pattern_env_file.findall(value)
+            if match:
+                new_value = value
+                for m in match:
+                    path = os.environ.get(m, '')
+                    if os.path.exists(path):
+                        with open(path, 'r') as f:
+                            new_value = new_value.replace('$(' + m + ')',
+                                                          f.read().strip())
+                return new_value
+            return value
+
+        yaml.SafeLoader.add_constructor(tag_env_file,
+                                        constructor_envfile_variables)
 
         try:
             with open(os.path.abspath(config_file)) as f:

tests/satosa/test_satosa_config.py

@@ -82,3 +82,12 @@ def test_can_substitute_from_environment_variable(self, monkeypatch):
                               "proxy_conf_environment_test.yaml"))
 
         assert config["COOKIE_STATE_NAME"] == 'oatmeal_raisin'
+
+    def test_can_substitute_from_environment_variable_file(self, monkeypatch):
+        cookie_file = os.path.join(TEST_RESOURCE_BASE_PATH,
+                                   'cookie_state_name')
+        monkeypatch.setenv("SATOSA_COOKIE_STATE_NAME_FILE", cookie_file)
+        config = SATOSAConfig(os.path.join(TEST_RESOURCE_BASE_PATH,
+                              "proxy_conf_environment_file_test.yaml"))
+
+        assert config["COOKIE_STATE_NAME"] == 'chocolate_chip'

tests/test_resources/cookie_state_name

@@ -0,0 +1 @@
+chocolate_chip

tests/test_resources/proxy_conf_environment_file_test.yaml

@@ -0,0 +1,10 @@
+BASE: https://example.com
+
+STATE_ENCRYPTION_KEY: state_encryption_key
+
+INTERNAL_ATTRIBUTES: {"attributes": {}}
+
+COOKIE_STATE_NAME: !ENVFILE $(SATOSA_COOKIE_STATE_NAME_FILE)
+
+BACKEND_MODULES: []
+FRONTEND_MODULES: []