Merge pull request #316 from skoranda/yaml_environment_variable_parsing

Pull YAML configuration values from environment

Author: c00kiemon5ter

doc/README.md

@@ -29,9 +29,46 @@ apt-get install libffi-dev libssl-dev xmlsec1
 Alternatively the application can be installed directly from PyPI (`pip install satosa`), or the [Docker image](https://hub.docker.com/r/satosa/) can be used.
 
 # Configuration
+SATOSA is configured using YAML.
+
 All default configuration files, as well as an example WSGI application for the proxy, can be found
 in the [example directory](../example).
 
+The default YAML syntax is extended to include the capability to resolve
+environment variables. The following tags are used to achieve this:
+
+* The `!ENV` tag
+
+The `!ENV` tag is followed by a string that denotes the environment variable
+name. It will be replaced by the value of the environment variable with the
+same name.
+
+In the example below `LDAP_BIND_PASSWORD` will, at runtime, be replaced with
+the value from the process environment variable of the same name. If the
+process environment has been set with `LDAP_BIND_PASSWORD=secret_password` then
+the configuration value for `bind_password` will be `secret_password`.
+
+```
+bind_password: !ENV LDAP_BIND_PASSWORD
+```
+
+* The `!ENVFILE` tag
+
+The `!ENVFILE` tag is followed by a string that denotes the environment
+variable name. It will be replaced by the value of the environment variable
+with the same name.
+
+In the example below `LDAP_BIND_PASSWORD_FILE` will, at runtime, be replaced
+with the value from the process environment variable of the same name. If the
+process environment has been set with
+`LDAP_BIND_PASSWORD_FILE=/etc/satosa/secrets/ldap.txt` then the configuration
+value for `bind_password` will be `secret_password`.
+
+```
+bind_password: !ENVFILE LDAP_BIND_PASSWORD_FILE
+```
+
+
 ## <a name="proxy_conf" style="color:#000000">SATOSA proxy configuration</a>: `proxy_conf.yaml.example`
 | Parameter name | Data type | Example value | Description |
 | -------------- | --------- | ------------- | ----------- |

example/plugins/microservices/ldap_attribute_store.yaml.example

@@ -8,7 +8,11 @@ config:
   "":
       ldap_url: ldaps://ldap.example.org
       bind_dn: cn=admin,dc=example,dc=org
-      bind_password: xxxxxxxx
+      # Obtain bind password from environment variable LDAP_BIND_PASSWORD.
+      bind_password: !ENV LDAP_BIND_PASSWORD
+      # Obtain bind password from file pointed to by
+      # environment variable LDAP_BIND_PASSWORD_FILE.
+      # bind_password: !ENVFILE LDAP_BIND_PASSWORD
       search_base: ou=People,dc=example,dc=org
       read_only: true
       auto_bind: true

src/satosa/plugin_loader.py

@@ -7,8 +7,8 @@
 from contextlib import contextmanager
 from pydoc import locate
 
-import yaml
-from yaml.error import YAMLError
+from satosa.yaml import load as yaml_load
+from satosa.yaml import YAMLError
 
 from .backends.base import BackendModule
 from .exception import SATOSAConfigurationError
@@ -143,7 +143,7 @@ def _response_micro_service_filter(cls):
 
 def _load_plugin_config(config):
     try:
-        return yaml.safe_load(config)
+        return yaml_load(config)
     except YAMLError as exc:
         if hasattr(exc, 'problem_mark'):
             mark = exc.problem_mark

src/satosa/satosa_config.py

@@ -3,10 +3,12 @@
 """
 import logging
 import os
+import os.path
 
-import yaml
+from satosa.exception import SATOSAConfigurationError
+from satosa.yaml import load as yaml_load
+from satosa.yaml import YAMLError
 
-from .exception import SATOSAConfigurationError
 
 logger = logging.getLogger(__name__)
 
@@ -143,10 +145,11 @@ def _load_yaml(self, config_file):
         :param config_file: config to load. Can be file path or yaml string
         :return: Loaded config
         """
+
         try:
             with open(os.path.abspath(config_file)) as f:
-                return yaml.safe_load(f.read())
-        except yaml.YAMLError as exc:
+                return yaml_load(f.read())
+        except YAMLError as exc:
             logger.error("Could not parse config as YAML: {}".format(exc))
             if hasattr(exc, 'problem_mark'):
                 mark = exc.problem_mark

src/satosa/yaml.py

@@ -0,0 +1,52 @@
+import os
+import re
+
+from yaml import SafeLoader as _safe_loader
+from yaml import YAMLError
+from yaml import safe_load as load
+
+
+def _constructor_env_variables(loader, node):
+    """
+    Extracts the environment variable from the node's value.
+    :param yaml.Loader loader: the yaml loader
+    :param node: the current node in the yaml
+    :return: value of the environment variable
+    """
+    raw_value = loader.construct_scalar(node)
+    new_value = os.environ.get(raw_value)
+    if new_value is None:
+        msg = "Cannot construct value from {node}: {value}".format(
+            node=node, value=new_value
+        )
+        raise YAMLError(msg)
+    return new_value
+
+
+def _constructor_envfile_variables(loader, node):
+    """
+    Extracts the environment variable from the node's value.
+    :param yaml.Loader loader: the yaml loader
+    :param node: the current node in the yaml
+    :return: value read from file pointed to by environment variable
+    """
+    raw_value = loader.construct_scalar(node)
+    filepath = os.environ.get(raw_value)
+    try:
+        with open(filepath, "r") as fd:
+            new_value = fd.read()
+    except (TypeError, IOError) as e:
+        msg = "Cannot construct value from {node}: {path}".format(
+            node=node, path=filepath
+        )
+        raise YAMLError(msg) from e
+    else:
+        return new_value
+
+
+TAG_ENV = "!ENV"
+TAG_ENVFILE = "!ENVFILE"
+
+
+_safe_loader.add_constructor(TAG_ENV, _constructor_env_variables)
+_safe_loader.add_constructor(TAG_ENVFILE, _constructor_envfile_variables)

tests/satosa/test_satosa_config.py

@@ -1,4 +1,5 @@
 import json
+import os
 from unittest.mock import mock_open, patch
 
 import pytest
@@ -7,6 +8,7 @@
 from satosa.exception import SATOSAConfigurationError
 from satosa.satosa_config import SATOSAConfig
 
+TEST_RESOURCE_BASE_PATH = os.path.join(os.path.dirname(__file__), "../test_resources")
 
 class TestSATOSAConfig:
     @pytest.fixture
@@ -73,3 +75,22 @@ def test_can_read_endpoint_configs_from_file(self, satosa_config_dict, modules_k
 
         with pytest.raises(SATOSAConfigurationError):
             SATOSAConfig(satosa_config_dict)
+
+    def test_can_substitute_from_environment_variable(self, monkeypatch):
+        monkeypatch.setenv("SATOSA_COOKIE_STATE_NAME", "oatmeal_raisin")
+        config = SATOSAConfig(
+            os.path.join(TEST_RESOURCE_BASE_PATH, "proxy_conf_environment_test.yaml")
+        )
+
+        assert config["COOKIE_STATE_NAME"] == 'oatmeal_raisin'
+
+    def test_can_substitute_from_environment_variable_file(self, monkeypatch):
+        cookie_file = os.path.join(TEST_RESOURCE_BASE_PATH, 'cookie_state_name')
+        monkeypatch.setenv("SATOSA_COOKIE_STATE_NAME_FILE", cookie_file)
+        config = SATOSAConfig(
+            os.path.join(
+                TEST_RESOURCE_BASE_PATH, "proxy_conf_environment_file_test.yaml"
+            )
+        )
+
+        assert config["COOKIE_STATE_NAME"] == 'chocolate_chip'

tests/test_resources/cookie_state_name

@@ -0,0 +1 @@
+chocolate_chip

tests/test_resources/proxy_conf_environment_file_test.yaml

@@ -0,0 +1,10 @@
+BASE: https://example.com
+
+STATE_ENCRYPTION_KEY: state_encryption_key
+
+INTERNAL_ATTRIBUTES: {"attributes": {}}
+
+COOKIE_STATE_NAME: !ENVFILE SATOSA_COOKIE_STATE_NAME_FILE
+
+BACKEND_MODULES: []
+FRONTEND_MODULES: []

tests/test_resources/proxy_conf_environment_test.yaml

@@ -0,0 +1,10 @@
+BASE: https://example.com
+
+STATE_ENCRYPTION_KEY: state_encryption_key
+
+INTERNAL_ATTRIBUTES: {"attributes": {}}
+
+COOKIE_STATE_NAME: !ENV SATOSA_COOKIE_STATE_NAME
+
+BACKEND_MODULES: []
+FRONTEND_MODULES: []