Remove and replace additional operations in enclave start (#1289)

abuabraham-ttd · web-flow · commit 775bacf4ec27 · 2025-01-30T10:45:26.000-08:00
* Remove and replace additional operations in enclave start
diff --git a/Makefile.eif b/Makefile.eif
@@ -13,46 +13,37 @@ all: build_eif
 
 build_eif: uid2operator.eif euidoperator.eif
 
-uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py
+uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile
 	cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar; rm -f ./uid2operator.tar
 	docker exec amazonlinux bash aws_nitro_eif.sh uid2operator
 
-euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py
+euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile
 	cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar; rm -f ./euidoperator.tar
 	docker exec amazonlinux bash aws_nitro_eif.sh euidoperator
 
-##################################################################################################################################################################
-
-# Config scripts
-
-build/load_config.py: ./scripts/aws/load_config.py
-	cp ./scripts/aws/load_config.py ./build/
-
-build/make_config.py: ./scripts/aws/make_config.py
-	cp ./scripts/aws/make_config.py ./build/
 
 ##################################################################################################################################################################
 
 # Configs
 
 .PHONY: build_configs
 
-build_configs: build/conf/default-config.json build/conf/prod-uid2-config.json build/conf/integ-uid2-config.json build/conf/prod-euid-config.json build/conf/integ-euid-config.json build/conf/logback.xml build/conf/logback-debug.xml
+build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml
 
 build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json
 	cp ./scripts/aws/conf/default-config.json ./build/conf/
 
-build/conf/prod-uid2-config.json: build_artifacts ./scripts/aws/conf/prod-uid2-config.json
-	cp ./scripts/aws/conf/prod-uid2-config.json ./build/conf/
+build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json
+	cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/
 
-build/conf/prod-euid-config.json: build_artifacts ./scripts/aws/conf/prod-euid-config.json
-	cp ./scripts/aws/conf/prod-euid-config.json ./build/conf/
+build/conf/euid-prod-config.json: build_artifacts ./scripts/aws/conf/euid-prod-config.json
+	cp ./scripts/aws/conf/euid-prod-config.json ./build/conf/
 
-build/conf/integ-uid2-config.json: build_artifacts ./scripts/aws/conf/integ-uid2-config.json
-	cp ./scripts/aws/conf/integ-uid2-config.json ./build/conf/
+build/conf/uid2-integ-config.json: build_artifacts ./scripts/aws/conf/uid2-integ-config.json
+	cp ./scripts/aws/conf/uid2-integ-config.json ./build/conf/
 
-build/conf/integ-euid-config.json: build_artifacts ./scripts/aws/conf/integ-euid-config.json
-	cp ./scripts/aws/conf/integ-euid-config.json ./build/conf/
+build/conf/uid2-prod-config.json: build_artifacts ./scripts/aws/conf/uid2-prod-config.json
+	cp ./scripts/aws/conf/uid2-prod-config.json ./build/conf/
 
 build/conf/logback.xml: build_artifacts ./scripts/aws/conf/logback.xml
 	cp ./scripts/aws/conf/logback.xml ./build/conf/
diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile
@@ -31,14 +31,10 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NA
 COPY ./static             /app/static
 COPY ./libjnsm.so         /app/lib/
 COPY ./vsockpx            /app/
-COPY ./make_config.py     /app/
 COPY ./entrypoint.sh      /app/
 COPY ./proxies.nitro.yaml /app/
 COPY ./conf/default-config.json    /app/conf/
-COPY ./conf/prod-uid2-config.json  /app/conf/
-COPY ./conf/integ-uid2-config.json /app/conf/
-COPY ./conf/prod-euid-config.json  /app/conf/
-COPY ./conf/integ-euid-config.json /app/conf/
+COPY ./conf/*.json  /app/conf/
 COPY ./conf/*.xml                  /app/conf/
 COPY ./syslog-ng-client.conf       /etc/syslog-ng/syslog-ng.conf
 
diff --git a/scripts/aws/EUID_CloudFormation.template.yml b/scripts/aws/EUID_CloudFormation.template.yml
@@ -165,7 +165,7 @@ Resources:
           - !If [IsIntegEnvironment, 'https://core.integ.euid.eu', 'https://core.prod.euid.eu']
           - ', "optout_base_url": '
           - !If [IsIntegEnvironment, 'https://optout.integ.euid.eu', 'https://optout.prod.euid.eu']
-          - ', "api_token": "'
+          - ', "operator_key": "'
           - Ref: APIToken
           - '"'
           - ', "service_instances": 6'
diff --git a/scripts/aws/UID_CloudFormation.template.yml b/scripts/aws/UID_CloudFormation.template.yml
@@ -193,7 +193,7 @@ Resources:
           - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com']
           - ', "optout_base_url": '
           - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com']
-          - ', "api_token": "'
+          - ', "operator_key": "'
           - Ref: APIToken
           - '"'
           - ', "service_instances": 6'
diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json
@@ -12,4 +12,4 @@
   "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate",
   "optout_s3_folder": "optout/",
   "allow_legacy_api": false
-}
+}
diff --git a/scripts/aws/conf/euid-prod-config.json b/scripts/aws/conf/euid-prod-config.json
@@ -28,4 +28,4 @@
   "enable_phone_support": true,
   "enable_v1_phone_support": false,
   "enable_v2_encryption": true
-}
+}
diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json
@@ -12,4 +12,4 @@
   "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate",
   "optout_s3_folder": "uid-optout-integ/",
   "allow_legacy_api": false
-}
+}
diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json
@@ -23,4 +23,4 @@
   "refresh_token_expires_after_seconds": 2592000,
   "refresh_identity_token_after_seconds": 3600,
   "allow_legacy_api": false
-}
+}
diff --git a/scripts/aws/config-server/app.py b/scripts/aws/config-server/app.py
@@ -10,25 +10,6 @@ def get_config():
         with open('/etc/secret/secret-value/config', 'r') as secret_file:
             secret_value = secret_file.read().strip()
             secret_value_json = json.loads(secret_value)
-            secret_value_json["environment"] = secret_value_json["environment"].lower()
-            if "core_base_url" in secret_value_json:
-                secret_value_json["core_base_url"] = secret_value_json["core_base_url"].lower()
-            if "optout_base_url" in secret_value_json:
-                secret_value_json["optout_base_url"] = secret_value_json["optout_base_url"].lower()
-            if "operator_type" in secret_value_json and secret_value_json["operator_type"].lower() == "public":
-                mount_path = '/etc/config/config-values'
-                if os.path.exists(mount_path):
-                    config_keys = [f for f in os.listdir(mount_path) if os.path.isfile(os.path.join(mount_path, f))]
-                    config = {}
-                    for k in config_keys:
-                        with open(os.path.join(mount_path, k), 'r') as value:
-                            config[k] = value.read()
-                            try:
-                                json.loads(config[k])
-                                config[k] = json.loads(config[k])
-                            except Exception:
-                                pass
-                    secret_value_json.update(config)
         return json.dumps(secret_value_json)
     except Exception as e:
         return str(e), 500
diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py
@@ -21,6 +21,8 @@
 class AWSConfidentialComputeConfig(ConfidentialComputeConfig):
     enclave_memory_mb: int
     enclave_cpu_count: int
+    core_api_token: str
+    optout_api_token: str
 
 class AuxiliaryConfig:
     FLASK_PORT: str = "27015"
@@ -51,7 +53,7 @@ def get_meta_url(cls) -> str:
 class EC2EntryPoint(ConfidentialCompute):
 
     def __init__(self):
-        self.configs: AWSConfidentialComputeConfig = {}
+        super().__init__()
 
     def __get_aws_token(self) -> str:
         """Fetches a temporary AWS EC2 metadata token."""
@@ -87,18 +89,21 @@ def __validate_aws_specific_config(self):
     def _set_confidential_config(self, secret_identifier: str) -> None:
         """Fetches a secret value from AWS Secrets Manager and adds defaults"""
 
-        def add_defaults(configs: Dict[str, any]) ->  None:
+        def add_defaults(configs: Dict[str, any]) ->  AWSConfidentialComputeConfig:
             """Adds default values to configuration if missing."""
             default_capacity = self.__get_max_capacity()
             configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"])
             configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"])
             configs.setdefault("debug_mode", False)
+            configs.setdefault("core_api_token", configs.get("api_token", ""))
+            configs.setdefault("optout_api_token", configs.get("api_token", ""))
+            return configs
         
         region = self.__get_current_region()
         print(f"Running in {region}")
         client = boto3.client("secretsmanager", region_name=region)
         try:
-            add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"]))
+            self.configs = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"]))
             self.__validate_aws_specific_config()
         except NoCredentialsError as _:
             raise MissingInstanceProfile(self.__class__.__name__)
@@ -203,7 +208,7 @@ def __run_nitro_enclave(self):
         if self.configs.get('debug_mode', False):
             print("Running in debug_mode")
             command += ["--debug-mode", "--attach-console"]
-        self.run_command(command, separate_process=True)
+        self.run_command(command, separate_process=False)
 
     def run_compute(self) -> None:
         """Main execution flow for confidential compute."""
diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh
@@ -7,93 +7,87 @@ LOG_FILE="/home/start.txt"
 set -x
 exec &> >(tee -a "$LOG_FILE")
 
-set -o pipefail
-ulimit -n 65536
-
-# -- setup loopback device
-echo "Setting up loopback device..."
-ifconfig lo 127.0.0.1
-
-# -- start vsock proxy
-echo "Starting vsock proxy..."
-/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3
-
-# -- load config from identity service
-echo "Loading config from identity service via proxy..."
-
-#wait for config service, then download config
-OVERRIDES_CONFIG="/app/conf/config-overrides.json"
-
-RETRY_COUNT=0
-MAX_RETRY=20
-until curl -s -f -o "${OVERRIDES_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig
-do
-  echo "Waiting for config service to be available"
-  RETRY_COUNT=$(( RETRY_COUNT + 1))
-  if [ $RETRY_COUNT -gt $MAX_RETRY ]; then
-      echo "Config Server did not return a response. Exiting"
-      exit 1
-  fi
-  sleep 2
-done
+PARAMETERIZED_CONFIG="/app/conf/config-overrides.json"
+OPERATOR_CONFIG="/tmp/final-config.json"
 
-DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}")
+setup_auxiliaries() {
+  set -o pipefail
+  ulimit -n 65536
+
+  # -- setup loopback device
+  echo "Setting up loopback device..."
+  ifconfig lo 127.0.0.1
+
+  # -- start vsock proxy
+  echo "Starting vsock proxy..."
+  /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3
 
-if [[ "$DEBUG_MODE" == "true" ]]; then
-  LOGBACK_CONF="./conf/logback-debug.xml"
-else
-  LOGBACK_CONF="./conf/logback.xml"
-  # -- setup syslog-ng
-  echo "Starting syslog-ng..."
   /usr/sbin/syslog-ng --verbose
-fi
+}
 
-# check the config is valid. Querying for a known missing element (empty) makes jq parse the file, but does not echo the results
-if jq empty "${OVERRIDES_CONFIG}"; then
-    echo "Identity service returned valid config"
-else
-    echo "Failed to get a valid config from identity service"
-    exit 1
-fi
 
-export DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${OVERRIDES_CONFIG}")
-export CORE_BASE_URL=$(jq -r ".core_base_url" < "${OVERRIDES_CONFIG}")
-export OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${OVERRIDES_CONFIG}")
-echo "DEPLOYMENT_ENVIRONMENT=${DEPLOYMENT_ENVIRONMENT}"
-if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then
-  echo "DEPLOYMENT_ENVIRONMENT cannot be empty"
-  exit 1
-fi
-if [ "${DEPLOYMENT_ENVIRONMENT}" != "prod" ] && [ "${DEPLOYMENT_ENVIRONMENT}" != "integ" ]; then
-  echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}"
-  exit 1
-fi
+build_parameterized_config() {
+  curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig
+  REQUIRED_KEYS=("optout_base_url" "core_base_url" "api_token" "environment")
+  for key in "${REQUIRED_KEYS[@]}"; do
+    if ! jq -e "has(\"${key}\")" "${PARAMETERIZED_CONFIG}" > /dev/null; then
+      echo "Error: Key '${key}' is missing. Please add it to flask config server"
+      exit 1
+    fi
+  done
+  FILTER=$(printf '. | {')
+  for key in "${REQUIRED_KEYS[@]}"; do
+    FILTER+="$key: .${key}, "
+  done
+  FILTER+="debug_mode: .debug_mode, "
+  FILTER=${FILTER%, }'}'
+  jq "${FILTER}" "${PARAMETERIZED_CONFIG}" > "${PARAMETERIZED_CONFIG}.tmp" && mv "${PARAMETERIZED_CONFIG}.tmp" "${PARAMETERIZED_CONFIG}"
+}
+
+build_operator_config() {
+  CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}")
+  OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}")
+  DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}")
+  DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}")
+
+  IDENTITY_SCOPE_LOWER=$(echo "${IDENTITY_SCOPE}" | tr '[:upper:]' '[:lower:]')
+  DEPLOYMENT_ENVIRONMENT_LOWER=$(echo "${DEPLOYMENT_ENVIRONMENT}" | tr '[:upper:]' '[:lower:]')
+  DEFAULT_CONFIG="/app/conf/${IDENTITY_SCOPE_LOWER}-${DEPLOYMENT_ENVIRONMENT_LOWER}-config.json"
+
+  jq -s '.[0] * .[1]' "${DEFAULT_CONFIG}" "${PARAMETERIZED_CONFIG}" > "${OPERATOR_CONFIG}"
+
+  if [[ "$DEPLOYMENT_ENVIRONMENT" == "prod" ]]; then
+    if [[ "$DEBUG_MODE" == "true" ]]; then
+      echo "Cannot run in DEBUG_MODE in production environment. Exiting."
+      exit 1
+    fi
+  fi
 
-echo "Loading config final..."
-export FINAL_CONFIG="/app/conf/config-final.json"
-if [ "${IDENTITY_SCOPE}" = "UID2" ]; then
-  python3 /app/make_config.py /app/conf/prod-uid2-config.json /app/conf/integ-uid2-config.json ${OVERRIDES_CONFIG} "$(nproc)" > ${FINAL_CONFIG}
-elif [ "${IDENTITY_SCOPE}" = "EUID" ]; then
-  python3 /app/make_config.py /app/conf/prod-euid-config.json /app/conf/integ-euid-config.json ${OVERRIDES_CONFIG} "$(nproc)" > ${FINAL_CONFIG}
-else
-  echo "Unrecognized IDENTITY_SCOPE ${IDENTITY_SCOPE}"
-  exit 1
-fi
+  #TODO: Remove below logic after remote config management is implemented
+
+  if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then
+    #Allow override of base URL in non-prod environments
+    CORE_PATTERN="https://core.*uidapi.com"
+    OPTOUT_PATTERN="https://optout.*uidapi.com"
+    if [[ "$DEPLOYMENT_ENVIRONMENT" == "euid" ]]; then
+      CORE_PATTERN="https://core.*euid.eu"
+      OPTOUT_PATTERN="https://optout.*euid.eu"
+    fi
+    sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}"
+    sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}"
+  fi
+  
+}
 
-# -- replace base URLs if both CORE_BASE_URL and OPTOUT_BASE_URL are provided
-# -- using hardcoded domains is fine because they should not be changed frequently
-if [ -n "${CORE_BASE_URL}" ] && [ "${CORE_BASE_URL}" != "null" ] && [ -n "${OPTOUT_BASE_URL}" ] && [ "${OPTOUT_BASE_URL}" != "null" ] && [ "${DEPLOYMENT_ENVIRONMENT}" != "prod" ]; then
-    echo "Replacing core and optout URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}..."
+setup_auxiliaries
+build_parameterized_config
+build_operator_config
 
-    sed -i "s#https://core-integ.uidapi.com#${CORE_BASE_URL}#g" "${FINAL_CONFIG}"
-    sed -i "s#https://core-prod.uidapi.com#${CORE_BASE_URL}#g" "${FINAL_CONFIG}"
-    sed -i "s#https://core.integ.euid.eu#${CORE_BASE_URL}#g" "${FINAL_CONFIG}"
-    sed -i "s#https://core.prod.euid.eu#${CORE_BASE_URL}#g" "${FINAL_CONFIG}"
+DEBUG_MODE=$(jq -r ".debug_mode" < "${OPERATOR_CONFIG}")
+LOGBACK_CONF="./conf/logback.xml"
 
-    sed -i "s#https://optout-integ.uidapi.com#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}"
-    sed -i "s#https://optout-prod.uidapi.com#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}"
-    sed -i "s#https://optout.integ.euid.eu#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}"
-    sed -i "s#https://optout.prod.euid.eu#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}"
+if [[ "$DEBUG_MODE" == "true" ]]; then
+  LOGBACK_CONF="./conf/logback-debug.xml"
 fi
 
 # -- set pwd to /app so we can find default configs
@@ -106,7 +100,7 @@ java \
   -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \
   -Djava.security.egd=file:/dev/./urandom \
   -Djava.library.path=/app/lib \
-  -Dvertx-config-path="${FINAL_CONFIG}" \
+  -Dvertx-config-path="${OPERATOR_CONFIG}" \
   -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \
   -Dlogback.configurationFile=${LOGBACK_CONF} \
   -Dhttp_proxy=socks5://127.0.0.1:3305 \
diff --git a/scripts/aws/load_config.py b/scripts/aws/load_config.py
diff --git a/scripts/aws/make_config.py b/scripts/aws/make_config.py
diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py
