From 950a1646c8aede88dd7fc9bfa14a828238955101 Mon Sep 17 00:00:00 2001 From: Gregory Rushton Date: Fri, 23 Oct 2026 10:32:31 -0400 Subject: [PATCH] [DT-4348][risk=no] Render consent.yaml, site.conf and oauth2.conf from terra-helmfile (2/5) Co-Authored-By: Claude Opus 5.5 --- DEVNOTES.md | 28 ++-- scripts/lib/common.sh | 30 +++++ scripts/render-chart-configs.sh | 200 ++++++++++++++++++++++++++++ scripts/render-configs.sh | 41 +++++- scripts/templates/local-values.yaml | 12 ++ 5 files changed, 296 insertions(+), 15 deletions(-) create mode 100755 scripts/render-chart-configs.sh create mode 100644 scripts/templates/local-values.yaml diff --git a/DEVNOTES.md b/DEVNOTES.md index 1d5cb48f7..4e96c2dbb 100644 --- a/DEVNOTES.md +++ b/DEVNOTES.md @@ -43,29 +43,35 @@ local stack needs. Do not edit these files by hand. The next render replaces the | File | Source | Option | |---|---|---| | `server.crt`, `server.key`, `ca-bundle.crt` | The `local-dev-cert` secret in the dev cluster | Always | +| `consent.yaml`, `site.conf`, `oauth2.conf` | The consent chart in `terra-helmfile`, with the dev values and [local-values.yaml](scripts/templates/local-values.yaml) | `--write_chart_configs true` | -You need `gcloud`, `kubectl` and `openssl`, and the non-split Broad VPN for the cluster steps. The -script stops at the first step that fails, and that step leaves the old files as they were. Run: +You need `gcloud`, `kubectl`, `helm`, `git` and `openssl`, and the non-split Broad VPN for the +cluster steps. The chart configs need read access to `broadinstitute/terra-helmfile`, with git +HTTPS credentials or a GitHub SSH key. The script tries HTTPS first, then SSH. The script stops at +the first step that fails, and that step leaves the old files as they were. The first time, run: ```bash -./scripts/render-configs.sh +./scripts/render-configs.sh --write_chart_configs true ``` -Every 3 months the certs rotate. Run the script again to get the new certs. +* After a cert rotation, run `./scripts/render-configs.sh` with no options. +* To test a `terra-helmfile` branch, run `./scripts/render-configs.sh --write_chart_configs true + --helmfile_ref my-branch`. For changes that are not pushed, use `--helmfile_dir + ~/develop/terra-helmfile`. To render the chart configs without the VPN, run + [render-chart-configs.sh](scripts/render-chart-configs.sh) on its own. -### Configure +The rendered files are the same as the files that dev runs. The only local changes are in +`local-values.yaml`. See [the plan](docs/plans/render-local-configs-plan.md#all-local-changes). -Copy `src/test/resources/consent-config.yml` to `config/consent.yaml` (the `config/` directory is -git-ignored) and fill in the values for your environment. The `googleStore` section is the only -part that needs cloud credentials; see the next section. +#### Liquibase `liquibase.runOnStartup` is an optional flag. The application runs the Liquibase changesets at startup only when the flag is `true`. If a config omits the flag or the whole `liquibase` block, the flag is `false`. Environments that run the Liquibase migration Job omit it. In `terra-helmfile`, that Job is a separate Kubernetes Job that Argo CD runs before the deployment -(sync-wave `-1`). The template sets it to `true`, because a local database -restored from an older dump needs the newer changesets. If you render `consent.yaml` from -`terra-helmfile`, add this block yourself, or pass `-Ddw.liquibase.runOnStartup=true`: +(sync-wave `-1`). So the rendered `consent.yaml` has no `liquibase` block. A local database +restored from an older dump needs the newer changesets, so pass +`-Ddw.liquibase.runOnStartup=true`, or add this block to your config: ```yaml liquibase: diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index cefc7215b..60170ceb3 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -8,3 +8,33 @@ gcloud_account() { gcloud auth list --filter=status:ACTIVE --format='value(account)' 2> /dev/null | head -1 } + +# Clone broadinstitute/terra-helmfile at REF into DEST, as duos-ui does: over HTTPS first, then +# over SSH. Other arguments go to `git clone`. GIT_TERMINAL_PROMPT=0 makes git fail instead of +# asking for a password, so a missing HTTPS credential falls through to SSH. +clone_helmfile() { + local ref="$1" dest="$2" + shift 2 + rm -rf "$dest" + if GIT_TERMINAL_PROMPT=0 git clone --quiet --depth 1 --branch "$ref" "$@" \ + https://github.com/broadinstitute/terra-helmfile.git "$dest" 2> /dev/null; then + return 0 + fi + echo "Could not clone terra-helmfile over HTTPS. Trying SSH." >&2 + rm -rf "$dest" + if GIT_TERMINAL_PROMPT=0 git clone --quiet --depth 1 --branch "$ref" "$@" \ + git@github.com:broadinstitute/terra-helmfile.git "$dest"; then + return 0 + fi + echo "ERROR: Could not clone terra-helmfile at $ref over HTTPS or SSH. Check that your git credentials or SSH key can read broadinstitute/terra-helmfile." >&2 + return 1 +} + +# Check that a flag value is true or false. +check_bool() { + local flag="$1" value="$2" + if [[ "$value" != "true" && "$value" != "false" ]]; then + echo "ERROR: $flag must be true or false, not '$value'." >&2 + exit 1 + fi +} diff --git a/scripts/render-chart-configs.sh b/scripts/render-chart-configs.sh new file mode 100755 index 000000000..ecfad14f3 --- /dev/null +++ b/scripts/render-chart-configs.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# +# Renders consent.yaml, site.conf and oauth2.conf from the consent chart in terra-helmfile, with +# the dev values and scripts/templates/local-values.yaml. The rendered files are the reference +# for local config: do not edit them. Put a local change in local-values.yaml, with its reason. +# +# You MUST have helm and git installed. Without --helmfile_dir, you MUST also have git read +# access to broadinstitute/terra-helmfile, over HTTPS or SSH. This script does not need the VPN. +# +# render-configs.sh calls this script. It also runs on its own. +# + +set -eu +set -o pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd) +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" +LOCAL_VALUES="$SCRIPT_DIR/templates/local-values.yaml" + +HELMFILE_REF="master" +HELMFILE_DIR="" +OUTPUT_DIR="$REPO_ROOT/config" + +usage() { + cat << EOF +Usage: $0 [OPTION]... +Render consent.yaml, site.conf and oauth2.conf from the consent chart in terra-helmfile. + + --helmfile_ref REF The terra-helmfile branch or tag to render. Defaults to master. + --helmfile_dir DIR Render a local terra-helmfile checkout as it is, with changes that are + not committed. Cannot be used with --helmfile_ref. + --output_dir DIR Where to write the files. Defaults to config/ in the repo root. + --help Show this help. +EOF + exit 0 +} + +error() { + echo "ERROR: $1" >&2 + exit 1 +} + +parse_cli_args() { + local ref_set=false + while [[ $# -gt 0 ]]; do + [[ "$1" == "--help" || $# -ge 2 ]] || error "Option $1 needs a value. Try --help to see a list of all options." + case "$1" in + --helmfile_ref) + HELMFILE_REF=$2 + ref_set=true + shift 2 + ;; + --helmfile_dir) + HELMFILE_DIR=$2 + shift 2 + ;; + --output_dir) + OUTPUT_DIR=$2 + shift 2 + ;; + --help) + usage + ;; + *) + error "Unknown option: $1. Try --help to see a list of all options." + ;; + esac + done + if [[ "$ref_set" == "true" && -n "$HELMFILE_DIR" ]]; then + error "Use --helmfile_ref or --helmfile_dir, not both." + fi +} + +# Secrets go only in config/.env. A key name like these in the tracked values file is almost +# always a mistake. This check cannot find a secret under a key with a neutral name. +check_local_values() { + local hits + hits=$(grep -inE '^[[:space:]]*[A-Za-z0-9_.-]*(password|secret|token|apikey|api_key|dsn)[A-Za-z0-9_.-]*[[:space:]]*:' "$LOCAL_VALUES" || true) + if [[ -n "$hits" ]]; then + echo "ERROR: $LOCAL_VALUES has a key that looks like a secret:" >&2 + echo "$hits" | sed -E 's/^([0-9]+):[[:space:]]*([^:]+):.*/ line \1: \2/' >&2 + echo "This file is tracked. Put secrets only in config/.env." >&2 + exit 1 + fi +} + +# Sets SOURCE_DIR to a terra-helmfile tree, and prints where it came from. +get_source() { + if [[ -n "$HELMFILE_DIR" ]]; then + [[ -d "$HELMFILE_DIR/charts/consent" ]] || error "$HELMFILE_DIR is not a terra-helmfile checkout." + SOURCE_DIR=$(cd "$HELMFILE_DIR" && pwd) + # Read-only git commands only. Never fetch, checkout, pull or stash in this checkout. + local commit="commit unknown" dirty="" + if git -C "$SOURCE_DIR" rev-parse HEAD > /dev/null 2>&1; then + commit="commit $(git -C "$SOURCE_DIR" rev-parse --short HEAD)" + if [[ -n "$(git -C "$SOURCE_DIR" status --porcelain 2> /dev/null)" ]]; then + dirty=" (uncommitted changes)" + fi + fi + echo "terra-helmfile: $SOURCE_DIR, $commit$dirty" + else + SOURCE_DIR="$WORK_DIR/terra-helmfile" + clone_helmfile "$HELMFILE_REF" "$SOURCE_DIR" || exit 1 + echo "terra-helmfile: $HELMFILE_REF, commit $(git -C "$SOURCE_DIR" rev-parse --short HEAD)" + fi +} + +# helm packages each file:// dependency with the charts/ folder it has at that time, and the +# vendored packages in a checkout can be stale. So build the dependencies of each dependency +# first, then the consent chart, all in a copy. +build_dependencies() { + cp -R "$SOURCE_DIR/charts" "$WORK_DIR/charts" + local consent="$WORK_DIR/charts/consent" dep + for dep in $(grep -E 'repository: *"?file://\.\./' "$consent/Chart.yaml" | sed -E 's#.*file://\.\./([^"]+)"?.*#\1#'); do + if grep -qE '^dependencies:' "$WORK_DIR/charts/$dep/Chart.yaml"; then + rm -rf "$WORK_DIR/charts/$dep/charts" + helm dependency update --skip-refresh "$WORK_DIR/charts/$dep" > /dev/null \ + || error "helm could not build the dependencies of the $dep chart." + fi + done + rm -rf "$consent/charts" + helm dependency update --skip-refresh "$consent" > /dev/null \ + || error "helm could not build the dependencies of the consent chart." +} + +# Helmfile renders the .gotmpl values files with the environment. Only these two fields are in +# use. Any other template syntax means a change that this script must learn about. +render_values() { + local src="$1" dest="$2" + sed -e 's/{{ *\.Values\.Environment\.Name *}}/dev/g' \ + -e 's/{{ *\.Values\.Destination\.ConfigBase *}}/live/g' "$src" > "$dest" + if grep -q '{{' "$dest"; then + error "$src has template syntax that this script cannot render. Update scripts/render-chart-configs.sh." + fi +} + +# Print the block value of one key of a rendered ConfigMap. helm writes each key at two spaces +# and its block content at four spaces. A line at zero or two spaces ends the block. +extract_key() { + local key="$1" file="$2" + awk -v k=" $key: " ' + index($0, k) == 1 { on = 1; next } + on && (/^[^ ]/ || /^ [^ ]/) { on = 0 } + on { sub(/^ /, ""); print } + ' "$file" +} + +render() { + local values="$WORK_DIR/values" + mkdir -p "$values" + render_values "$SOURCE_DIR/values/app/global.yaml.gotmpl" "$values/1-global.yaml" + render_values "$SOURCE_DIR/values/app/global/live.yaml.gotmpl" "$values/2-global-live.yaml" + render_values "$SOURCE_DIR/values/app/consent/live.yaml.gotmpl" "$values/4-consent-live.yaml" + + # The same order as helmfile, with the local values last. + helm template consent "$WORK_DIR/charts/consent" \ + -f "$values/1-global.yaml" \ + -f "$values/2-global-live.yaml" \ + -f "$SOURCE_DIR/values/app/global/live/dev.yaml" \ + -f "$values/4-consent-live.yaml" \ + -f "$SOURCE_DIR/values/app/consent/live/dev.yaml" \ + -f "$LOCAL_VALUES" \ + --show-only templates/configmap.yaml \ + --show-only templates/proxy-configmap.yaml > "$WORK_DIR/rendered.yaml" \ + || error "helm template failed for the consent chart." + + local stage="$WORK_DIR/out" + mkdir -p "$stage" + extract_key consent.yaml "$WORK_DIR/rendered.yaml" > "$stage/consent.yaml" + extract_key oauth2-config "$WORK_DIR/rendered.yaml" > "$stage/oauth2.conf" + extract_key apache-httpd-proxy-config "$WORK_DIR/rendered.yaml" > "$stage/site.conf" + + local f + for f in consent.yaml oauth2.conf site.conf; do + [[ -s "$stage/$f" ]] || error "The render made an empty $f. The chart layout changed. Update scripts/render-chart-configs.sh." + if grep -q '{{' "$stage/$f"; then + error "The rendered $f has template syntax. Update scripts/render-chart-configs.sh." + fi + done + grep -q 'ProxyPass' "$stage/site.conf" || error "The rendered site.conf has no ProxyPass. Update scripts/render-chart-configs.sh." + + # Move the files into place only after all three are good. + mkdir -p "$OUTPUT_DIR" + mv "$stage/consent.yaml" "$stage/oauth2.conf" "$stage/site.conf" "$OUTPUT_DIR/" + echo "Wrote consent.yaml, site.conf and oauth2.conf to $OUTPUT_DIR" +} + +parse_cli_args "$@" +command -v helm > /dev/null || error "Rendering the chart needs helm. See https://helm.sh/docs/intro/install/" +command -v git > /dev/null || error "Rendering the chart needs git." +check_local_values + +WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/render-chart-configs.XXXXXX") +trap 'rm -rf "$WORK_DIR"' EXIT + +get_source +build_dependencies +render diff --git a/scripts/render-configs.sh b/scripts/render-configs.sh index b0cd3ee5b..065a9b7a1 100755 --- a/scripts/render-configs.sh +++ b/scripts/render-configs.sh @@ -3,10 +3,12 @@ # Makes the local config in config/ that config/docker-compose.yaml needs. See DEVNOTES.md # ("Render Configs") and docs/plans/render-local-configs-plan.md. # -# Writes the certs, which rotate every 3 months. +# A run with no options writes only the certs, which rotate every 3 months. Other options turn +# on the other files. # -# You MUST be on the non-split Broad VPN, because the cert step reads the dev cluster. You MUST -# be logged in with gcloud. +# You MUST be on the non-split Broad VPN for the cert step, which reads the dev cluster. You MUST +# be logged in with gcloud. The chart configs need git read access to +# broadinstitute/terra-helmfile, over HTTPS or SSH. # # Each step writes to a staging folder first, and moves its files into place only when the step # succeeds. The script stops at the first step that fails, and that step leaves the old files as @@ -26,15 +28,23 @@ CLUSTER="terra-dev" # Default values that command line options can change. PROJECT="broad-dsde-dev" +HELMFILE_REF="master" +HELMFILE_REF_SET="false" +HELMFILE_DIR="" CONFIG_DIR="$REPO_ROOT/config" +WRITE_CHART_CONFIGS="false" usage() { cat << EOF Usage: $0 [OPTION]... -Write the certs for the local proxy to config/. +Make the local config files in config/. With no options, write only the certs. --project PROJECT Google project of the dev cluster, for the certs. Defaults to broad-dsde-dev. + --helmfile_ref REF The terra-helmfile branch or tag to render. Defaults to master. + --helmfile_dir DIR Render a local terra-helmfile checkout as it is, with changes + that are not committed. Cannot be used with --helmfile_ref. + --write_chart_configs true|false Write consent.yaml, site.conf and oauth2.conf. Defaults to false. --config_dir DIR Where to write the files. Defaults to config/ in the repo root. --help Show this help. EOF @@ -52,11 +62,21 @@ parse_cli_args() { [[ $# -ge 2 ]] || error "Option $1 needs a value. Try --help to see a list of all options." case "$1" in --project) PROJECT=$2 ;; + --helmfile_ref) HELMFILE_REF=$2; HELMFILE_REF_SET="true" ;; + --helmfile_dir) HELMFILE_DIR=$2 ;; + --write_chart_configs) check_bool "$1" "$2"; WRITE_CHART_CONFIGS=$2 ;; --config_dir) CONFIG_DIR=$2 ;; *) error "Unknown option: $1. Try --help to see a list of all options." ;; esac shift 2 done + + if [[ "$HELMFILE_REF_SET" == "true" && -n "$HELMFILE_DIR" ]]; then + error "Use --helmfile_ref or --helmfile_dir, not both." + fi + if [[ -n "$HELMFILE_DIR" ]]; then + HELMFILE_DIR=$(cd "$HELMFILE_DIR" 2> /dev/null && pwd) || error "--helmfile_dir: no such folder." + fi } # Get the credentials for the dev cluster, one time. The script uses its own kubeconfig, so the @@ -93,6 +113,16 @@ write_certs() { mv "$stage/server.crt" "$stage/server.key" "$stage/ca-bundle.crt" "$CONFIG_DIR/" } +write_chart_configs() { + local stage="$STAGE_DIR/chart" + local source_args=(--helmfile_ref "$HELMFILE_REF") + if [[ -n "$HELMFILE_DIR" ]]; then + source_args=(--helmfile_dir "$HELMFILE_DIR") + fi + "$SCRIPT_DIR/render-chart-configs.sh" --output_dir "$stage" "${source_args[@]}" + mv "$stage/consent.yaml" "$stage/site.conf" "$stage/oauth2.conf" "$CONFIG_DIR/" +} + parse_cli_args "$@" mkdir -p "$CONFIG_DIR" || error "Could not make $CONFIG_DIR." # config/ holds secrets. Mode 700 keeps other users on this machine out of it. @@ -110,3 +140,6 @@ trap cleanup EXIT trap 'exit 130' INT TERM write_certs +if [[ "$WRITE_CHART_CONFIGS" == "true" ]]; then + write_chart_configs +fi diff --git a/scripts/templates/local-values.yaml b/scripts/templates/local-values.yaml new file mode 100644 index 000000000..aa322131b --- /dev/null +++ b/scripts/templates/local-values.yaml @@ -0,0 +1,12 @@ +# Helm values for local development. scripts/render-chart-configs.sh applies this file last, +# after the terra-helmfile values for dev. Each value here is a difference from dev, so keep the +# list short and give the reason for each value. +# +# Do not put a secret in this file. It is tracked. Secrets go only in config/.env. + +# The dev Elasticsearch hosts do not exist on a laptop. Use the compose `elastic` service. +elasticSearchServers: + - elastic + +# One local instance. The chart uses this value for rateLimit.podCount. +replicas: 1