rushtong · Oct 23, 2026
@@ -43,29 +43,35 @@ local stack needs. Do not edit these files by hand. The next render replaces the
 | File | Source | Option |
 |---|---|---|
 | `server.crt`, `server.key`, `ca-bundle.crt` | The `local-dev-cert` secret in the dev cluster | Always |
+| `consent.yaml`, `site.conf`, `oauth2.conf` | The consent chart in `terra-helmfile`, with the dev values and [local-values.yaml](scripts/templates/local-values.yaml) | `--write_chart_configs true` |
 
-You need `gcloud`, `kubectl` and `openssl`, and the non-split Broad VPN for the cluster steps. The
-script stops at the first step that fails, and that step leaves the old files as they were. Run:
+You need `gcloud`, `kubectl`, `helm`, `git` and `openssl`, and the non-split Broad VPN for the
+cluster steps. The chart configs need read access to `broadinstitute/terra-helmfile`, with git
+HTTPS credentials or a GitHub SSH key. The script tries HTTPS first, then SSH. The script stops at
+the first step that fails, and that step leaves the old files as they were. The first time, run:
 
 ```bash
-./scripts/render-configs.sh
+./scripts/render-configs.sh --write_chart_configs true
 ```
 
-Every 3 months the certs rotate. Run the script again to get the new certs.
+* After a cert rotation, run `./scripts/render-configs.sh` with no options.
+* To test a `terra-helmfile` branch, run `./scripts/render-configs.sh --write_chart_configs true
+  --helmfile_ref my-branch`. For changes that are not pushed, use `--helmfile_dir
+  ~/develop/terra-helmfile`. To render the chart configs without the VPN, run
+  [render-chart-configs.sh](scripts/render-chart-configs.sh) on its own.
 
-### Configure
+The rendered files are the same as the files that dev runs. The only local changes are in
+`local-values.yaml`. See [the plan](docs/plans/render-local-configs-plan.md#all-local-changes).
 
-Copy `src/test/resources/consent-config.yml` to `config/consent.yaml` (the `config/` directory is
-git-ignored) and fill in the values for your environment. The `googleStore` section is the only
-part that needs cloud credentials; see the next section.
+#### Liquibase
 
 `liquibase.runOnStartup` is an optional flag. The application runs the Liquibase changesets at
 startup only when the flag is `true`. If a config omits the flag or the whole `liquibase` block,
 the flag is `false`. Environments that run the Liquibase migration Job omit it. In
 `terra-helmfile`, that Job is a separate Kubernetes Job that Argo CD runs before the deployment
-(sync-wave `-1`). The template sets it to `true`, because a local database
-restored from an older dump needs the newer changesets. If you render `consent.yaml` from
-`terra-helmfile`, add this block yourself, or pass `-Ddw.liquibase.runOnStartup=true`:
+(sync-wave `-1`). So the rendered `consent.yaml` has no `liquibase` block. A local database
+restored from an older dump needs the newer changesets, so pass
+`-Ddw.liquibase.runOnStartup=true`, or add this block to your config:
 
 ```yaml
 liquibase:

@@ -8,3 +8,33 @@
 gcloud_account() {
   gcloud auth list --filter=status:ACTIVE --format='value(account)' 2> /dev/null | head -1
 }
+
+# Clone broadinstitute/terra-helmfile at REF into DEST, as duos-ui does: over HTTPS first, then
+# over SSH. Other arguments go to `git clone`. GIT_TERMINAL_PROMPT=0 makes git fail instead of
+# asking for a password, so a missing HTTPS credential falls through to SSH.
+clone_helmfile() {
+  local ref="$1" dest="$2"
+  shift 2
+  rm -rf "$dest"
+  if GIT_TERMINAL_PROMPT=0 git clone --quiet --depth 1 --branch "$ref" "$@" \
+    https://github.com/broadinstitute/terra-helmfile.git "$dest" 2> /dev/null; then
+    return 0
+  fi
+  echo "Could not clone terra-helmfile over HTTPS. Trying SSH." >&2
+  rm -rf "$dest"
+  if GIT_TERMINAL_PROMPT=0 git clone --quiet --depth 1 --branch "$ref" "$@" \
+    git@github.com:broadinstitute/terra-helmfile.git "$dest"; then
+    return 0
+  fi
+  echo "ERROR: Could not clone terra-helmfile at $ref over HTTPS or SSH. Check that your git credentials or SSH key can read broadinstitute/terra-helmfile." >&2
+  return 1
+}
+
+# Check that a flag value is true or false.
+check_bool() {
+  local flag="$1" value="$2"
+  if [[ "$value" != "true" && "$value" != "false" ]]; then
+    echo "ERROR: $flag must be true or false, not '$value'." >&2
+    exit 1
+  fi
+}
@@ -0,0 +1,200 @@
+#!/bin/bash
+#
+# Renders consent.yaml, site.conf and oauth2.conf from the consent chart in terra-helmfile, with
+# the dev values and scripts/templates/local-values.yaml. The rendered files are the reference
+# for local config: do not edit them. Put a local change in local-values.yaml, with its reason.
+#
+# You MUST have helm and git installed. Without --helmfile_dir, you MUST also have git read
+# access to broadinstitute/terra-helmfile, over HTTPS or SSH. This script does not need the VPN.
+#
+# render-configs.sh calls this script. It also runs on its own.
+#
+
+set -eu
+set -o pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR/lib/common.sh"
+LOCAL_VALUES="$SCRIPT_DIR/templates/local-values.yaml"
+
+HELMFILE_REF="master"
+HELMFILE_DIR=""
+OUTPUT_DIR="$REPO_ROOT/config"
+
+usage() {
+  cat << EOF
+Usage: $0 [OPTION]...
+Render consent.yaml, site.conf and oauth2.conf from the consent chart in terra-helmfile.
+
+  --helmfile_ref REF   The terra-helmfile branch or tag to render. Defaults to master.
+  --helmfile_dir DIR   Render a local terra-helmfile checkout as it is, with changes that are
+                       not committed. Cannot be used with --helmfile_ref.
+  --output_dir DIR     Where to write the files. Defaults to config/ in the repo root.
+  --help               Show this help.
+EOF
+  exit 0
+}
+
+error() {
+  echo "ERROR: $1" >&2
+  exit 1
+}
+
+parse_cli_args() {
+  local ref_set=false
+  while [[ $# -gt 0 ]]; do
+    [[ "$1" == "--help" || $# -ge 2 ]] || error "Option $1 needs a value. Try --help to see a list of all options."
+    case "$1" in
+      --helmfile_ref)
+        HELMFILE_REF=$2
+        ref_set=true
+        shift 2
+        ;;
+      --helmfile_dir)
+        HELMFILE_DIR=$2
+        shift 2
+        ;;
+      --output_dir)
+        OUTPUT_DIR=$2
+        shift 2
+        ;;
+      --help)
+        usage
+        ;;
+      *)
+        error "Unknown option: $1. Try --help to see a list of all options."
+        ;;
+    esac
+  done
+  if [[ "$ref_set" == "true" && -n "$HELMFILE_DIR" ]]; then
+    error "Use --helmfile_ref or --helmfile_dir, not both."
+  fi
+}
+
+# Secrets go only in config/.env. A key name like these in the tracked values file is almost
+# always a mistake. This check cannot find a secret under a key with a neutral name.
+check_local_values() {
+  local hits
+  hits=$(grep -inE '^[[:space:]]*[A-Za-z0-9_.-]*(password|secret|token|apikey|api_key|dsn)[A-Za-z0-9_.-]*[[:space:]]*:' "$LOCAL_VALUES" || true)
+  if [[ -n "$hits" ]]; then
+    echo "ERROR: $LOCAL_VALUES has a key that looks like a secret:" >&2
+    echo "$hits" | sed -E 's/^([0-9]+):[[:space:]]*([^:]+):.*/  line \1: \2/' >&2
+    echo "This file is tracked. Put secrets only in config/.env." >&2
+    exit 1
+  fi
+}
+
+# Sets SOURCE_DIR to a terra-helmfile tree, and prints where it came from.
+get_source() {
+  if [[ -n "$HELMFILE_DIR" ]]; then
+    [[ -d "$HELMFILE_DIR/charts/consent" ]] || error "$HELMFILE_DIR is not a terra-helmfile checkout."
+    SOURCE_DIR=$(cd "$HELMFILE_DIR" && pwd)
+    # Read-only git commands only. Never fetch, checkout, pull or stash in this checkout.
+    local commit="commit unknown" dirty=""
+    if git -C "$SOURCE_DIR" rev-parse HEAD > /dev/null 2>&1; then
+      commit="commit $(git -C "$SOURCE_DIR" rev-parse --short HEAD)"
+      if [[ -n "$(git -C "$SOURCE_DIR" status --porcelain 2> /dev/null)" ]]; then
+        dirty=" (uncommitted changes)"
+      fi
+    fi
+    echo "terra-helmfile: $SOURCE_DIR, $commit$dirty"
+  else
+    SOURCE_DIR="$WORK_DIR/terra-helmfile"
+    clone_helmfile "$HELMFILE_REF" "$SOURCE_DIR" || exit 1
+    echo "terra-helmfile: $HELMFILE_REF, commit $(git -C "$SOURCE_DIR" rev-parse --short HEAD)"
+  fi
+}
+
+# helm packages each file:// dependency with the charts/ folder it has at that time, and the
+# vendored packages in a checkout can be stale. So build the dependencies of each dependency
+# first, then the consent chart, all in a copy.
+build_dependencies() {
+  cp -R "$SOURCE_DIR/charts" "$WORK_DIR/charts"
+  local consent="$WORK_DIR/charts/consent" dep
+  for dep in $(grep -E 'repository: *"?file://\.\./' "$consent/Chart.yaml" | sed -E 's#.*file://\.\./([^"]+)"?.*#\1#'); do
+    if grep -qE '^dependencies:' "$WORK_DIR/charts/$dep/Chart.yaml"; then
+      rm -rf "$WORK_DIR/charts/$dep/charts"
+      helm dependency update --skip-refresh "$WORK_DIR/charts/$dep" > /dev/null \
+        || error "helm could not build the dependencies of the $dep chart."
+    fi
+  done
+  rm -rf "$consent/charts"
+  helm dependency update --skip-refresh "$consent" > /dev/null \
+    || error "helm could not build the dependencies of the consent chart."
+}
+
+# Helmfile renders the .gotmpl values files with the environment. Only these two fields are in
+# use. Any other template syntax means a change that this script must learn about.
+render_values() {
+  local src="$1" dest="$2"
+  sed -e 's/{{ *\.Values\.Environment\.Name *}}/dev/g' \
+      -e 's/{{ *\.Values\.Destination\.ConfigBase *}}/live/g' "$src" > "$dest"
+  if grep -q '{{' "$dest"; then
+    error "$src has template syntax that this script cannot render. Update scripts/render-chart-configs.sh."
+  fi
+}
+
+# Print the block value of one key of a rendered ConfigMap. helm writes each key at two spaces
+# and its block content at four spaces. A line at zero or two spaces ends the block.
+extract_key() {
+  local key="$1" file="$2"
+  awk -v k="  $key: " '
+    index($0, k) == 1 { on = 1; next }
+    on && (/^[^ ]/ || /^  [^ ]/) { on = 0 }
+    on { sub(/^    /, ""); print }
+  ' "$file"
+}
+
+render() {
+  local values="$WORK_DIR/values"
+  mkdir -p "$values"
+  render_values "$SOURCE_DIR/values/app/global.yaml.gotmpl" "$values/1-global.yaml"
+  render_values "$SOURCE_DIR/values/app/global/live.yaml.gotmpl" "$values/2-global-live.yaml"
+  render_values "$SOURCE_DIR/values/app/consent/live.yaml.gotmpl" "$values/4-consent-live.yaml"
+
+  # The same order as helmfile, with the local values last.
+  helm template consent "$WORK_DIR/charts/consent" \
+    -f "$values/1-global.yaml" \
+    -f "$values/2-global-live.yaml" \
+    -f "$SOURCE_DIR/values/app/global/live/dev.yaml" \
+    -f "$values/4-consent-live.yaml" \
+    -f "$SOURCE_DIR/values/app/consent/live/dev.yaml" \
+    -f "$LOCAL_VALUES" \
+    --show-only templates/configmap.yaml \
+    --show-only templates/proxy-configmap.yaml > "$WORK_DIR/rendered.yaml" \
+    || error "helm template failed for the consent chart."
+
+  local stage="$WORK_DIR/out"
+  mkdir -p "$stage"
+  extract_key consent.yaml "$WORK_DIR/rendered.yaml" > "$stage/consent.yaml"
+  extract_key oauth2-config "$WORK_DIR/rendered.yaml" > "$stage/oauth2.conf"
+  extract_key apache-httpd-proxy-config "$WORK_DIR/rendered.yaml" > "$stage/site.conf"
+
+  local f
+  for f in consent.yaml oauth2.conf site.conf; do
+    [[ -s "$stage/$f" ]] || error "The render made an empty $f. The chart layout changed. Update scripts/render-chart-configs.sh."
+    if grep -q '{{' "$stage/$f"; then
+      error "The rendered $f has template syntax. Update scripts/render-chart-configs.sh."
+    fi
+  done
+  grep -q 'ProxyPass' "$stage/site.conf" || error "The rendered site.conf has no ProxyPass. Update scripts/render-chart-configs.sh."
+
+  # Move the files into place only after all three are good.
+  mkdir -p "$OUTPUT_DIR"
+  mv "$stage/consent.yaml" "$stage/oauth2.conf" "$stage/site.conf" "$OUTPUT_DIR/"
+  echo "Wrote consent.yaml, site.conf and oauth2.conf to $OUTPUT_DIR"
+}
+
+parse_cli_args "$@"
+command -v helm > /dev/null || error "Rendering the chart needs helm. See https://helm.sh/docs/intro/install/"
+command -v git > /dev/null || error "Rendering the chart needs git."
+check_local_values
+
+WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/render-chart-configs.XXXXXX")
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+get_source
+build_dependencies
+render
@@ -3,10 +3,12 @@
 # Makes the local config in config/ that config/docker-compose.yaml needs. See DEVNOTES.md
 # ("Render Configs") and docs/plans/render-local-configs-plan.md.
 #
-# Writes the certs, which rotate every 3 months.
+# A run with no options writes only the certs, which rotate every 3 months. Other options turn
+# on the other files.
 #
-# You MUST be on the non-split Broad VPN, because the cert step reads the dev cluster. You MUST
-# be logged in with gcloud.
+# You MUST be on the non-split Broad VPN for the cert step, which reads the dev cluster. You MUST
+# be logged in with gcloud. The chart configs need git read access to
+# broadinstitute/terra-helmfile, over HTTPS or SSH.
 #
 # Each step writes to a staging folder first, and moves its files into place only when the step
 # succeeds. The script stops at the first step that fails, and that step leaves the old files as
@@ -26,15 +28,23 @@ CLUSTER="terra-dev"
 
 # Default values that command line options can change.
 PROJECT="broad-dsde-dev"
+HELMFILE_REF="master"
+HELMFILE_REF_SET="false"
+HELMFILE_DIR=""
 CONFIG_DIR="$REPO_ROOT/config"
+WRITE_CHART_CONFIGS="false"
 
 usage() {
   cat << EOF
 Usage: $0 [OPTION]...
-Write the certs for the local proxy to config/.
+Make the local config files in config/. With no options, write only the certs.
 
   --project PROJECT                 Google project of the dev cluster, for the certs.
                                     Defaults to broad-dsde-dev.
+  --helmfile_ref REF                The terra-helmfile branch or tag to render. Defaults to master.
+  --helmfile_dir DIR                Render a local terra-helmfile checkout as it is, with changes
+                                    that are not committed. Cannot be used with --helmfile_ref.
+  --write_chart_configs true|false  Write consent.yaml, site.conf and oauth2.conf. Defaults to false.
   --config_dir DIR                  Where to write the files. Defaults to config/ in the repo root.
   --help                            Show this help.
 EOF
@@ -52,11 +62,21 @@ parse_cli_args() {
     [[ $# -ge 2 ]] || error "Option $1 needs a value. Try --help to see a list of all options."
     case "$1" in
       --project) PROJECT=$2 ;;
+      --helmfile_ref) HELMFILE_REF=$2; HELMFILE_REF_SET="true" ;;
+      --helmfile_dir) HELMFILE_DIR=$2 ;;
+      --write_chart_configs) check_bool "$1" "$2"; WRITE_CHART_CONFIGS=$2 ;;
       --config_dir) CONFIG_DIR=$2 ;;
       *) error "Unknown option: $1. Try --help to see a list of all options." ;;
     esac
     shift 2
   done
+
+  if [[ "$HELMFILE_REF_SET" == "true" && -n "$HELMFILE_DIR" ]]; then
+    error "Use --helmfile_ref or --helmfile_dir, not both."
+  fi
+  if [[ -n "$HELMFILE_DIR" ]]; then
+    HELMFILE_DIR=$(cd "$HELMFILE_DIR" 2> /dev/null && pwd) || error "--helmfile_dir: no such folder."
+  fi
 }
 
 # Get the credentials for the dev cluster, one time. The script uses its own kubeconfig, so the
@@ -93,6 +113,16 @@ write_certs() {
   mv "$stage/server.crt" "$stage/server.key" "$stage/ca-bundle.crt" "$CONFIG_DIR/"
 }
 
+write_chart_configs() {
+  local stage="$STAGE_DIR/chart"
+  local source_args=(--helmfile_ref "$HELMFILE_REF")
+  if [[ -n "$HELMFILE_DIR" ]]; then
+    source_args=(--helmfile_dir "$HELMFILE_DIR")
+  fi
+  "$SCRIPT_DIR/render-chart-configs.sh" --output_dir "$stage" "${source_args[@]}"
+  mv "$stage/consent.yaml" "$stage/site.conf" "$stage/oauth2.conf" "$CONFIG_DIR/"
+}
+
 parse_cli_args "$@"
 mkdir -p "$CONFIG_DIR" || error "Could not make $CONFIG_DIR."
 # config/ holds secrets. Mode 700 keeps other users on this machine out of it.
@@ -110,3 +140,6 @@ trap cleanup EXIT
 trap 'exit 130' INT TERM
 
 write_certs
+if [[ "$WRITE_CHART_CONFIGS" == "true" ]]; then
+  write_chart_configs
+fi
@@ -0,0 +1,12 @@
+# Helm values for local development. scripts/render-chart-configs.sh applies this file last,
+# after the terra-helmfile values for dev. Each value here is a difference from dev, so keep the
+# list short and give the reason for each value.
+#
+# Do not put a secret in this file. It is tracked. Secrets go only in config/.env.
+
+# The dev Elasticsearch hosts do not exist on a laptop. Use the compose `elastic` service.
+elasticSearchServers:
+  - elastic
+
+# One local instance. The chart uses this value for rateLimit.podCount.
+replicas: 1