alert: cache the vault key so a logicsrc outage does not silence alerts

On 2026-10-28 app.logicsrc.com crash-looped 16:10-16:16Z; every alert in that
window failed with "no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)" and
fell back to email, including the qrypt.chat sweep alert. The key was in the
vault the whole time. Now the last key the vault returned is kept in
~/.local/state/alert/smshub-key (0600) and used when the vault cannot answer.
A key smshub rejects is still reported loudly.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: ralyodio

bin/alert

@@ -20,6 +20,9 @@
 # about the key without alerting anyone and sends that same email when it is
 # rejected, so a revoked key is found by cron, not by the next real outage.
 #
+# The vault key is cached in ~/.local/state/alert/smshub-key (0600) and used
+# when the vault cannot be read, so a logicsrc outage does not silence alerts.
+#
 # Exit 0 when sent or deliberately skipped, 1 when nothing went out.
 # Log: ~/.local/state/alert/alert.log
 set -uo pipefail
@@ -75,6 +78,17 @@ for r in csv.DictReader(open(sys.argv[1])):
     if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv")
   fi
   rm -rf "$tmp"
+  # The vault (app.logicsrc.com) lives on the same box this script watches, so it
+  # is down exactly when alerts matter most. Keep the last key it gave us (0600)
+  # and use that when it cannot answer; a key smshub rejects is still caught below.
+  cache="$STATE/smshub-key"
+  if [ -n "$apikey" ]; then
+    [ "$(cat "$cache" 2>/dev/null)" = "$apikey" ] || (umask 077; printf '%s\n' "$apikey" >"$cache.tmp" && mv "$cache.tmp" "$cache")
+  elif [ -s "$cache" ]; then
+    apikey=$(cat "$cache")
+    keysrc="the cached copy $cache (vault smshub--prod did not answer)"
+    log "vault unreachable or key missing: using cached smshub key"
+  fi
 fi
 
 email() { # subject body

test/alert.sh

@@ -56,4 +56,19 @@ run bad --check; rc=$?
 run bad --check; [ ! -s "$tmp/home/mail.out" ] && ok "--check: one key email per 6h" || bad "--check repeated email"
 run good --check && [ ! -f "$tmp/home/.local/state/alert/smshub-key-broken.last" ] && ok "--check on good key: exit 0, clears" || bad "--check good"
 
+
+# Vault outage: the last key the vault gave is cached and used. No logicsrc in
+# this HOME means "vault did not answer"; a fake logicsrc stands in for the vault.
+: >"$tmp/home/mail.out"; rm -f "$tmp/home/.local/state/alert/"*.last
+cat >"$tmp/home/.local/bin/logicsrc" <<'L'
+#!/usr/bin/env bash
+while [ $# -gt 0 ]; do [ "$1" = -o ] && { printf 'team,project,env,category,key,value,updated_at\np,smshub,prod,,SMSHUB_ALERT_API_KEY,good,\n' >"$2"; }; shift; done
+L
+chmod +x "$tmp/home/.local/bin/logicsrc"
+vrun() { HOME="$tmp/home" PATH="$tmp/home/.local/bin:$PATH" "$root/bin/alert" "$@" >/dev/null 2>&1; }
+vrun --key v1 "vault up" && [ "$(stat -c %a "$tmp/home/.local/state/alert/smshub-key")" = 600 ] && ok "vault key used and cached 0600" || bad "vault key cache"
+rm "$tmp/home/.local/bin/logicsrc"
+vrun --key v2 "vault down" && [ ! -s "$tmp/home/mail.out" ] && grep -q 'using cached smshub key' "$tmp/home/.local/state/alert/alert.log" \
+  && ok "vault down: cached key still calls, no email" || bad "vault down: $(mailed)"
+
 exit $fails