fix(H-11): /health carries the deployed commit, and both verifiers check it

.github/workflows/deploy.yml

@@ -241,22 +241,42 @@ jobs:
             # as a fallback for the gateway signal only — it must never mask a
             # dead Caddy (that exact blind spot let a full outage ship green).
             # The 4GB box can be slow right after a rebuild — allow a warm-up.
-            echo "── Verifying public workbench + gateway health ──"
+            #
+            # ── …AND it must be OUR commit answering ──────────────────────
+            # Liveness is not identity. In Aug 2026 a forked product was
+            # deployed onto this box and served `/health` 200 with a 200
+            # workbench for two days — this function would have called that a
+            # successful deploy on every run, because "something answers" was
+            # the entire test. `/health` now carries the commit it is serving,
+            # so the question becomes "did MY commit ship?". A foreign stack,
+            # a gateway that never restarted, and a mid-apply SSH timeout all
+            # fail that question; all three passed the old one.
+            echo "── Verifying public workbench + gateway identity ──"
+            echo "   expecting sha=${GITHUB_SHA}"
             for i in $(seq 1 24); do
-              gw_ok=0
-              if curl -fsS --max-time 10 "$GATEWAY_URL/health" >/dev/null 2>&1; then
-                gw_ok=1
-              elif curl -fsS --max-time 10 "http://$SSH_HOST:8080/health" >/dev/null 2>&1; then
-                gw_ok=1
+              if ! body=$(curl -fsS --max-time 10 "$GATEWAY_URL/health" 2>/dev/null); then
+                body=$(curl -fsS --max-time 10 "http://$SSH_HOST:8080/health" 2>/dev/null || echo "")
               fi
+              gw_ok=0
+              [ -n "$body" ] && gw_ok=1
+              served_sha=$(printf '%s' "$body" \
+                | grep -o '"sha"[[:space:]]*:[[:space:]]*"[0-9a-f]\{40\}"' \
+                | grep -o '[0-9a-f]\{40\}' | head -1 || true)
               wb_code=$(curl -s -o /dev/null -m 10 -w '%{http_code}' "$WORKBENCH_URL/" 2>/dev/null || echo 000)
-              if [ "$gw_ok" = 1 ] && printf '%s' "$wb_code" | grep -qE '^[23]'; then
-                echo "  gateway healthy; workbench / -> HTTP $wb_code (poll $i)"
+              if [ "$gw_ok" = 1 ] && [ "$served_sha" = "$GITHUB_SHA" ] \
+                 && printf '%s' "$wb_code" | grep -qE '^[23]'; then
+                echo "  ✅ gateway serving ${served_sha}; workbench / -> HTTP $wb_code (poll $i)"
                 return 0
               fi
-              echo "  not healthy yet (gateway_ok=$gw_ok workbench=$wb_code, poll $i/24) — 10s…"
+              echo "  not there yet (gateway_ok=$gw_ok sha=${served_sha:-none} workbench=$wb_code, poll $i/24) — 10s…"
               sleep 10
             done
+            echo "  ❌ never converged on ${GITHUB_SHA}. Read the last sha= above:"
+            echo "     none    → the gateway did not restart, OR something that"
+            echo "               is not CommandCenter is answering this hostname."
+            echo "     <other> → the box serves a different commit. Check that"
+            echo "               HOSTINGER_HOST is the box you meant, and that a"
+            echo "               pull timer is not fighting this deploy."
             return 1
           }
 

.github/workflows/vps-health.yml

@@ -46,6 +46,13 @@ jobs:
       healthy: ${{ steps.probe.outputs.healthy }}
       report: ${{ steps.probe.outputs.report }}
     steps:
+      # Full history — the identity check below asks whether the commit the
+      # box is serving exists in OUR repository, which a shallow clone cannot
+      # answer (every older commit would read as foreign).
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
       - name: Probe gateway + workbench
         id: probe
         env:
@@ -80,6 +87,76 @@ jobs:
             fi
           done
 
+          # ── Identity: is it US answering, or merely SOMETHING? ───────────
+          # The loop above deliberately accepts any HTTP response ("only a
+          # total absence of response is an outage"). That is the correct rule
+          # for liveness — and it is exactly why this workflow reported green
+          # every hour for two days in Aug 2026 while a FORKED PRODUCT was
+          # deployed onto the CommandCenter box and served this hostname. The
+          # owner's apps were gone; every check we owned said healthy. Liveness
+          # and identity are two different questions and only one was asked.
+          #
+          # Two independent signals, because each covers the other's blind
+          # spot:
+          #   * the gateway's `/health` sha — strong, because a fork inherits
+          #     our string constants but cannot inherit our commit ids;
+          #     absent on any box older than the commit that added it.
+          #   * the workbench <title> — present on every build ever shipped,
+          #     but a fork that never rebrands would sail through it.
+          # Either one reading "foreign" is an outage.
+
+          IDENTITY="unverified"
+          served_sha="$(curl -s -m 25 "$GATEWAY_URL" 2>/dev/null \
+            | grep -o '"sha"[[:space:]]*:[[:space:]]*"[0-9a-f]\{40\}"' \
+            | grep -o '[0-9a-f]\{40\}' | head -1 || true)"
+          if [ -n "$served_sha" ]; then
+            if git cat-file -e "${served_sha}^{commit}" 2>/dev/null; then
+              IDENTITY="ours"
+              REPORT="${REPORT}- **identity**: OK — serving \`${served_sha}\`, a commit in this repo"$'\n'
+              echo "identity OK (${served_sha})"
+            else
+              IDENTITY="foreign"
+              HEALTHY=0
+              REPORT="${REPORT}- **identity**: 🔴 FOREIGN — \`${served_sha}\` is not a commit in this repository. Another product is deployed on this hostname."$'\n'
+              echo "::error::gateway is serving ${served_sha}, which is not a CommandCenter commit"
+            fi
+          fi
+
+          BRAND="unknown"
+          title="$(curl -s -m 25 "${WORKBENCH_URL}/signin" 2>/dev/null \
+            | grep -o '<title>[^<]*</title>' | head -1 | sed -e 's/<[^>]*>//g' || true)"
+          if [ -n "$title" ]; then
+            case "$title" in
+              *CommandCenter*)
+                BRAND="ours"
+                REPORT="${REPORT}- **brand**: OK — workbench titled \"${title}\""$'\n'
+                echo "brand OK (${title})"
+                ;;
+              *)
+                BRAND="foreign"
+                HEALTHY=0
+                REPORT="${REPORT}- **brand**: 🔴 FOREIGN — workbench titled \"${title}\", which is not CommandCenter."$'\n'
+                echo "::error::workbench is titled '${title}' — not CommandCenter"
+                ;;
+            esac
+          fi
+
+          # A box predating the identity endpoint is not itself an outage, but
+          # it must not read as a clean bill of health: until it redeploys the
+          # strong signal is unavailable and only the title defends the
+          # hostname. With neither signal, nothing here proves this is us —
+          # which is the state the Aug 2026 takeover would have presented, and
+          # it must not be silent.
+          if [ "$IDENTITY" = "unverified" ]; then
+            if [ "$BRAND" = "ours" ]; then
+              REPORT="${REPORT}- **identity**: unverified — \`/health\` carries no \`sha\`, so this box predates the identity endpoint. Brand says ours; redeploy to restore the strong check."$'\n'
+            else
+              HEALTHY=0
+              REPORT="${REPORT}- **identity**: 🔴 UNVERIFIED — \`/health\` carries no \`sha\`, and the workbench title did not identify CommandCenter either. Nothing here proves this hostname is serving us."$'\n'
+              echo "::error::nothing identifies this hostname as CommandCenter"
+            fi
+          fi
+
           echo "healthy=${HEALTHY}" >> "$GITHUB_OUTPUT"
           {
             echo "report<<REPORT_EOF"
@@ -164,7 +241,7 @@ jobs:
           if [ "${HEALTHY}" = "1" ]; then
             if [ -n "$EXISTING" ]; then
               gh issue comment "$EXISTING" --repo "${{ github.repository }}" \
-                --body "Recovered — public endpoints are responding again as of $(date -u '+%Y-%m-%dT%H:%M:%SZ'). Closing."
+                --body "Recovered — CommandCenter is reachable **and** identified itself as of $(date -u '+%Y-%m-%dT%H:%M:%SZ'). Closing."
               gh issue close "$EXISTING" --repo "${{ github.repository }}"
               echo "Closed outage issue #$EXISTING"
             else
@@ -175,7 +252,9 @@ jobs:
 
           BODY_FILE="$(mktemp)"
           {
-            echo "The scheduled health check could not reach CommandCenter."
+            echo "The scheduled health check did not confirm that CommandCenter"
+            echo "is serving its hostnames. Read the per-signal lines below: an"
+            echo "UNREACHABLE line and a FOREIGN line are very different faults."
             echo
             printf '%s\n' "${REPORT}"
             echo "Checked at $(date -u '+%Y-%m-%dT%H:%M:%SZ') from a GitHub runner."
@@ -192,20 +271,37 @@ jobs:
             echo "   listener state captured every 10 minutes, including during the outage."
             echo "2. Re-run this workflow with **diagnose = true** to pull that log out over SSH."
             echo "3. Hostinger VPS metrics — if uptime keeps climbing, the box never rebooted."
+            echo
+            echo "### If a FOREIGN or UNVERIFIED identity line is above"
+            echo "The box is up and serving — it is just not serving **us**. This"
+            echo "is not an outage to restart your way out of; restarting the"
+            echo "services would only restart the other product. It happened on"
+            echo "2026-08-25, when a forked repo's deploy pointed at this box and"
+            echo "replaced CommandCenter in place for two days."
+            echo
+            echo "1. \`curl -s https://commandcenter.fracktal.in/signin | grep -o '<title>.*</title>'\`"
+            echo "   names the product actually being served."
+            echo "2. Check which box \`HOSTINGER_HOST\` points at in **both** repos"
+            echo "   before deploying anything — a deploy aimed at the wrong box"
+            echo "   is how the hostname was taken, and re-aiming it carelessly"
+            echo "   is how the OTHER product's box gets taken in revenge."
+            echo "3. On the box, \`git -C /opt/acb/app remote -v\` and the pull"
+            echo "   timer say which repository it now tracks. Stop the timer"
+            echo "   before repointing, or the next tick undoes the repair."
           } > "$BODY_FILE"
 
           if [ -n "$EXISTING" ]; then
             gh issue comment "$EXISTING" --repo "${{ github.repository }}" --body-file "$BODY_FILE"
             echo "Commented on existing outage issue #$EXISTING"
           else
             gh issue create --repo "${{ github.repository }}" \
-              --title "VPS unreachable — $(date -u '+%Y-%m-%d %H:%M UTC')" \
+              --title "CommandCenter not confirmed serving — $(date -u '+%Y-%m-%d %H:%M UTC')" \
               --label "vps-outage" --body-file "$BODY_FILE"
           fi
           rm -f "$BODY_FILE"
 
       - name: Fail the run if unhealthy
         if: ${{ needs.probe.outputs.healthy != '1' }}
         run: |
-          echo "::error::CommandCenter is unreachable — see the vps-outage issue."
+          echo "::error::CommandCenter not confirmed serving — see the vps-outage issue."
           exit 1

apps/services/gateway/gateway/main.py

@@ -2,8 +2,11 @@
 from __future__ import annotations
 
 import os
+import subprocess
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
+from functools import lru_cache
+from pathlib import Path
 
 from acb_auth import (UserContext, UserRole, get_current_user,
                       require_authenticated, require_role)
@@ -1213,6 +1216,59 @@ async def relayed_generator():
 class Health(BaseModel):
     status: str
     env: str
+    # The commit this box is actually serving. `None` when it cannot be
+    # resolved (no git, no checkout) — absent evidence, never a fake answer.
+    sha: str | None = None
+
+
+@lru_cache(maxsize=1)
+def _deployed_sha() -> str | None:
+    """The commit this process is serving, resolved once at first probe.
+
+    **Why a liveness endpoint carries an identity.** `/health` answering 200
+    proves *something* is serving this hostname. It does not prove it is US.
+    On 2026-08-26 that gap ran for two days: a different product was deployed
+    onto the CommandCenter box, and every verifier we own went green through
+    it — the hourly `vps-health` probe (it only asks for any HTTP response),
+    and `deploy.yml`'s own `verify()` (it only asks that `/health` returns
+    200). Both would have blessed a deploy that shipped nothing onto a box
+    running someone else's code.
+
+    A SHA is the discriminator that survives that, and survives a *fork*
+    specifically: a rebranded fork inherits every string constant we could put
+    here, so a `product: "commandcenter"` field would keep saying the
+    reassuring thing after the takeover. A commit id cannot be inherited —
+    `git cat-file -e <sha>` in this repo is true only for our own history.
+    That is why the identity is a SHA and not a name.
+
+    Resolution order is explicit-then-derived: `ACB_GIT_SHA` lets the deploy
+    pin what it believes it shipped, and the git fallback covers the box,
+    which runs from a checkout. Cached because watchdogs poll this endpoint
+    and the answer cannot change without a restart.
+    """
+    pinned = os.environ.get("ACB_GIT_SHA", "").strip()
+    if pinned:
+        return pinned
+    try:
+        repo_root = Path(__file__).resolve().parents[4]
+        out = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=repo_root,
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+        sha = out.stdout.strip()
+        # 40 hex chars or nothing. A partial/garbage answer read as an
+        # identity is worse than no identity at all.
+        if out.returncode == 0 and len(sha) == 40 and all(
+            c in "0123456789abcdef" for c in sha
+        ):
+            return sha
+    except Exception:  # identity is best-effort, never fatal
+        pass
+    return None
 
 
 def _runtime_checks() -> dict[str, dict]:
@@ -1259,7 +1315,7 @@ def _runtime_checks() -> dict[str, dict]:
 
 @app.get("/health", response_model=Health, tags=["meta"])
 async def health() -> Health:
-    return Health(status="ok", env=get_settings().acb_env)
+    return Health(status="ok", env=get_settings().acb_env, sha=_deployed_sha())
 
 
 @app.get("/health/runtime", tags=["meta"])