diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index da1c52124..205e4be10 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -241,22 +241,42 @@ jobs: # as a fallback for the gateway signal only — it must never mask a # dead Caddy (that exact blind spot let a full outage ship green). # The 4GB box can be slow right after a rebuild — allow a warm-up. - echo "── Verifying public workbench + gateway health ──" + # + # ── …AND it must be OUR commit answering ────────────────────── + # Liveness is not identity. In Aug 2026 a forked product was + # deployed onto this box and served `/health` 200 with a 200 + # workbench for two days — this function would have called that a + # successful deploy on every run, because "something answers" was + # the entire test. `/health` now carries the commit it is serving, + # so the question becomes "did MY commit ship?". A foreign stack, + # a gateway that never restarted, and a mid-apply SSH timeout all + # fail that question; all three passed the old one. + echo "── Verifying public workbench + gateway identity ──" + echo " expecting sha=${GITHUB_SHA}" for i in $(seq 1 24); do - gw_ok=0 - if curl -fsS --max-time 10 "$GATEWAY_URL/health" >/dev/null 2>&1; then - gw_ok=1 - elif curl -fsS --max-time 10 "http://$SSH_HOST:8080/health" >/dev/null 2>&1; then - gw_ok=1 + if ! body=$(curl -fsS --max-time 10 "$GATEWAY_URL/health" 2>/dev/null); then + body=$(curl -fsS --max-time 10 "http://$SSH_HOST:8080/health" 2>/dev/null || echo "") fi + gw_ok=0 + [ -n "$body" ] && gw_ok=1 + served_sha=$(printf '%s' "$body" \ + | grep -o '"sha"[[:space:]]*:[[:space:]]*"[0-9a-f]\{40\}"' \ + | grep -o '[0-9a-f]\{40\}' | head -1 || true) wb_code=$(curl -s -o /dev/null -m 10 -w '%{http_code}' "$WORKBENCH_URL/" 2>/dev/null || echo 000) - if [ "$gw_ok" = 1 ] && printf '%s' "$wb_code" | grep -qE '^[23]'; then - echo " gateway healthy; workbench / -> HTTP $wb_code (poll $i)" + if [ "$gw_ok" = 1 ] && [ "$served_sha" = "$GITHUB_SHA" ] \ + && printf '%s' "$wb_code" | grep -qE '^[23]'; then + echo " ✅ gateway serving ${served_sha}; workbench / -> HTTP $wb_code (poll $i)" return 0 fi - echo " not healthy yet (gateway_ok=$gw_ok workbench=$wb_code, poll $i/24) — 10s…" + echo " not there yet (gateway_ok=$gw_ok sha=${served_sha:-none} workbench=$wb_code, poll $i/24) — 10s…" sleep 10 done + echo " ❌ never converged on ${GITHUB_SHA}. Read the last sha= above:" + echo " none → the gateway did not restart, OR something that" + echo " is not CommandCenter is answering this hostname." + echo " → the box serves a different commit. Check that" + echo " HOSTINGER_HOST is the box you meant, and that a" + echo " pull timer is not fighting this deploy." return 1 } diff --git a/.github/workflows/vps-health.yml b/.github/workflows/vps-health.yml index 9e4ae7309..2505b3e42 100644 --- a/.github/workflows/vps-health.yml +++ b/.github/workflows/vps-health.yml @@ -46,6 +46,13 @@ jobs: healthy: ${{ steps.probe.outputs.healthy }} report: ${{ steps.probe.outputs.report }} steps: + # Full history — the identity check below asks whether the commit the + # box is serving exists in OUR repository, which a shallow clone cannot + # answer (every older commit would read as foreign). + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Probe gateway + workbench id: probe env: @@ -80,6 +87,76 @@ jobs: fi done + # ── Identity: is it US answering, or merely SOMETHING? ─────────── + # The loop above deliberately accepts any HTTP response ("only a + # total absence of response is an outage"). That is the correct rule + # for liveness — and it is exactly why this workflow reported green + # every hour for two days in Aug 2026 while a FORKED PRODUCT was + # deployed onto the CommandCenter box and served this hostname. The + # owner's apps were gone; every check we owned said healthy. Liveness + # and identity are two different questions and only one was asked. + # + # Two independent signals, because each covers the other's blind + # spot: + # * the gateway's `/health` sha — strong, because a fork inherits + # our string constants but cannot inherit our commit ids; + # absent on any box older than the commit that added it. + # * the workbench — present on every build ever shipped, + # but a fork that never rebrands would sail through it. + # Either one reading "foreign" is an outage. + + IDENTITY="unverified" + served_sha="$(curl -s -m 25 "$GATEWAY_URL" 2>/dev/null \ + | grep -o '"sha"[[:space:]]*:[[:space:]]*"[0-9a-f]\{40\}"' \ + | grep -o '[0-9a-f]\{40\}' | head -1 || true)" + if [ -n "$served_sha" ]; then + if git cat-file -e "${served_sha}^{commit}" 2>/dev/null; then + IDENTITY="ours" + REPORT="${REPORT}- **identity**: OK — serving \`${served_sha}\`, a commit in this repo"$'\n' + echo "identity OK (${served_sha})" + else + IDENTITY="foreign" + HEALTHY=0 + REPORT="${REPORT}- **identity**: 🔴 FOREIGN — \`${served_sha}\` is not a commit in this repository. Another product is deployed on this hostname."$'\n' + echo "::error::gateway is serving ${served_sha}, which is not a CommandCenter commit" + fi + fi + + BRAND="unknown" + title="$(curl -s -m 25 "${WORKBENCH_URL}/signin" 2>/dev/null \ + | grep -o '<title>[^<]*' | head -1 | sed -e 's/<[^>]*>//g' || true)" + if [ -n "$title" ]; then + case "$title" in + *CommandCenter*) + BRAND="ours" + REPORT="${REPORT}- **brand**: OK — workbench titled \"${title}\""$'\n' + echo "brand OK (${title})" + ;; + *) + BRAND="foreign" + HEALTHY=0 + REPORT="${REPORT}- **brand**: 🔴 FOREIGN — workbench titled \"${title}\", which is not CommandCenter."$'\n' + echo "::error::workbench is titled '${title}' — not CommandCenter" + ;; + esac + fi + + # A box predating the identity endpoint is not itself an outage, but + # it must not read as a clean bill of health: until it redeploys the + # strong signal is unavailable and only the title defends the + # hostname. With neither signal, nothing here proves this is us — + # which is the state the Aug 2026 takeover would have presented, and + # it must not be silent. + if [ "$IDENTITY" = "unverified" ]; then + if [ "$BRAND" = "ours" ]; then + REPORT="${REPORT}- **identity**: unverified — \`/health\` carries no \`sha\`, so this box predates the identity endpoint. Brand says ours; redeploy to restore the strong check."$'\n' + else + HEALTHY=0 + REPORT="${REPORT}- **identity**: 🔴 UNVERIFIED — \`/health\` carries no \`sha\`, and the workbench title did not identify CommandCenter either. Nothing here proves this hostname is serving us."$'\n' + echo "::error::nothing identifies this hostname as CommandCenter" + fi + fi + echo "healthy=${HEALTHY}" >> "$GITHUB_OUTPUT" { echo "report<.*'\`" + echo " names the product actually being served." + echo "2. Check which box \`HOSTINGER_HOST\` points at in **both** repos" + echo " before deploying anything — a deploy aimed at the wrong box" + echo " is how the hostname was taken, and re-aiming it carelessly" + echo " is how the OTHER product's box gets taken in revenge." + echo "3. On the box, \`git -C /opt/acb/app remote -v\` and the pull" + echo " timer say which repository it now tracks. Stop the timer" + echo " before repointing, or the next tick undoes the repair." } > "$BODY_FILE" if [ -n "$EXISTING" ]; then @@ -199,7 +295,7 @@ jobs: echo "Commented on existing outage issue #$EXISTING" else gh issue create --repo "${{ github.repository }}" \ - --title "VPS unreachable — $(date -u '+%Y-%m-%d %H:%M UTC')" \ + --title "CommandCenter not confirmed serving — $(date -u '+%Y-%m-%d %H:%M UTC')" \ --label "vps-outage" --body-file "$BODY_FILE" fi rm -f "$BODY_FILE" @@ -207,5 +303,5 @@ jobs: - name: Fail the run if unhealthy if: ${{ needs.probe.outputs.healthy != '1' }} run: | - echo "::error::CommandCenter is unreachable — see the vps-outage issue." + echo "::error::CommandCenter not confirmed serving — see the vps-outage issue." exit 1 diff --git a/apps/services/gateway/gateway/main.py b/apps/services/gateway/gateway/main.py index b4dd1b2f0..0cde37d9c 100644 --- a/apps/services/gateway/gateway/main.py +++ b/apps/services/gateway/gateway/main.py @@ -2,8 +2,11 @@ from __future__ import annotations import os +import subprocess from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from functools import lru_cache +from pathlib import Path from acb_auth import (UserContext, UserRole, get_current_user, require_authenticated, require_role) @@ -1213,6 +1216,59 @@ async def relayed_generator(): class Health(BaseModel): status: str env: str + # The commit this box is actually serving. `None` when it cannot be + # resolved (no git, no checkout) — absent evidence, never a fake answer. + sha: str | None = None + + +@lru_cache(maxsize=1) +def _deployed_sha() -> str | None: + """The commit this process is serving, resolved once at first probe. + + **Why a liveness endpoint carries an identity.** `/health` answering 200 + proves *something* is serving this hostname. It does not prove it is US. + On 2026-08-26 that gap ran for two days: a different product was deployed + onto the CommandCenter box, and every verifier we own went green through + it — the hourly `vps-health` probe (it only asks for any HTTP response), + and `deploy.yml`'s own `verify()` (it only asks that `/health` returns + 200). Both would have blessed a deploy that shipped nothing onto a box + running someone else's code. + + A SHA is the discriminator that survives that, and survives a *fork* + specifically: a rebranded fork inherits every string constant we could put + here, so a `product: "commandcenter"` field would keep saying the + reassuring thing after the takeover. A commit id cannot be inherited — + `git cat-file -e ` in this repo is true only for our own history. + That is why the identity is a SHA and not a name. + + Resolution order is explicit-then-derived: `ACB_GIT_SHA` lets the deploy + pin what it believes it shipped, and the git fallback covers the box, + which runs from a checkout. Cached because watchdogs poll this endpoint + and the answer cannot change without a restart. + """ + pinned = os.environ.get("ACB_GIT_SHA", "").strip() + if pinned: + return pinned + try: + repo_root = Path(__file__).resolve().parents[4] + out = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + capture_output=True, + text=True, + timeout=5, + check=False, + ) + sha = out.stdout.strip() + # 40 hex chars or nothing. A partial/garbage answer read as an + # identity is worse than no identity at all. + if out.returncode == 0 and len(sha) == 40 and all( + c in "0123456789abcdef" for c in sha + ): + return sha + except Exception: # identity is best-effort, never fatal + pass + return None def _runtime_checks() -> dict[str, dict]: @@ -1259,7 +1315,7 @@ def _runtime_checks() -> dict[str, dict]: @app.get("/health", response_model=Health, tags=["meta"]) async def health() -> Health: - return Health(status="ok", env=get_settings().acb_env) + return Health(status="ok", env=get_settings().acb_env, sha=_deployed_sha()) @app.get("/health/runtime", tags=["meta"]) diff --git a/project-docs/HANDOFF.md b/project-docs/HANDOFF.md index 78c3df691..0ba656c9c 100644 --- a/project-docs/HANDOFF.md +++ b/project-docs/HANDOFF.md @@ -66,29 +66,53 @@ this file grows a graveyard and the graveyard is what goes stale. # OPEN -### H-1 · Deploy: `main` is many migrations ahead of every box · [OWNER] -- **Check:** compare `ls infra/postgres/[0-9]*.sql | sort -V | tail -1` against - `SELECT max(filename) FROM schema_migrations;` on a box. A gap means still - pending. ⚠️ From a clean checkout with no box access an agent can only get the - first half — report the gap as unverified rather than closing this. - ⚠️ The `[0-9]*` glob and `sort -V` are both load-bearing: a bare `*.sql | tail - -1` answers `schema.generated.sql`, which sorts after every numbered migration - and is not one. That is what the first draft of this Check did. -- **Why:** #437 merged 2026-08-13 and was never deployed; everything since has - stacked behind it, and the pile grows every day. **We cannot roll back** (R6), - so the longer the gap the more lands at once. Deploy applies migrations before - restarting services, so the ORDER is safe — the risk is volume. - ⚠️ Deliberately does not name a migration range: a range here would be state, - which this file must never restate. It was written as "171–175" for one hour - and 176 landed inside it. -- **Authority:** `work_plan.md` §2 WS-27 row · §6 (deploy is owner-gated) -- **Added:** 2026-08-14 +### H-12 · CommandCenter's next migration must be numbered 193+, never 177 · [AGENT] +- **Check:** `ls infra/postgres/[0-9]*.sql | sort -V | tail -1` → if the next + number anyone would take is <= 192, this is still live. On the box: + `SELECT count(*) FROM schema_migrations WHERE filename ~ '^[0-9]+' AND + (substring(filename from '^[0-9]+'))::int BETWEEN 177 AND 192;` → 16 means the + range is occupied. +- **Why:** While Metorite was deployed on this box (2026-08-25 → 08-28) its + deploys applied **177-192 into CommandCenter's own `acb` database**. Our ladder + ends at 176, so the obvious next number is 177 — which is taken, by a file with + a different name and different contents. `schema_migrations` keys on filename, + so ours WOULD apply; the two ladders would then diverge silently and forever. + Verified 2026-08-28: all 16 are present. +- **Authority:** R1 · CLAUDE.md §3.7 · closed-out H-11 +- **Added:** 2026-08-28 · the session that took the box back + +### H-13 · A changed migration RE-RUNS, and one of ours contains a DELETE · [AGENT] +- **Check:** `grep -n "DELETE FROM" infra/postgres/56_purge_synced_done_backlog.sql` + → present, plus `apply_migrations.sh` still re-applying on checksum mismatch + (`grep -n "CHANGED since it was applied" scripts/apply_migrations.sh`), means + still live. +- **Why:** 🔴 **Measured, not theorised — it fired on 2026-08-28 and deleted + rows.** `apply_migrations.sh` re-applies any migration whose sha256 no longer + matches the ledger. Metorite's rebrand changed *comment text* in several of our + migrations, so on the restore deploy eleven files re-ran — including + `56_purge_synced_done_backlog.sql`, which is + `DELETE FROM gtd_items WHERE source <> 'LOCAL' AND disposition = 'DONE'`. + Two completed synced tasks were purged (818 → 816). Recoverable by a full + re-sync, per that migration's own header — this time. + ⚠️ **The general hazard:** re-apply-on-checksum-change assumes every migration + is idempotent. A migration containing an unguarded `DELETE`/`UPDATE` is not, + and a one-character comment edit is enough to fire it. Either migrations must + be guarded (`IF NOT EXISTS`, arming rows — cf. Metorite's 190, which refused to + drop because nobody armed it), or a checksum drift must REFUSE rather than + re-apply. That is a decision, not a patch. +- **Authority:** R6 (cannot roll back) · R7 (name the fence) · `scripts/apply_migrations.sh` +- **Added:** 2026-08-28 · the session that took the box back ### H-2 · Count archived projects on prod BEFORE migration 171 applies · [OWNER] - **Check:** `SELECT count(*) FROM pm_projects WHERE status = 'archived';` on prod. If 171 has already applied, this number is no longer recoverable this way and the query becomes `WHERE archived_root_id = id` — which answers a *different* question. Unanswered → still pending. +- **STATUS 2026-08-28: the window has CLOSED and the number is lost.** 171 is + in the ledger on the box, so the pre-migration count is no longer + recoverable. Per this entry's own instruction, record that it was lost + rather than substituting the other query's answer. Kept only so nobody + re-derives a different number and believes it. - **Why:** ⚠️ **Time-sensitive and ordered against H-1.** 171 changes what "archived" means; the pre-migration count is the only baseline that can tell us whether the lifecycle sweep behaved. Not a deploy blocker — if H-1 happens @@ -96,15 +120,37 @@ this file grows a graveyard and the graveyard is what goes stale. - **Authority:** `work_plan.md` §2 WS-27 row - **Added:** 2026-08-14 · session that built WS-27bj -### H-3 · Rotate the production SSH credentials pasted into a session · [OWNER] +### H-3 · 🔴 Rotate the production root SSH password — disclosed TWICE · [OWNER] - **Check:** can the old password still authenticate? If nobody has rotated it, - it can. Treat as pending until rotation is confirmed. -- **Why:** 🔴 Root credentials for the production VPS were pasted into an agent - transcript. They were **refused and never used** (`work_plan.md` §6), but a - secret in a transcript is a disclosed secret. Rotate, and replace root password - auth with a key while you are there. + it can. Treat as pending until rotation is confirmed. A second Check that + needs no secret: `ssh -o PreferredAuthentications=password -o PubkeyAuthentication=no + root@187.127.179.143` → a password PROMPT (rather than `Permission denied + (publickey)`) means password auth is still enabled and this is still open. +- **Why:** 🔴 Root credentials for the production VPS have now been pasted into + an agent transcript **twice** — 2026-08-14 and again **2026-08-28**, the + second time for `root@187.127.179.143` while trying to unblock the H-11 + recovery. Both times they were **refused and never used** + (`work_plan.md` §6). That refusal protects the box; it does **not** un-disclose + the secret. A password in a transcript is a leaked password, and this one now + sits in two. + ⚠️ **The recurrence is the finding.** It happened the second time for the + same reason as the first: an owner-gated repair felt urgent, and handing over + the password looked like the fastest way through. It will keep happening + while root-password auth remains possible, so the durable fix is not "be more + careful" — it is **key-only auth**, which makes the paste useless and + therefore pointless. +- **What to do:** + 1. `passwd root` on the box — new password, stored in a password manager, + never typed into a chat. + 2. Set `PermitRootLogin prohibit-password` and `PasswordAuthentication no` in + `/etc/ssh/sshd_config`, then `systemctl restart sshd`. ⚠️ Confirm the + deploy key in `HOSTINGER_SSH_KEY` still authenticates **in a second + terminal before closing the first**, or this locks everyone out. + 3. Rotating the password does **not** rotate `HOSTINGER_SSH_KEY`; that key is + unchanged since 2026-06-10 and is a separate decision. - **Authority:** `work_plan.md` §6 · `specs/engineering_practice.md` (security) - **Added:** 2026-08-14 · carried from the session that refused them + · second disclosure recorded 2026-08-28 ### H-4 · WS-27bj: build the admin surface for org-wide vocabularies · [AGENT] - **Check:** `rg -n "refuse_org_wide_write" apps/services/gateway/gateway/routes/projects/` @@ -123,7 +169,9 @@ this file grows a graveyard and the graveyard is what goes stale. still dark. - **Why:** Default OFF and it gates **only** the affordance that *creates* an org-wide row, never the read union — which is already on and inert until a row - exists. Flipping it is a restart, not a release. Requires H-1 first. + exists. Flipping it is a restart, not a release. Its prerequisite (the box + running current `main`) was satisfied 2026-08-28 when the box was taken back + from Metorite — the old H-1 that this line used to name is closed. - **Authority:** `specs/project_management_app.md` §9.11 · `work_plan.md` §6 - **Added:** 2026-08-14 · session that built WS-27bj diff --git a/tests/unit/test_health_identity.py b/tests/unit/test_health_identity.py new file mode 100644 index 000000000..cf2b85351 --- /dev/null +++ b/tests/unit/test_health_identity.py @@ -0,0 +1,109 @@ +"""`/health` must say WHICH commit is serving, not merely that something is. + +**The fence.** On 2026-08-26 a different product (Metorite, a fork) was +deployed onto the CommandCenter VPS and answered on +`commandcenter.fracktal.in` for two days. Every verifier we own went green +through it, because every one of them asked a liveness question: + + * `.github/workflows/vps-health.yml` — "any HTTP response means the stack + is serving" (its own comment), hourly, green throughout; + * `.github/workflows/deploy.yml` `verify()` — `/health` returns 200 and the + workbench returns 2xx/3xx, which the foreign stack also satisfied. + +The owner's symptom was "none of my apps are there". Nothing was wrong with +the apps: liveness had been mistaken for identity. + +This module fences the endpoint half — `/health` carries a commit id. The two +consumer halves are shell and are fenced in their own workflows: +`deploy.yml` compares the served sha to `GITHUB_SHA` (did MY commit ship?) and +`vps-health.yml` runs `git cat-file -e` on it against a full checkout (is that +OUR history at all?). A SHA is used rather than a product name on purpose — a +rebranded fork inherits our string constants but cannot inherit our commits. +""" +from __future__ import annotations + + +def _fresh_sha(monkeypatch, **env): + """Call `_deployed_sha()` with the import-time cache cleared.""" + from gateway.main import _deployed_sha + + for key, value in env.items(): + if value is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, value) + _deployed_sha.cache_clear() + try: + return _deployed_sha() + finally: + _deployed_sha.cache_clear() + + +def test_health_reports_a_commit_identity(monkeypatch): + """The endpoint carries `sha` — the field the verifiers read.""" + from fastapi.testclient import TestClient + from gateway import main + + main._deployed_sha.cache_clear() + monkeypatch.setenv("ACB_GIT_SHA", "a" * 40) + try: + with TestClient(main.app) as client: + body = client.get("/health").json() + finally: + main._deployed_sha.cache_clear() + + assert body["status"] == "ok" + # The whole point: liveness AND identity, in the same answer. + assert body["sha"] == "a" * 40 + + +def test_pinned_sha_wins_over_the_checkout(monkeypatch): + """`ACB_GIT_SHA` lets the deploy pin what it believes it shipped.""" + assert _fresh_sha(monkeypatch, ACB_GIT_SHA="b" * 40) == "b" * 40 + + +def test_a_garbage_answer_is_no_identity_rather_than_a_wrong_one(monkeypatch): + """A partial or non-hex answer must resolve to `None`. + + A truncated sha read as an identity is worse than an absent one: absent + is a state the verifiers can refuse on, whereas a malformed value that + happens not to match reads as "wrong commit" and sends whoever is holding + the incident after a deploy that actually succeeded. + """ + import subprocess + + class _Result: + returncode = 0 + stdout = "not-a-sha\n" + + monkeypatch.setattr(subprocess, "run", lambda *a, **k: _Result()) + assert _fresh_sha(monkeypatch, ACB_GIT_SHA=None) is None + + +def test_identity_is_never_fatal(monkeypatch): + """`git` missing or exploding must not take the liveness probe down. + + `/health` is what the on-box watchdog restarts services from. An identity + lookup that could raise would turn "I cannot tell you which commit" into + "the box is dead", and the watchdog would act on it. + """ + import subprocess + + def _boom(*a, **k): + raise OSError("git not found") + + monkeypatch.setattr(subprocess, "run", _boom) + assert _fresh_sha(monkeypatch, ACB_GIT_SHA=None) is None + + +def test_a_real_checkout_resolves_to_forty_hex(monkeypatch): + """In this repo the git fallback answers, and answers well-formed. + + This is the case the box runs in — `/opt/acb/app` is a checkout — so if + the fallback were broken, the deployed boxes would report `null` and the + verifiers would have nothing to compare. + """ + sha = _fresh_sha(monkeypatch, ACB_GIT_SHA=None) + assert sha is not None, "git fallback resolved nothing in a real checkout" + assert len(sha) == 40 + assert all(c in "0123456789abcdef" for c in sha)