fix(corpus): preserve verified decisions and promotion evidence

.github/scripts/__tests__/agents-verifier-context.test.js

@@ -283,6 +283,8 @@ test('buildVerifierContext allows non-default base branches when acceptance crit
   assert.equal(result.shouldRun, true);
   assert.equal(core.outputs.should_run, 'true');
   assert.equal(core.outputs.pr_number, '99');
+  assert.equal(core.outputs.pr_head_sha, 'head-sha-99');
+  assert.equal(core.outputs.target_sha, 'merge-sha-99');
   assert.equal(core.outputs.skip_reason, '');
 });
 

.github/scripts/agents_verifier_context.js

@@ -650,6 +650,7 @@ async function buildVerifierContext({ github, context, core, ciWorkflows }) {
     fs.writeFileSync(diffPath, diffText + '\n', 'utf8');
   }
 
+  core?.setOutput?.('pr_head_sha', pull.head?.sha || '');
   core?.setOutput?.('should_run', 'true');
   core?.setOutput?.('skip_reason', '');
   core?.setOutput?.('pr_number', String(pull.number || ''));

.github/sync-manifest.yml

@@ -548,7 +548,7 @@ scripts:
     description: "Updates PR body with stable completed dependency-provenance results, observer exclusion, and exact-head Gate visibility"
 
   - source: .github/scripts/agents_verifier_context.js
-    description: "Context gathering for agent verifier"
+    description: "Verifier context with exact PR-head identity for replayable corpus evidence"
 
   - source: .github/scripts/agents_dispatch_summary.js
     description: "Summarizes agent dispatch operations"

.github/workflows/maint-79-verifier-corpus-harvest.yml

@@ -72,12 +72,13 @@ jobs:
           body: |
             Automated corpus growth (stranske/Workflows#2819 move 2).
 
-            High-confidence cases derived from realized PR outcomes — see the run
-            summary for the promoted case list. Ambiguous cases were routed to the
-            auto-expiring staging file, not here.
+            This PR may contain staging-only changes; staging is FYI-only and does
+            not affect approval metrics. Only changes to config/model_eval_pilot.json
+            represent promotions. The run summary lists promoted and staged counts.
 
-            Expected verdicts here come from what the world already adjudicated by
-            merging or reverting each PR. The semantic NON_PASS categories
+            Expected verdicts require an exact-head verifier decision and realized
+            outcome, with the verifier run and durable source URL retained. The semantic
+            NON_PASS categories
             (stale-verifier-claim, review-thread-debt, missing-acceptance-criterion)
             remain owner-sourced and are never machine-added.
       - name: Enforce review window before auto-merge

.github/workflows/reusable-agents-verifier.yml

@@ -1257,11 +1257,17 @@ jobs:
           }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ steps.context.outputs.pr_number }}
+          PR_HEAD_SHA: ${{ steps.context.outputs.pr_head_sha }}
+          EVALUATED_SHA: ${{ steps.context.outputs.target_sha }}
+          CI_FAILED: ${{ steps.context.outputs.ci_failed }}
         run: |
           set -euo pipefail
-          pr_number="${{ steps.context.outputs.pr_number }}"
-          if [ -n "$pr_number" ] && [ -f "comparison-comment.md" ]; then
-            gh pr comment "$pr_number" --body-file comparison-comment.md || \
+          verifier_pr="$PR_NUMBER"
+          if [ -n "$verifier_pr" ] && [ -f "comparison-comment.md" ]; then
+            python .workflows-lib/tools/verifier_corpus_evidence.py \
+              --comparison comparison.json --comment comparison-comment.md
+            gh pr comment "$verifier_pr" --body-file comparison-comment.md || \
               echo "Warning: Could not post comment"
           fi
 

config/backplane_participants.json

@@ -90,7 +90,7 @@
       "status": "planned",
       "rationale": "Near-verbatim blueprint signature run_simulation(config, returns) -> RunResult (api.py:400); content-addressed run_id (bundle.py:75-81); 3/3 artifact_discipline + ci_reference_runs. evidence is out of role (0/3 N/A for a computational engine), so 'evidence_refs' is intentionally NOT required. Needs the single replayable envelope + cost/latency + structured warnings + a funds/managers identity map.",
       "issue_deferred": {
-        "expires_at": "2026-09-15T00:00:00Z",
+        "expires_at": "2026-10-15T00:00:00Z",
         "reason": "Deferred until its producer issue is selected; keep planned and do not fabricate implementation status."
       },
       "reference_state": "missing"
@@ -121,7 +121,7 @@
       "status": "planned",
       "rationale": "Among the readiest (sum 14/18); schema'd manifest, first-class data_quality (3/3), validated name registry (3/3 identity). Candidate THIRD prototype tool. Must first register the console script + add pandas, then enforce manifest_schema end-to-end and add tool/git/python provenance + top-level schema_version. evidence is 1/3 and partly out of role; not required.",
       "issue_deferred": {
-        "expires_at": "2026-09-15T00:00:00Z",
+        "expires_at": "2026-10-15T00:00:00Z",
         "reason": "Deferred until console-script/runtime dependency blockers are resolved in a repo-local issue."
       },
       "reference_state": "missing"
@@ -149,7 +149,7 @@
       "status": "planned",
       "rationale": "Near-ready computational tool (5 of 6 standards strong once ev/id discounted as out-of-role for a Monte Carlo engine). Hashed manifest + verifiable bundle + content-addressed idempotency + CI golden runs. Alternate THIRD tool. Needs the unified run.json that finally captures warnings (currently stderr-only) + a cost stub, and the golden gate flipped from ::warning:: to hard-fail. evidence_objects (0/3 N/A) and identity_map (1/3 N/A) are out of role -> NOT required.",
       "issue_deferred": {
-        "expires_at": "2026-09-15T00:00:00Z",
+        "expires_at": "2026-10-15T00:00:00Z",
         "reason": "Deferred until unified run envelope and warning/cost capture are selected as repo-local work."
       },
       "reference_state": "missing"
@@ -175,7 +175,7 @@
       "status": "planned",
       "rationale": "Tier-2: included as the IDENTITY SOURCE (best-in-fleet identity_map 3/3: canonical manager_id + aliases + cik + lei + registry_ids, live EDGAR resolution). NOT a first-class orchestrated tool in the prototype. Its highest-leverage contribution is surfacing canonical manager/provider IDs the other repos join on. Full RunResult envelope + CI-gating its existing eval thresholds are P3+. Therefore only 'identity_refs' is required at first.",
       "issue_deferred": {
-        "expires_at": "2026-09-15T00:00:00Z",
+        "expires_at": "2026-10-15T00:00:00Z",
         "reason": "Deferred until identity-source surfacing is selected as repo-local work."
       },
       "reference_state": "missing"
@@ -202,7 +202,7 @@
       "status": "planned",
       "rationale": "Tier-2: included as the EVIDENCE SOURCE (evidence_objects 2/3, enforced source-doc+page provenance). NOT a first-class orchestrated tool yet: needs a real ingest(package) entry point + manifest + durable storage and real firm/fund alias resolution (aliases_json hard-coded None) before it can be CALLED. Tool-ification deferred to P3+.",
       "issue_deferred": {
-        "expires_at": "2026-09-15T00:00:00Z",
+        "expires_at": "2026-10-15T00:00:00Z",
         "reason": "Deferred until a headless ingest/package entry point and durable manifest work are selected."
       },
       "reference_state": "missing"
@@ -225,7 +225,7 @@
       "status": "candidate",
       "rationale": "CANDIDATE consumer, NOT ACTIVE (status: candidate -> the conformance gate is a no-op for it). Captured architecturally so the cross-over use case is supported: investment-tool evidence objects + canonical identity refs flowing INTO the learning system (e.g. attributing a learning artifact to a source document or a canonical entity). As a CONSUMER it ingests the satellite schemas listed in 'ingests' and is NEVER required to emit a run-contract/v1 run.json. This does NOT make LMS a backplane PRODUCER: LMS remains in the 'excluded' block as a producer (its own EvidenceRecord/MasteryEstimate domain contracts must NOT be conflated with backplane evidence objects). Activating LMS (candidate -> consumer) is a deliberate, reviewable charter decision; P0 only records the architecture.",
       "issue_deferred": {
-        "expires_at": "2026-09-15T00:00:00Z",
+        "expires_at": "2026-10-15T00:00:00Z",
         "reason": "Candidate consumer only; activation needs a separate charter decision before any active issue."
       },
       "reference_state": "not-applicable"
@@ -250,5 +250,5 @@
       "reason": "Workflows is the CONTRACT OWNER / harness, not a research tool that emits run envelopes. It hosts the contract, schemas, registry, validator, and conformance workflow; it does not conform to the contract as a participant. Mirrors langsmith-fleet rollout_status 'contract-owner'. (Workflows-as-app is explicitly excluded.)"
     }
   ],
-  "updated_at": "2026-08-15T07:30:00Z"
+  "updated_at": "2026-09-15T00:52:00Z"
 }

docs/MODEL_SELECTION_POLICY.md

@@ -102,13 +102,17 @@ quality gate and an explicit approval update.
 ### Prepared promotions and rollbacks
 
 `tools/prepare_model_promotion.py` (run by `maint-86`) can *prepare* a selection
-change from a passing benchmark, but never applies one on its own. It only
-prepares a candidate that is the **same family** as the incumbent (e.g. openai
-`gpt-5.x`, anthropic `claude-<line>`), **passed every quality gate** (including
-paired non-inferiority), and costs **≤** the incumbent per accepted review.
-Cross-family swaps are never auto-prepared. It writes the registry mutation
-(recording the prior selection in `selection_history`) and opens a PR; merging
-that PR is the human approval this policy requires — `human_approval_required`
+change from a passing benchmark, but never applies one on its own. Candidates
+must pass every benchmark quality gate and have known, finite, nonnegative costs.
+Same-family candidates (e.g. openai `gpt-5.x`, anthropic `claude-<line>`) costing
+**≤** the incumbent receive `preparation_mode=bounded`. Cross-family or pricier
+candidates receive `preparation_mode=approval-required` and explicit
+`approval_reasons`. The tool selects at most one candidate per provider, preferring
+bounded changes, then lower cost and latency. Both modes retain
+`human_approval_required=true`; preparation metadata does not authorize auto-merge.
+The tool writes the registry mutation (recording the prior selection in
+`selection_history`) for the workflow to open as a PR; merging that PR is the
+human approval this policy requires — `human_approval_required`
 stays true. The inverse path prepares a rollback to the prior selection when the
 active model shows a failed workload-benchmark (a quality-gate breach).
 
@@ -141,3 +145,23 @@ Review at least every 30 days and immediately after any of:
 Update the facts and catalog baseline first, run the paired benchmark, attach
 evidence, then update the explicit selection. Maint-68 propagates the registry;
 consumer slot provider preferences remain intact.
+
+### Replayable corpus evidence
+
+`maint-79` harvests only PR outcomes joined to a bot-published
+`verifier-corpus-decision/v1` record. The comparison verifier records the PR head,
+evaluated merge SHA, repository/PR, run ID and attempt beside the durable report.
+A candidate retains that decision and its comment URL. A stable merge without a
+matching decision is excluded; a NON_PASS decision cannot become a clean PASS
+just because the PR merged. Provider errors and unavailable reviews are not
+benchmark verdicts. A failed merge CI check floors the structured verdict to
+NON_PASS even when every provider says PASS. Missing or invalid CI-gate context
+suppresses publication rather than creating unverifiable benchmark evidence.
+Historical reports without these fields are not backfilled
+from merge metadata. They can enter future harvests after fresh verification.
+
+Case identity includes repository, PR, head and verifier run/attempt. Replaying
+the same evidence does not duplicate a case. Existing adjudicated corpus entries
+keep their historical identifiers. The staging file is FYI-only; a staging-only
+PR does not grow approval metrics. Only additions to `model_eval_pilot.json` count
+as promotions, and existing category/size caps and model approval policy remain.

docs/ci/WORKFLOWS.md

@@ -233,7 +233,7 @@ Scheduled health jobs keep the automation ecosystem aligned:
 * [`maint-77-model-registry-freshness.yml`](../../.github/workflows/maint-77-model-registry-freshness.yml) checks the canonical LLM registry for overdue or unproved decisions, invalid lifecycle/evidence references, and profile/slot drift. Scheduled and manual runs also perform credential-gated provider catalog discovery; catalog additions become review candidates and never auto-promote (scheduled weekly, manual dispatch, PR gate for registry/slot/policy/checker changes).
 * [`maint-78-model-evaluation-pilot.yml`](../../.github/workflows/maint-78-model-evaluation-pilot.yml) runs the frozen 30-case verifier corpus against explicit incumbent and candidate models using repository credentials, then uploads artifact-only paired results. The pilot narrows candidates; it cannot approve or migrate a model.
 * [`maint-79-verifier-corpus-harvest.yml`](../../.github/workflows/maint-79-verifier-corpus-harvest.yml) grows the frozen verifier evaluation corpus from realized PR outcomes (stable merge, revert, resolved follow-up). High-confidence cases auto-promote only after the PR head is unchanged for seven minutes, required checks report and pass, and no active review threads remain; ambiguous cases land in an auto-expiring staging file. It only narrows/expands the corpus; it cannot approve or migrate a model.
-* [`maint-86-model-promotion-prepare.yml`](../../.github/workflows/maint-86-model-promotion-prepare.yml) turns a passing benchmark into a *prepared* verifier-model selection change: a same-family, non-inferior, cost≤ promotion (or a gate-breach rollback) is written to the registry and opened as a PR. The PR is **not** auto-merged — a human merges it to approve, so `human_approval_required` stays true. Dispatch-only until a trustworthy approval benchmark exists.
+* [`maint-86-model-promotion-prepare.yml`](../../.github/workflows/maint-86-model-promotion-prepare.yml) turns a passing benchmark into a *prepared* verifier-model selection change: a passing promotion (preferring same-family, cost≤ candidates; riskier swaps require explicit approval) (or a gate-breach rollback) is written to the registry and opened as a PR. The PR is **not** auto-merged — a human merges it to approve, so `human_approval_required` stays true. Dispatch-only until a trustworthy approval benchmark exists.
 * [`maint-80-langsmith-metrics-dashboard.yml`](../../.github/workflows/maint-80-langsmith-metrics-dashboard.yml) generates weekly LangSmith trace coverage dashboard - downloads metrics from autopilot artifacts, computes coverage, creates issue report (scheduled Monday 9AM UTC, manual dispatch).
 * [`maint-81-langsmith-fleet-conformance.yml`](../../.github/workflows/maint-81-langsmith-fleet-conformance.yml) validates fleet artifact coverage against `config/langsmith_fleet_registry.json` and reports missing/stale/invalid records (scheduled Monday 9:30AM UTC, manual dispatch with optional enforcement).
 * [`maint-82-sync-dependency-campaign.yml`](../../.github/workflows/maint-82-sync-dependency-campaign.yml) refreshes a GitHub-visible sync/dependency campaign issue, persists plan-bound Maint 71 continuation handoffs, and wakes due generated-delivery lanes with idempotency keys. A transient delivery continuation is serialized ahead of candidate work, but an actionable delivery blocker does not suppress a corrective candidate wave.
@@ -249,3 +249,12 @@ Together these workflows define the CI surface area referenced by Gate and the G
 
 * [`selftest-ci.yml`](../../.github/workflows/selftest-ci.yml) runs the repository's own test suite (JS + Python tests, linting, YAML validation) on push and PR, including the langchain verdict, verifier, and structured-output contract tests.
 * [`health-keepalive-e2e.yml`](../../.github/workflows/health-keepalive-e2e.yml) path-filtered E2E test for the keepalive system. Runs only when keepalive-related files change. Supports two modes: orchestration-only (default) and real Codex ping (via `e2e:codex-ping` label).
+
+### Verifier corpus provenance
+
+The reusable verifier publishes `verifier-corpus-decision/v1` beside comparison
+reports using the exact PR head and evaluated target captured in its context.
+Maint 79 joins this bot-published evidence (run/attempt and report URL included)
+to current merged-PR outcomes; missing identities and clean merges with NON_PASS
+decisions cannot create PASS benchmark cases. Staging-only PRs do not change
+approval metrics. See `docs/MODEL_SELECTION_POLICY.md` for the evidence contract.