CXF-221: agent skills for post-sync verification, update/rollback, and failure diagnosis

evals/README.md

@@ -15,7 +15,7 @@ fixture and produces a scored JSONL record with the full S0–S11 stage funnel.
 | `evals/runner/` | Runner + scorer (`run.ts` CLI, driver interfaces, stage gates) |
 | `evals/runner/drivers/` | Driver implementations — Tier-0 local/static driver; authoring contract in `drivers/README.md` |
 | `evals/scenarios/` | Scenario definitions (`tier1-directory.json`, `tier1-directory-guide-only.json`, `tier1-directory-full.json`, `pre1-directory-proceed.json`, `pre1-noiam-park.json`) |
-| `evals/skills-bundle/` | Skill-bundle mount point (v0.3.0 manifest — seven skills in `skills/`) |
+| `evals/skills-bundle/` | Skill-bundle mount point (v0.4.0 manifest — ten skills in `skills/`) |
 | `evals/results/` | JSONL run records (gitignored; `.gitkeep` committed) |
 
 ## How to run
@@ -242,7 +242,6 @@ real-tenant driver and the tool surface are available.
 
 ## Non-goals
 
-- The remaining three skills (verify-connector-output, update-and-rollback, diagnose-authoring-failure) — later PRs.
 - Tier-2 real sandbox providers and the qualitative LLM-judge tier.
 - Operator-side activation E2E leg (redeeming the approval token) — those two
   fields are `skipped_human_boundary`.

evals/runner/agent.test.ts

@@ -78,7 +78,7 @@ test("buildPrompt pre1 branch names the pre1Path, both skills, and the output co
       basicAuth: {username: "connector@example.com", password: "fixture-token"},
       bearerToken: "fixture-token",
     },
-    skillBundle: {mode: "full", version: "0.3.0"},
+    skillBundle: {mode: "full", version: "0.4.0"},
     model: "together/deepseek-ai/DeepSeek-V4-Flash-0731",
     reasoningEffort: "high",
     kind: "pre1",

evals/runner/scenario.test.ts

@@ -100,7 +100,7 @@ test("loadScenario loads the pre1-directory-proceed scenario (kind pre1, proceed
   assert.equal(s.seed, undefined)
   assert.equal(s.requiredSourceFiles, undefined)
   assert.equal(s.skillBundle.mode, "full")
-  assert.equal(s.skillBundle.version, "0.3.0")
+  assert.equal(s.skillBundle.version, "0.4.0")
 })
 
 test("loadScenario loads the pre1-noiam-park scenario (kind pre1, park)", () => {

evals/runner/skills_bundle.test.ts

@@ -14,8 +14,8 @@ import {loadScenario} from "./scenario.ts"
 const execFileAsync = promisify(execFile)
 const RUN = "evals/runner/run.ts"
 const BUNDLE = "evals/skills-bundle/bundle.json"
-const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec"]
-const VERSION = "0.3.0"
+const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec", "verify-connector-output", "update-and-rollback", "diagnose-authoring-failure"]
+const VERSION = "0.4.0"
 
 function readBundle(): {version: string; skills: {name: string; version: string; path: string}[]} {
   return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; version: string; path: string}[]}
@@ -56,7 +56,7 @@ test("(a) each SKILL.md exists with the locked frontmatter contract", () => {
 test("(b) every bundle.json path resolves to an existing file", () => {
   const bundle = readBundle()
   assert.equal(bundle.version, VERSION)
-  assert.equal(bundle.skills.length, 7)
+  assert.equal(bundle.skills.length, 10)
   assert.deepEqual(bundle.skills.map((s) => s.name), SKILLS)
   const skillsRoot = resolve("skills")
   for (const skill of bundle.skills) {
@@ -133,6 +133,52 @@ const SKILL_LITERALS: Record<string, string[]> = {
     "Parking with evidence",
     "authority ladder",
   ],
+  "verify-connector-output": [
+    "never invent data to make a demo appear complete",
+    "SYNC_STATUS_DONE",
+    "SYNC_STATUS_RUNNING",
+    "terminal status after ~10 polls, STOP and report",
+    "data-anomaly auto-pause",
+    "significant drop in sync data",
+    "sync_disabled",
+    "every grant principal references an emitted resource",
+    "entitlement ID references an emitted entitlement",
+    "ID stability",
+    "/admin/connector/",
+  ],
+  "update-and-rollback": [
+    "serve image does not match the revision-pinned runtime image",
+    "/api/v1/connector-authoring/rollbacks",
+    "target_revision_id",
+    "approval_token_id",
+    "activation_epoch",
+    "image digest",
+    "Do not redeem the approval token",
+    "Do not clear runtime fields, call the provisioner directly, or mutate the",
+    "evidence is unsatisfied",
+    "if no ACTIVE row after ~10 polls, STOP and report",
+    "no terminal status after ~10 polls, STOP and report",
+    "SYNC_STATUS_ERROR",
+    "SYNC_STATUS_DISABLED",
+    "Do not print, log, or otherwise expose the OWNER bearer token value",
+  ],
+  "diagnose-authoring-failure": [
+    "262144 byte compile limit",
+    "1048576 byte limit",
+    "is_secret",
+    "credential re-entry required",
+    "missing type",
+    "unregistered transport",
+    "ticketing.enabled must be true when ticketing is configured",
+    "activation evidence is unsatisfied",
+    "Invalid token provided",
+    "ConnectionOK",
+    "HostCallOK",
+    "Poll with backoff",
+    "row after ~10 polls, stop and report",
+    "c1_connector_service_get",
+    "status.lastError",
+  ],
 }
 
 test("(c) each SKILL.md carries the locked section markers, content literals, ASCII-only bodies, and stays <= 200 lines", () => {
@@ -152,7 +198,7 @@ test("(c) each SKILL.md carries the locked section markers, content literals, AS
 test("(d) the full-mode scenario parses with mode full and the two pinned scenarios keep their locked modes", () => {
   const full = loadScenario("evals/scenarios/tier1-directory-full.json")
   assert.equal(full.skillBundle.mode, "full")
-  assert.equal(full.skillBundle.version, "0.3.0")
+  assert.equal(full.skillBundle.version, "0.4.0")
   assert.equal(full.id, "tier1-directory-full")
   const none = loadScenario("evals/scenarios/tier1-directory.json")
   assert.equal(none.skillBundle.mode, "none")
@@ -201,8 +247,25 @@ test("(e) CLI end-to-end: full-mode Tier-0 run exits 0 and the record meta carri
     const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n")
     const meta = JSON.parse(lines[0]) as Record<string, unknown>
     assert.equal(meta.skill_bundle_mode, "full")
-    assert.equal(meta.skill_bundle_version, "0.3.0")
+    assert.equal(meta.skill_bundle_version, "0.4.0")
   } finally {
     rmSync(dir, {recursive: true, force: true})
   }
 })
+
+test("(f) every skill ships a non-empty SOURCES.md naming its pinned sources", () => {
+  const skillsRoot = resolve("skills")
+  for (const name of SKILLS) {
+    const sourcesPath = join(skillsRoot, name, "SOURCES.md")
+    assert.ok(existsSync(sourcesPath), `missing SOURCES.md: ${name}`)
+    const content = readFileSync(sourcesPath, "utf8")
+    assert.ok(content.length > 0, `empty SOURCES.md: ${name}`)
+  }
+  // The three new skills must name the c1 pin (decision 5: nothing written
+  // from model memory) so a dropped or truncated pin fails the gate.
+  const c1Pin = "2e5f53eb441a93087d9754085ca17a5061e125ea"
+  for (const name of ["verify-connector-output", "update-and-rollback", "diagnose-authoring-failure"]) {
+    const content = readFileSync(join(skillsRoot, name, "SOURCES.md"), "utf8")
+    assert.ok(content.includes(c1Pin), `${name}: SOURCES.md does not name the c1 pin`)
+  }
+})

evals/scenarios/pre1-directory-proceed.json

@@ -29,7 +29,7 @@
   },
   "skillBundle": {
     "mode": "full",
-    "version": "0.3.0"
+    "version": "0.4.0"
   },
   "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
   "reasoningEffort": "high"

evals/scenarios/pre1-noiam-park.json

@@ -27,7 +27,7 @@
   },
   "skillBundle": {
     "mode": "full",
-    "version": "0.3.0"
+    "version": "0.4.0"
   },
   "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
   "reasoningEffort": "high"

evals/scenarios/tier1-directory-full.json

@@ -27,7 +27,7 @@
   },
   "skillBundle": {
     "mode": "full",
-    "version": "0.3.0"
+    "version": "0.4.0"
   },
   "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
   "reasoningEffort": "high",

evals/skills-bundle/README.md

@@ -2,6 +2,24 @@
 
 This directory is the skill-bundle mount point for the eval harness.
 
+## v0.4.0 - the ten skills
+
+The bundle ships ten skills: the seven prior skills plus three post-funnel
+and cross-cutting skills at `0.1.0`:
+
+- `verify-connector-output` - post-11: post-sync verification (counts,
+  grant wiring, ID stability across re-sync, UI spot-check).
+- `update-and-rollback` - post-activation: same-catalog update flow with
+  the image-digest reuse rule, the rotation STOP/escalate limitation, and
+  REST-only OWNER-gated rollback.
+- `diagnose-authoring-failure` - cross-cutting: symptom -> cause -> fix
+  router over the common-failures table, the draft-test FAIL reading, and
+  where logs live.
+
+The seven prior skills are unchanged; the bundle version is `0.4.0`. The
+three new skills pin the c1 contract sources at `2e5f53eb…` (see each
+skill's SOURCES.md).
+
 ## v0.3.0 — the seven skills
 
 The bundle ships seven skills: the five funnel skills plus two new pre-1
@@ -43,4 +61,4 @@ names a skill and a `path` relative to this directory pointing into
 `skills/` (the canonical home). Private drivers mount per the manifest —
 the manifest indirection is the contract, not a directory copy. The scenario
 file selects the bundle via `skillBundle.mode` (`full`) and pins
-`skillBundle.version` (`0.2.0`); the runner records both in every run record.
+`skillBundle.version` (`0.4.0`); the runner records both in every run record.

evals/skills-bundle/bundle.json

@@ -1,12 +1,15 @@
 {
-  "version": "0.3.0",
+  "version": "0.4.0",
   "skills": [
     {"name": "author-in-app-connector", "version": "0.2.1", "path": "../../skills/author-in-app-connector/SKILL.md"},
     {"name": "read-authoring-contract", "version": "0.2.0", "path": "../../skills/read-authoring-contract/SKILL.md"},
     {"name": "write-connector-source", "version": "0.2.0", "path": "../../skills/write-connector-source/SKILL.md"},
     {"name": "build-and-test", "version": "0.2.0", "path": "../../skills/build-and-test/SKILL.md"},
     {"name": "deploy-and-activate", "version": "0.2.0", "path": "../../skills/deploy-and-activate/SKILL.md"},
     {"name": "design-access-model", "version": "0.1.0", "path": "../../skills/design-access-model/SKILL.md"},
-    {"name": "source-openapi-spec", "version": "0.1.0", "path": "../../skills/source-openapi-spec/SKILL.md"}
+    {"name": "source-openapi-spec", "version": "0.1.0", "path": "../../skills/source-openapi-spec/SKILL.md"},
+    {"name": "verify-connector-output", "version": "0.1.0", "path": "../../skills/verify-connector-output/SKILL.md"},
+    {"name": "update-and-rollback", "version": "0.1.0", "path": "../../skills/update-and-rollback/SKILL.md"},
+    {"name": "diagnose-authoring-failure", "version": "0.1.0", "path": "../../skills/diagnose-authoring-failure/SKILL.md"}
   ]
 }

skills/README.md

@@ -1,8 +1,12 @@
 # Agent skills
 
-The seven skills shipped in this batch, authored against
-the v0.0.26 DSL contract and the 23-tool tenant MCP surface. Each skill's
-`SOURCES.md` names the pinned sources with their SHAs.
+The ten skills shipped in this batch: the five funnel skills and two
+pre-1 skills authored against the v0.0.26 DSL contract and the 23-tool
+tenant MCP surface, plus three post-funnel and cross-cutting skills
+(`verify-connector-output`, `update-and-rollback`,
+`diagnose-authoring-failure`) over the tenant connector and authoring
+tool surface. Each skill's `SOURCES.md` names the pinned sources with
+their SHAs.
 
 | Skill | Stage coverage | Source basis |
 |---|---|---|
@@ -13,6 +17,9 @@ the v0.0.26 DSL contract and the 23-tool tenant MCP surface. Each skill's
 | `deploy-and-activate` | Stages 6–8, 11 — app, provision, configure, deploy, mint, handoff | MCP-served guide, `authoring.proto`, lifecycle doc |
 | `design-access-model` | Pre-1 — access-model design for net-new providers | baton-admin `design-baton-access-model` @ `6fe6886f…` |
 | `source-openapi-spec` | Pre-1 — OpenAPI spec sourcing + IAM go/no-go | claude-marketplace `source-openapi-spec` @ `0cc5ac2a…` |
+| `verify-connector-output` | Post-11 - post-sync verification (counts, grant wiring, ID stability, UI spot-check) | MCP-served guide, lifecycle doc, `probe-contracts.md` @ `0cc5ac2a…` |
+| `update-and-rollback` | Post-activation - same-catalog update + REST rollback | MCP-served guide, `authoring.proto`, lifecycle doc |
+| `diagnose-authoring-failure` | Cross-cutting - symptom -> cause -> fix router | lifecycle doc, baton-admin `diagnose-connector-failure` @ `6fe6886f…` |
 
 The eval bundle (`evals/skills-bundle/bundle.json`) is a manifest pointing
 into this directory; the skill bodies live here as the single source of

skills/diagnose-authoring-failure/SKILL.md

@@ -0,0 +1,91 @@
+---
+name: diagnose-authoring-failure
+description: Use when a build, draft test, activation, or production sync fails and you need the symptom-to-cause-to-fix route. Do not use when the connector is healthy and you are verifying sync output - use verify-connector-output; do not use when updating or rolling back a healthy live connector - use update-and-rollback.
+version: 0.1.0
+---
+
+# diagnose-authoring-failure
+
+Symptom -> cause -> fix router for authored-connector failures. Built on the
+lifecycle doc's common-failures table, the draft-test evidence reading, and
+where logs live. Ports the taxonomy approach of baton-admin
+`diagnose-connector-failure`; the baton-admin skill's repo-local CLI
+tooling is replaced by the tenant MCP tools named below.
+
+## Workflow
+
+1. Start from the symptom: the exact error text, the failing step (build,
+   draft test, activation, production sync), and the IDs at hand.
+2. Route the symptom through the table below; collect the smallest evidence
+   needed for that cause.
+3. Apply the fix, then re-run the smallest verification step that failed.
+4. Output a chat diagnostic summary: failure summary, classified cause,
+   evidence used, likely owner (source, schema, runtime, credentials, or
+   platform), one concrete next fix, and the smallest verification command.
+
+## Symptom -> cause -> fix
+
+| Symptom | Cause / fix |
+|---------|-------------|
+| Build rejected: `connector source exceeds the 262144 byte compile limit` | The esbuild-bundled source is over 256 KiB. Trim the source set; large OpenAPI-derived spec assets are the usual weight. |
+| Build rejected: `connector bundle with embedded runtime specs is <N> bytes, above the 1048576 byte limit` | The bundled `connector.js` plus its embedded runtime specs is over 1 MiB - a separate cap from the 256 KiB source limit, hit after bundling. Trim the source or the generated spec assets it embeds. |
+| Build rejected: `credential-class config field must be marked is_secret` | A token/secret/password-named field in `runtime-schema.json` lacks `is_secret: true`. The `secret:` spelling is not read. |
+| Draft test: `credential re-entry required: <fields>` | Configure the instance credentials before the draft test. |
+| Draft test: `connector config field X is missing type` | Add `"type": "string"` (etc.) to the field in `runtime.config_schema`. |
+| Sync error mentions an `unregistered transport` | A `node` or `reuse` references a transport missing from `connector({ transports: ... })`. Register that same transport object, rebuild, and retest. |
+| Draft test: `ticketing.enabled must be true when ticketing is configured` | The runtime-schema carries a `ticketing` block the connector code does not back. Remove the block (and any `actions` / `policy_surface` entries referencing dropped code); `enabled: false` is not a valid off-switch. |
+| Activation reports `activation evidence is unsatisfied` | No PASS test-sync evidence binds this revision. Run the draft test (with credentials set) and confirm it passed before asking the OWNER to review a fresh approval URL. |
+| Production sync `Invalid token provided` | The API token is wrong or truncated. Re-configure with the full token, then re-run. |
+
+## Draft-test FAIL reading
+
+The evidence row is authoritative: poll
+`c1_connector_authoring_get_test_run_evidence` and read the `result` (PASS
+or FAIL) and the `error` field. Poll with backoff (e.g. every 5-10s); if no
+row after ~10 polls, stop and report `NotFound`/pending. The FAIL reason
+lives on the evidence row; it is not always logged when the read activity
+succeeds but the outcome evaluation returns FAIL. PASS requires all of:
+
+- `ConnectionOK` - Validate succeeded
+- `HostCallOK` - GetMetadata succeeded
+- No read error and no write attempt
+- The config version handle matches the candidate revision
+- The runtime image digest matches the revision-pinned image
+
+A FAIL row (or no row yet) means activation stays `evidence is unsatisfied`
+until a new draft test writes PASS.
+
+## Where logs live
+
+Use the product's connector activity and sync logs - the same surface you
+use to view any connector in your tenant. There is no separate
+operator-only log surface. The connector's status row
+(`c1_connector_service_get` -> `status.status`, `status.lastError`) is the
+authoritative outcome for a sync.
+
+## Exit criteria
+
+- Every one of the nine common-failures rows routes to its documented fix.
+- A draft-test FAIL is read from the evidence row, not from completion.
+- The logs location and the status row are named.
+- The body contains the literals `262144 byte compile limit`,
+  `1048576 byte limit`, `is_secret`, `credential re-entry required`,
+  `missing type`, `unregistered transport`,
+  `ticketing.enabled must be true when ticketing is configured`,
+  `activation evidence is unsatisfied`, `Invalid token provided`,
+  `ConnectionOK`, `HostCallOK`, `c1_connector_service_get`, and
+  `status.lastError`.
+
+## Anti-patterns
+
+- Do not start with broad full-suite runs when a small probe can isolate
+  the issue.
+- Do not hide unresolved drift with waivers or mock shaping.
+- Do not expose auth material, tokens, or customer data in diagnostics.
+- Do not guess a fix without reading the evidence row first.
+
+## Blocker protocol
+
+If the same validation or runtime error remains unchanged after 2 failed
+fix cycles on the same error, stop and report the exact error text instead
+of guessing further.

skills/diagnose-authoring-failure/SOURCES.md

@@ -0,0 +1,13 @@
+# Sources - diagnose-authoring-failure
+
+Authored against the pinned sources below (decision 5: nothing written from
+model memory). Source-of-truth precedence: (a) MCP-served guide, (b)
+`authoring.proto`, (c) lifecycle doc, (e) baton-admin
+`diagnose-connector-failure`.
+
+| Source | Pin / SHA | What this skill quotes |
+|---|---|---|
+| MCP-served guide | c1 `2e5f53eb441a93087d9754085ca17a5061e125ea` | The Caps table (262144-byte compile limit, 1048576-byte bundle limit) and the Evidence and credentials contract (`credential re-entry required`; activation fails closed until a PASS evidence row binds the revision digests). |
+| Authoring proto | c1 `2e5f53eb441a93087d9754085ca17a5061e125ea` | `GetTestRunEvidence` (poll `(catalog_id, revision_id, test_run_id)`; `result` PASS/FAIL + `error` on the evidence row). |
+| Lifecycle doc | c1 `2e5f53eb441a93087d9754085ca17a5061e125ea` | The Debugging section: the common-failures table (all nine rows), the draft-test FAIL reading (evidence row authoritative; PASS requires `ConnectionOK`, `HostCallOK`, no read error and no write attempt, config version handle match, runtime image digest match), and where logs live (product connector activity and sync logs; the status row `c1_connector_service_get` -> `status.status`, `status.lastError`). |
+| baton-admin `diagnose-connector-failure` | `6fe6886f607ed0d2e48a616c30e7ce4bffc32489` | The taxonomy approach: start from the symptom, classify the failure surface, collect the smallest evidence, keep the diagnosis focused on symptom/evidence/owner/next-fix/rerun-target, and the output contract (failure summary, classified surface, evidence used, likely owner, one concrete next fix, smallest verification command). |