From 9a8488681edb0672072741690ccd7bcb15682c8b Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 18:10:37 -0400 Subject: [PATCH 01/21] =?UTF-8?q?feat:=20Skills=20extension=20phase=203=20?= =?UTF-8?q?=E2=80=94=20CLI,=20TUI,=20directory=20reads,=20frontmatter=20ch?= =?UTF-8?q?eck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2248. Completes SEP-2640 support across all three clients, and closes the one obligation #2234 deliberately left open. - `checkSkillFrontmatterMatch` compares a served SKILL.md's own YAML frontmatter against the entry the listing advertised, field by field. No digest can cover this: a digest is taken over the bytes the server served, so it proves the file was not altered in transit and says nothing about whether the listing described it honestly. Needs a real YAML parser; `yaml` was already a root dependency, so this adds no package — but it does newly put it on core/'s import graph, hence the three bundler `external` lists. - `resources/directory/read`: result schemas defined against the normative text, `InspectorClient.readResourceDirectory`, a Directory section on the Skills screen, and `--method resources/directory/read` in the CLI. The client refuses the call locally when the server did not declare `directoryRead`, which is the SEP's MUST NOT. A child the directory lists but the entry does not is marked `not listed` rather than merged into the manifest — the SEP calls a directory result a live observation and forbids treating it as extending the manifest. - CLI: `skills/list`, `skills/get` and `resources/directory/read`, plus `--verify` — one NDJSON report per skill, a summary on stderr, exit 7 on a violation. Reads are sequential; a conforming manifest may declare 512 entries. - TUI: a Skills pane, shown only when the server declares the extension. Each row carries its conformance verdict as a glyph as well as a colour, since the pane is read over ssh and through script(1). Enter verifies. - `verifySkills` re-throws `AuthRecoveryRequiredError` rather than recording it per file: it says the session's authorization expired, so absorbing it would report N identical read failures and swallow the error the TUI and the web commands key off to reauthorize. Two open questions settled, both in code comments where they will be found: `skills/get` carries no caching attributes because SEP-2640 leaves the question open in as many words, so requiring them would fail a conforming server; and there is no `PagedSkillsState` because every consumer of this list is a whole-catalog verdict — computed over page one of three, "this server's skills conform" is not merely partial but wrong. The fixture grows two skills, both for checks that were otherwise undemonstrable: `lying-listing` (listing and file disagree, digest still verifies) and `stale-manifest` (serves a file its manifest does not declare). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/cli/README.md | 66 ++- .../cli/__tests__/run-method-skills.test.ts | 222 ++++++++ .../cli/__tests__/skills-verify-cli.test.ts | 150 +++++ clients/cli/src/cli.ts | 29 +- clients/cli/src/error-handler.ts | 10 + clients/cli/src/handlers/consume-outcome.ts | 14 +- clients/cli/src/handlers/method-types.ts | 34 +- clients/cli/src/handlers/run-method.ts | 84 +++ clients/cli/src/handlers/skills-verify.ts | 36 ++ clients/cli/tsup.config.ts | 7 + clients/tui/README.md | 3 +- clients/tui/__tests__/App.test.tsx | 55 ++ clients/tui/__tests__/SkillsTab.test.tsx | 537 ++++++++++++++++++ clients/tui/__tests__/Tabs.test.tsx | 32 ++ clients/tui/src/App.tsx | 69 +++ clients/tui/src/components/SkillsTab.tsx | 416 ++++++++++++++ clients/tui/src/components/Tabs.tsx | 12 + clients/tui/src/components/tabsConfig.ts | 6 + clients/tui/tsup.config.ts | 7 + clients/web/src/App.tsx | 10 + .../SkillsScreen/SkillsScreen.test.tsx | 508 +++++++++++++++-- .../screens/SkillsScreen/SkillsScreen.tsx | 445 ++++++++++++++- .../views/InspectorView/InspectorView.tsx | 2 + .../components/views/InspectorView/types.ts | 12 +- .../web/src/hooks/useServerCommands.test.tsx | 70 +++ clients/web/src/hooks/useServerCommands.tsx | 40 +- .../core/mcp/inspectorClient-skills.test.ts | 160 ++++++ .../core/mcp/skillFile.test.ts} | 53 +- .../core/mcp}/skillFileBytes.test.ts | 2 +- clients/web/src/test/core/mcp/skills.test.ts | 135 +++++ .../src/test/core/mcp/skillsSchemas.test.ts | 124 ++++ .../test/core/mcp/skillsVerification.test.ts | 329 +++++++++++ .../mcp/inspectorClient-skills.test.ts | 144 ++++- clients/web/src/utils/skillFileBytes.ts | 35 -- clients/web/src/utils/splitSkillFile.ts | 55 -- clients/web/tsup.runner.config.ts | 7 + core/mcp/inspectorClient.ts | 76 +++ core/mcp/inspectorClientProtocol.ts | 14 + core/mcp/skillFile.ts | 112 ++++ core/mcp/skills.ts | 152 ++++- core/mcp/skillsSchemas.ts | 92 ++- core/mcp/skillsVerification.ts | 221 +++++++ core/mcp/state/managedSkillsState.ts | 27 + docs/test-servers.md | 62 +- test-servers/src/composable-test-server.ts | 23 +- test-servers/src/load-config.ts | 4 +- test-servers/src/skills.ts | 238 +++++++- 47 files changed, 4750 insertions(+), 191 deletions(-) create mode 100644 clients/cli/__tests__/run-method-skills.test.ts create mode 100644 clients/cli/__tests__/skills-verify-cli.test.ts create mode 100644 clients/cli/src/handlers/skills-verify.ts create mode 100644 clients/tui/__tests__/SkillsTab.test.tsx create mode 100644 clients/tui/src/components/SkillsTab.tsx rename clients/web/src/{utils/splitSkillFile.test.ts => test/core/mcp/skillFile.test.ts} (52%) rename clients/web/src/{utils => test/core/mcp}/skillFileBytes.test.ts (95%) create mode 100644 clients/web/src/test/core/mcp/skillsVerification.test.ts delete mode 100644 clients/web/src/utils/skillFileBytes.ts delete mode 100644 clients/web/src/utils/splitSkillFile.ts create mode 100644 core/mcp/skillFile.ts create mode 100644 core/mcp/skillsVerification.ts diff --git a/clients/cli/README.md b/clients/cli/README.md index ee2cc7611f..ec4ed88267 100644 --- a/clients/cli/README.md +++ b/clients/cli/README.md @@ -108,11 +108,12 @@ Options that specify the MCP server (catalog/config file, ad-hoc command/URL, en | Option | Description | | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--method ` | MCP method to invoke. Supports `initialize` (connect-only probe → `{serverInfo, protocolVersion, capabilities, instructions}`), `tools/list`, `tools/call`, `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list`, `prompts/get`, `logging/setLevel`, plus catalog-only `servers/list` / `servers/show` (no MCP connect). Stream / session-only methods (e.g. `logging/tail`) are rejected. | +| `--method ` | MCP method to invoke. Supports `initialize` (connect-only probe → `{serverInfo, protocolVersion, capabilities, instructions}`), `tools/list`, `tools/call`, `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list`, `prompts/get`, `logging/setLevel`, `skills/list`, `skills/get`, `resources/directory/read`, plus catalog-only `servers/list` / `servers/show` (no MCP connect). Stream / session-only methods (e.g. `logging/tail`) are rejected. | | `--tool-name ` | Tool name (for `tools/call`). | | `--tool-arg ` | Tool argument; repeat for multiple. Use `key='{"json":true}'` for JSON. Values are coerced (JSON-parsed, so `count=1` becomes a number). | | `--tool-args-json ` | Tool arguments as a single JSON object (e.g. `'{"zip":"10001"}'`). Passed verbatim — no `key=value` coercion, so `"012"` stays a string. Mutually exclusive with `--tool-arg`. | -| `--uri ` | Resource URI (for `resources/read`). | +| `--uri ` | Resource URI (`resources/read`), directory URI (`resources/directory/read`), or skill URI (`skills/get`). | +| `--cursor ` | Opaque pagination cursor for `resources/directory/read` — pass back the `nextCursor` from the previous page. The listing is not recursive and pages are not aggregated: SEP-2640 gives the cursor to the client, and descending is the caller's job. | | `--prompt-name ` | Prompt name (for `prompts/get`). | | `--prompt-args ` | Prompt arguments; repeat for multiple. | | `--log-level ` | Logging level for `logging/setLevel` (e.g. `debug`, `info`). | @@ -121,6 +122,7 @@ Options that specify the MCP server (catalog/config file, ad-hoc command/URL, en | `--connect-timeout ` | Connection timeout in ms. Defaults to `15000` for ad-hoc `--server-url`/target runs (so a black-holed host fails fast) and to the file-level timeout for `--catalog`/`--config` runs. `0` disables the timeout. | | `--app-info` | Probe a tool's MCP App UI metadata without invoking it. With `--method tools/call --tool-name `: prints one JSON line (`hasApp`, `resourceUri`, `csp`, `permissions`, `domain`, …) and exits `0` if the tool has an app or `2` (`no_app`) if not. With `--method tools/list`: emits NDJSON — one app-info line per tool over a single connection. | | `--strict` | With `--method tools/list`: report tool-schema portability problems in full (path, issue, suggested fix) on stderr, and exit `6` if any is error-severity. Without it, a one-line count is printed instead. See [Schema portability](#schema-portability---strict). | +| `--verify` | With `--method skills/list` or `--method skills/get`: run the SEP-2640 conformance, digest and frontmatter checks over the skills returned, emit one JSON report per skill on stdout, and exit `7` if any fails. See [Skill verification](#skill-verification---verify). | | `--format ` | Output format. `text` (default) pretty-prints the result. `json` emits a single JSON object on stdout (`{ "result": … }`, plus `{ "appInfo": … }` as a sibling key for App tools) with no banners, so the whole output pipes cleanly into `jq`. | | `--relogin` | Delete stored OAuth for this server URL from the shared store before connect; interactive login still only runs if the server requires auth. Requires an HTTP/SSE URL (rejected for stdio). Conflicts with `--stored-auth-only` / `--use-stored-auth` / `--wait-for-auth` / catalog short-circuits. | | `--no-revoke` | With `--relogin`, skip the [RFC 7009](https://datatracker.ietf.org/doc/html/rfc7009) revocation request that would otherwise end the grant at the authorization server when the local state is deleted. The per-server `oauth.revokeOnClear` setting is the persistent form of the same opt-out; either one is enough to skip it. See [Revoking on `--relogin`](#revoking-on---relogin). | @@ -333,6 +335,65 @@ mcp-inspector --cli --transport http --server-url https://api.example/mcp \ --wait-for-auth 120 --method tools/list ``` +#### Skill verification (`--verify`) + +SEP-2640 puts real obligations on whoever consumes a skill: verify each fetched +file against the digest its manifest advertised, check that the served +`SKILL.md`'s frontmatter matches the one the listing advertised, and honour the +per-skill limits. `--verify` runs all of them over a whole catalog and turns the +answer into an exit code, so a server author can gate CI on it: + +```sh +mcp-inspector --cli --method skills/list --verify +``` + +Stdout is **NDJSON, one report per skill**, in listing order: + +```json +{ + "uri": "skill://tampered-notes/SKILL.md", + "name": "tampered-notes", + "conformance": [], + "frontmatter": [], + "files": [ + { "uri": "skill://tampered-notes/SKILL.md", "status": "verified", "…": "…" }, + { "uri": "skill://tampered-notes/notes.md", "status": "mismatch", "…": "…" } + ], + "ok": false +} +``` + +Stderr gets a one-line summary, so a reader who piped stdout into `jq` still +sees the verdict. `--method skills/get --uri ` verifies exactly one +skill, in the same shape. + +**What fails the run.** `ok` is false — and the exit code is `7` — for anything +SEP-2640 makes a MUST: an error-severity conformance finding, a digest or size +mismatch, or a manifest file that could not be read. A **warning** does not fail +it. That distinction matters most for `resources: "dynamic"`, which is a +*conforming* wire form for generated content: it means integrity cannot be +verified, which is worth reporting, but failing CI for it would tell server +authors their valid skill is broken. + +**Three checks, three different jobs**, and the second is the one nothing else +covers: + +- **Conformance** — structural checks against the entry as listed (name grammar, + the name/URI invariant, digest and size formats, manifest completeness, the + interoperability limits). +- **Frontmatter** — the served `SKILL.md`'s own YAML frontmatter, compared field + by field against the frontmatter the listing advertised. A digest cannot cover + this: it is taken over the bytes the server served, so it proves the file was + not altered in transit and says nothing about whether the *listing* described + it honestly. A server can advertise one description, serve another, and pass + every digest check. +- **Files** — each manifest entry fetched and hashed. Reads are sequential: a + conforming manifest may declare 512 entries, and a parallel walk would open + 512 `resources/read` calls against the server under test. + +A read failure is recorded against the file it happened on and the walk +continues, so one unreadable file never hides the findings after it. + ## Exit codes & error envelopes Every non-zero exit maps to a stable failure class, so a programmatic caller @@ -348,6 +409,7 @@ prose from stderr: | `4` | Server unreachable (DNS, connection refused, timeout, `fetch failed`). | | `5` | Tool error (`tools/call` returned `isError:true`, or the tool was not found). | | `6` | `--strict` found an error-severity tool-schema portability problem (`schema_unportable` — the schema is valid JSON Schema, just not portable). | +| `7` | `--verify` found a SEP-2640 violation (`skills_nonconformant` — a conformance error, a digest or size mismatch, or an unreadable manifest file). | On any non-zero exit the CLI also writes a single JSON line to **stderr** — the `ErrorEnvelope`: diff --git a/clients/cli/__tests__/run-method-skills.test.ts b/clients/cli/__tests__/run-method-skills.test.ts new file mode 100644 index 0000000000..b496e30cb1 --- /dev/null +++ b/clients/cli/__tests__/run-method-skills.test.ts @@ -0,0 +1,222 @@ +import { describe, it, expect, vi } from "vitest"; +import { runMethod } from "../src/handlers/run-method.js"; +import { summarizeSkillVerification } from "../src/handlers/skills-verify.js"; +import { EXIT_CODES } from "../src/error-handler.js"; +import type { InspectorClient } from "@inspector/core/mcp/index.js"; +import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js"; +import { sha256Digest } from "@inspector/core/mcp/skills.js"; +import type { SkillVerifyReport } from "@inspector/core/mcp/skillsVerification.js"; + +/** + * The three SEP-2640 methods the CLI gained in #2248, plus `--verify`. + * + * The store's cursor walk and the verification checks are covered where they + * live (`managedSkillsState.test.ts`, `skillsVerification.test.ts`); what these + * pin is the dispatcher's own decisions — which method reaches which client + * call, what shape leaves as a result, and when the report sets a non-zero exit + * code. + */ +const SKILL_MD = "---\nname: demo\ndescription: A demo\n---\n\n# Demo\n"; + +async function cleanEntry(): Promise { + const bytes = new TextEncoder().encode(SKILL_MD); + return { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/SKILL.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; +} + +function mockClient(overrides: Record = {}): InspectorClient { + return { + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + getStatus: vi.fn().mockReturnValue("connected"), + getSkillsExtension: vi.fn().mockReturnValue({ directoryRead: true }), + listSkills: vi.fn().mockResolvedValue({ skills: [] }), + getSkill: vi.fn(), + readResourceDirectory: vi.fn(), + readResource: vi.fn().mockResolvedValue({ + result: { contents: [{ uri: "skill://demo/SKILL.md", text: SKILL_MD }] }, + }), + ...overrides, + } as unknown as InspectorClient; +} + +describe("runMethod skills dispatch (#2248)", () => { + it("returns the walked list for skills/list", async () => { + const entry = await cleanEntry(); + const client = mockClient({ + listSkills: vi.fn().mockResolvedValue({ skills: [entry] }), + }); + const outcome = await runMethod(client, { method: "skills/list" }); + expect(outcome).toEqual({ + kind: "result", + result: { skills: [entry] }, + appInfo: undefined, + }); + }); + + it("rejects skills/list with a usage exit code when the server declares no extension", async () => { + // The store answers "no extension" with an empty list, which is right for + // a UI that must render something and wrong for a CLI: "no skills" and + // "does not serve skills" are answers a script has to tell apart. + const client = mockClient({ + getSkillsExtension: vi.fn().mockReturnValue(undefined), + }); + await expect(runMethod(client, { method: "skills/list" })).rejects.toThrow( + /does not declare/i, + ); + await expect( + runMethod(client, { method: "skills/list" }), + ).rejects.toMatchObject({ exitCode: EXIT_CODES.USAGE }); + }); + + it("keeps the { skill } envelope on skills/get", async () => { + // The client unwraps it for callers that want the entry; a CLI whose + // contract is "print the result" must not quietly reshape the wire form. + const entry = await cleanEntry(); + const client = mockClient({ getSkill: vi.fn().mockResolvedValue(entry) }); + const outcome = await runMethod(client, { + method: "skills/get", + uri: entry.uri, + }); + expect(outcome).toMatchObject({ result: { skill: entry } }); + }); + + it("requires --uri for skills/get", async () => { + await expect( + runMethod(mockClient(), { method: "skills/get" }), + ).rejects.toThrow(/URI is required/); + }); + + it("requires --uri for resources/directory/read", async () => { + await expect( + runMethod(mockClient(), { method: "resources/directory/read" }), + ).rejects.toThrow(/URI is required/); + }); + + it("returns one page of resources/directory/read and forwards the cursor", async () => { + // One page, not a walk: the SEP says the listing is not recursive and the + // client descends, so aggregating here would present a subtree as a + // directory. + const page = { resources: [], nextCursor: "2" }; + const readResourceDirectory = vi.fn().mockResolvedValue(page); + const client = mockClient({ readResourceDirectory }); + const outcome = await runMethod(client, { + method: "resources/directory/read", + uri: "skill://demo", + cursor: "1", + }); + expect(readResourceDirectory).toHaveBeenCalledWith( + "skill://demo", + "1", + undefined, + ); + expect(outcome).toMatchObject({ result: page }); + }); + + it("--verify emits one NDJSON report per skill with no exit code when clean", async () => { + const entry = await cleanEntry(); + const client = mockClient({ + listSkills: vi.fn().mockResolvedValue({ skills: [entry] }), + }); + const outcome = await runMethod(client, { + method: "skills/list", + verify: true, + }); + expect(outcome.kind).toBe("ndjson"); + if (outcome.kind !== "ndjson") throw new Error("unreachable"); + expect(outcome.lines).toHaveLength(1); + expect((outcome.lines[0] as SkillVerifyReport).ok).toBe(true); + expect(outcome.summary).toMatch(/no conformance errors/); + expect(outcome.exitCode).toBeUndefined(); + }); + + it("--verify sets the skills exit code when a skill fails", async () => { + const entry = await cleanEntry(); + const client = mockClient({ + listSkills: vi.fn().mockResolvedValue({ skills: [entry] }), + readResource: vi.fn().mockResolvedValue({ + result: { + contents: [{ uri: entry.uri, text: "totally different bytes" }], + }, + }), + }); + const outcome = await runMethod(client, { + method: "skills/list", + verify: true, + }); + if (outcome.kind !== "ndjson") throw new Error("unreachable"); + expect(outcome.exitCode).toBe(EXIT_CODES.SKILL_NONCONFORMANT); + // Its own code, not SCHEMA_UNPORTABLE — an unportable tool schema and a + // tampered skill digest are different CI failures. + expect(EXIT_CODES.SKILL_NONCONFORMANT).not.toBe( + EXIT_CODES.SCHEMA_UNPORTABLE, + ); + }); + + it("--verify works on a single skills/get", async () => { + const entry = await cleanEntry(); + const client = mockClient({ getSkill: vi.fn().mockResolvedValue(entry) }); + const outcome = await runMethod(client, { + method: "skills/get", + uri: entry.uri, + verify: true, + }); + expect(outcome.kind).toBe("ndjson"); + if (outcome.kind !== "ndjson") throw new Error("unreachable"); + expect(outcome.lines).toHaveLength(1); + }); +}); + +describe("summarizeSkillVerification (#2248)", () => { + const report = ( + over: Partial = {}, + ): SkillVerifyReport => ({ + uri: "skill://demo/SKILL.md", + name: "demo", + conformance: [], + frontmatter: [], + files: [{ uri: "skill://demo/SKILL.md", status: "verified" }], + ok: true, + ...over, + }); + + it("reports a clean run with singular wording for one skill", () => { + expect(summarizeSkillVerification([report()])).toBe( + "Verified 1 skill and 1 file: no conformance errors.", + ); + }); + + it("pluralizes for more than one", () => { + expect(summarizeSkillVerification([report(), report()])).toBe( + "Verified 2 skills and 2 files: no conformance errors.", + ); + }); + + it("counts failures and digest mismatches separately", () => { + // A skill can fail on a conformance error with no mismatched file at all, + // so collapsing the two counts would misreport the cause. + const failed = report({ + ok: false, + files: [{ uri: "skill://demo/SKILL.md", status: "mismatch" }], + }); + expect(summarizeSkillVerification([report(), failed])).toBe( + "1 of 2 skills failed verification (1 digest/size mismatch across 2 files).", + ); + }); + + it("reports a failure with no mismatched file", () => { + const failed = report({ ok: false, files: [] }); + expect(summarizeSkillVerification([failed])).toBe( + "1 of 1 skill failed verification (0 digest/size mismatch across 0 files).", + ); + }); +}); diff --git a/clients/cli/__tests__/skills-verify-cli.test.ts b/clients/cli/__tests__/skills-verify-cli.test.ts new file mode 100644 index 0000000000..1a4b89d0e1 --- /dev/null +++ b/clients/cli/__tests__/skills-verify-cli.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect } from "vitest"; +import { runCli } from "../src/cli.js"; +import { consumeMethodOutcome } from "../src/handlers/consume-outcome.js"; +import { EXIT_CODES } from "../src/error-handler.js"; + +/** + * `--verify`'s argument validation and its NDJSON consumption path (#2248). + * + * The validation sits with `--strict`'s, ahead of every short-circuit return in + * `parseArgs`, for the same reason: the returns below it never reach + * `runMethod`, so a check placed further down would let the flag be accepted + * and then silently ignored. + */ +describe("--verify argument validation", () => { + it("is rejected with a method other than skills/list or skills/get", async () => { + await expect( + runCli([ + "node", + "cli", + "--cli", + "--method", + "tools/list", + "--verify", + "--server-url", + "http://127.0.0.1:1/mcp", + ]), + ).rejects.toThrow( + "--verify requires --method skills/list or --method skills/get.", + ); + }); + + it.each([ + ["servers/list", ["--method", "servers/list"]], + ["--list-stored-auth", ["--method", "servers/list", "--list-stored-auth"]], + ])( + "is rejected on the %s short-circuit path, which never reaches the report", + async (_label, extra) => { + await expect( + runCli(["node", "cli", "--cli", "--verify", ...extra]), + ).rejects.toThrow( + "--verify requires --method skills/list or --method skills/get.", + ); + }, + ); + + it("is accepted with skills/get", async () => { + // Reaches the connect and fails there — which is the point: the flag + // itself was not what was rejected. + await expect( + runCli([ + "node", + "cli", + "--cli", + "--method", + "skills/get", + "--uri", + "skill://demo/SKILL.md", + "--verify", + "--server-url", + "http://127.0.0.1:1/mcp", + ]), + ).rejects.not.toThrow(/--verify requires/); + }); +}); + +describe("consumeMethodOutcome NDJSON summary and exit code (#2248)", () => { + function captureStreams() { + let stdout = ""; + let stderr = ""; + const write = (sink: (s: string) => void) => + ((chunk: unknown, ...rest: unknown[]) => { + sink(typeof chunk === "string" ? chunk : String(chunk)); + const cb = rest.find((r) => typeof r === "function") as + | (() => void) + | undefined; + cb?.(); + return true; + }) as typeof process.stdout.write; + const originalOut = process.stdout.write; + const originalErr = process.stderr.write; + process.stdout.write = write((s) => (stdout += s)); + process.stderr.write = write((s) => (stderr += s)); + return { + get stdout() { + return stdout; + }, + get stderr() { + return stderr; + }, + restore() { + process.stdout.write = originalOut; + process.stderr.write = originalErr; + }, + }; + } + + it("writes the summary to stderr so it cannot contaminate the NDJSON", async () => { + const streams = captureStreams(); + try { + await consumeMethodOutcome( + { kind: "ndjson", lines: [{ ok: true }], summary: "all good" }, + {}, + ); + } finally { + streams.restore(); + } + expect(JSON.parse(streams.stdout.trim())).toEqual({ ok: true }); + expect(streams.stderr).toBe("all good\n"); + }); + + it("throws the exit code AFTER writing the report", async () => { + // The report is the output a CI job reads; failing before writing it would + // give the reader an exit code and nothing to act on. + const streams = captureStreams(); + let thrown: unknown; + try { + await consumeMethodOutcome( + { + kind: "ndjson", + lines: [{ ok: false }], + summary: "one failed", + exitCode: EXIT_CODES.SKILL_NONCONFORMANT, + }, + {}, + ); + } catch (err) { + thrown = err; + } finally { + streams.restore(); + } + expect(streams.stdout.trim()).toBe('{"ok":false}'); + expect(thrown).toMatchObject({ + exitCode: EXIT_CODES.SKILL_NONCONFORMANT, + envelope: { code: "skills_nonconformant" }, + }); + }); + + it("leaves an --app-info NDJSON outcome unchanged", async () => { + // No summary, no exit code — the field is additive and the older caller + // must behave exactly as before. + const streams = captureStreams(); + try { + await consumeMethodOutcome({ kind: "ndjson", lines: [{ a: 1 }] }, {}); + } finally { + streams.restore(); + } + expect(streams.stderr).toBe(""); + expect(streams.stdout.trim()).toBe('{"a":1}'); + }); +}); diff --git a/clients/cli/src/cli.ts b/clients/cli/src/cli.ts index 17aa5ba088..fe19ee84ad 100644 --- a/clients/cli/src/cli.ts +++ b/clients/cli/src/cli.ts @@ -679,7 +679,14 @@ async function parseArgs(argv?: string[]): Promise { parseKeyValuePair, {}, ) - .option("--uri ", "URI of the resource (for resources/read method)") + .option( + "--uri ", + "URI of the resource (resources/read, resources/directory/read) or of the skill (skills/get)", + ) + .option( + "--cursor ", + "Opaque pagination cursor (for resources/directory/read; pass back the nextCursor from the previous page).", + ) .option( "--prompt-name ", "Name of the prompt (for prompts/get method)", @@ -743,6 +750,10 @@ async function parseArgs(argv?: string[]): Promise { "--strict", "Report tool-schema portability problems in full (path, issue, suggested fix) on stderr, and exit 6 if any is error-severity. Use with --method tools/list. Without it, a one-line count is printed instead.", ) + .option( + "--verify", + "Run the SEP-2640 conformance and digest checks over the skills returned, emit one JSON report per skill on stdout, and exit 7 if any fails. Use with --method skills/list or --method skills/get.", + ) .option( "--connect-timeout ", `Connection timeout in ms (default ${DEFAULT_CONNECT_TIMEOUT_MS} for ad-hoc --server-url / target invocations; 0 = no timeout).`, @@ -848,6 +859,8 @@ async function parseArgs(argv?: string[]): Promise { header?: Record; appInfo?: boolean; strict?: boolean; + verify?: boolean; + cursor?: string; connectTimeout?: number; format?: OutputFormat; toolArgsJson?: string; @@ -919,6 +932,18 @@ async function parseArgs(argv?: string[]): Promise { } } + // `--verify` is checked here for exactly the reason `--strict` is: the + // short-circuit returns below never reach `runMethod`, so validating further + // down would let `--verify --method servers/list` succeed while silently + // ignoring a flag documented as skills-only. + if (options.verify) { + if (options.method !== "skills/list" && options.method !== "skills/get") { + throw new Error( + "--verify requires --method skills/list or --method skills/get.", + ); + } + } + // State-path precedence (getStateFilePath): MCP_INSPECTOR_OAUTH_STATE_PATH → // /oauth.json → ~/.mcp-inspector/storage/oauth.json — the // same file the web backend writes, so tokens are shared across surfaces. @@ -1147,6 +1172,8 @@ async function parseArgs(argv?: string[]): Promise { toolMeta: options.toolMetadata, appInfo: options.appInfo === true, strict: options.strict === true, + verify: options.verify === true, + cursor: options.cursor, format: options.format, }; diff --git a/clients/cli/src/error-handler.ts b/clients/cli/src/error-handler.ts index 0deaf527b6..7f8b432bd3 100644 --- a/clients/cli/src/error-handler.ts +++ b/clients/cli/src/error-handler.ts @@ -25,6 +25,16 @@ export const EXIT_CODES = { UNREACHABLE: 4, TOOL_ERROR: 5, SCHEMA_UNPORTABLE: 6, + /** + * `--verify` found a SEP-2640 violation: a conformance error, a digest or + * size mismatch, or a manifest file that could not be read (#2248). + * + * Its own code rather than reusing `SCHEMA_UNPORTABLE`, for the reason that + * one exists at all: a CI job that fails on an unportable tool schema and a + * CI job that fails on a tampered skill digest are different jobs, and + * collapsing them would make `if [ $? -eq 6 ]` ambiguous. + */ + SKILL_NONCONFORMANT: 7, } as const; /** Machine-readable error envelope written as one JSON line on stderr. */ diff --git a/clients/cli/src/handlers/consume-outcome.ts b/clients/cli/src/handlers/consume-outcome.ts index 3147098c49..6891c30afd 100644 --- a/clients/cli/src/handlers/consume-outcome.ts +++ b/clients/cli/src/handlers/consume-outcome.ts @@ -1,4 +1,5 @@ -import { awaitableLog } from "../utils/awaitable-log.js"; +import { awaitableError, awaitableLog } from "../utils/awaitable-log.js"; +import { CliExitCodeError } from "../error-handler.js"; import { emitResult } from "./emit-result.js"; import type { MethodArgs, MethodOutcome } from "./method-types.js"; @@ -21,6 +22,17 @@ export async function consumeMethodOutcome( for (const line of outcome.lines) { await awaitableLog(JSON.stringify(line) + "\n"); } + // Summary on **stderr**, after the report, so it cannot contaminate the + // NDJSON a consumer is parsing on stdout. + if (outcome.summary) await awaitableError(`${outcome.summary}\n`); + // Thrown rather than returned so it routes through the CLI's single exit + // path — the report has already been written, which is why this is the + // last thing that happens. + if (outcome.exitCode) { + throw new CliExitCodeError(outcome.exitCode, outcome.summary ?? "", { + code: "skills_nonconformant", + }); + } return; } diff --git a/clients/cli/src/handlers/method-types.ts b/clients/cli/src/handlers/method-types.ts index 51d8657c60..958552dcea 100644 --- a/clients/cli/src/handlers/method-types.ts +++ b/clients/cli/src/handlers/method-types.ts @@ -34,6 +34,18 @@ export type MethodArgs = { taskId?: string; /** When true, tools/call uses callToolStream (task-augmented). */ task?: boolean; + /** + * `--verify`: run the SEP-2640 conformance and digest checks over the skills + * a `skills/list` / `skills/get` returned, emit one NDJSON report per skill, + * and exit non-zero when any fails (#2248). + */ + verify?: boolean; + /** + * Opaque pagination cursor. Used by `resources/directory/read`, whose result + * pages exactly as `resources/list` does — and where the caller descends the + * tree itself, so there is no store to walk it. + */ + cursor?: string; /** roots/set payload (JSON array of {uri, name?}). */ rootsJson?: string; /** prompts/complete: argument name / value / ref. */ @@ -53,7 +65,18 @@ export type McpResponse = Record; export type MethodOutcome = | { kind: "result"; result: McpResponse; appInfo?: CliAppInfo } /** One JSON object per line (e.g. tools/list --app-info). Caller writes stdout. */ - | { kind: "ndjson"; lines: unknown[] } + | { + kind: "ndjson"; + lines: unknown[]; + /** + * A line for **stderr**, written after the NDJSON. `--verify` uses it for + * its one-line summary, so a reader who piped stdout into `jq` still sees + * the verdict; `--app-info` sets nothing and behaves as before. + */ + summary?: string; + /** Non-zero when the emitted report is itself a failure (`--verify`). */ + exitCode?: number; + } | { kind: "stream"; /** Human label for errors. */ @@ -76,6 +99,7 @@ export const SESSION_RPC_METHODS = [ "resources/list", "resources/read", "resources/templates/list", + "resources/directory/read", "resources/subscribe", "resources/unsubscribe", "prompts/list", @@ -89,6 +113,8 @@ export const SESSION_RPC_METHODS = [ "tasks/result", "roots/list", "roots/set", + "skills/list", + "skills/get", ] as const; export type SessionRpcMethod = (typeof SESSION_RPC_METHODS)[number]; @@ -109,6 +135,12 @@ export const ONE_SHOT_METHODS = [ "prompts/list", "prompts/get", "logging/setLevel", + // SEP-2640. All three are ordinary one-shot request/response calls — no + // stream, no long-lived subscription — so they belong here alongside the + // other list verbs rather than being reachable only from the session CLI. + "skills/list", + "skills/get", + "resources/directory/read", ] as const; export type OneShotMethod = (typeof ONE_SHOT_METHODS)[number]; diff --git a/clients/cli/src/handlers/run-method.ts b/clients/cli/src/handlers/run-method.ts index 68b490ca66..83a6bac717 100644 --- a/clients/cli/src/handlers/run-method.ts +++ b/clients/cli/src/handlers/run-method.ts @@ -6,10 +6,17 @@ import { ManagedResourceTemplatesState, ManagedPromptsState, ManagedRequestorTasksState, + ManagedSkillsState, MessageLogState, } from "@inspector/core/mcp/state/index.js"; +import { SKILLS_EXTENSION_KEY } from "@inspector/core/mcp/skillsSchemas.js"; import { CliExitCodeError, EXIT_CODES } from "../error-handler.js"; import { collectAppInfo } from "./collect-app-info.js"; +import { summarizeSkillVerification } from "./skills-verify.js"; +import { + allSkillsVerified, + verifySkills, +} from "@inspector/core/mcp/skillsVerification.js"; import type { CliAppInfo, McpResponse, @@ -35,6 +42,7 @@ export async function runMethod( null; let managedPromptsState: ManagedPromptsState | null = null; let managedTasksState: ManagedRequestorTasksState | null = null; + let managedSkillsState: ManagedSkillsState | null = null; try { let result: McpResponse; @@ -283,6 +291,81 @@ export async function runMethod( result = (await inspectorClient.getRequestorTaskResult( args.taskId, )) as McpResponse; + } else if (args.method === "skills/list") { + // The store's cursor walk is reused rather than re-implemented — it + // carries the repeated-cursor and page-cap guards, and a second copy of + // a pagination walk is how the two come to disagree. What the CLI adds + // is the check below: the store answers "no extension" with an empty + // list, which is right for a UI that must render *something*, and wrong + // for a CLI where "this server has no skills" and "this server does not + // serve skills at all" are different answers a script has to tell apart. + if (!inspectorClient.getSkillsExtension()) { + throw new CliExitCodeError( + EXIT_CODES.USAGE, + `Server does not declare the ${SKILLS_EXTENSION_KEY} extension, so ${args.method} is not available.`, + { code: "skills_unsupported" }, + ); + } + managedSkillsState = new ManagedSkillsState(inspectorClient); + const skills = await managedSkillsState.refresh(args.metadata); + if (args.verify) { + const reports = await verifySkills( + inspectorClient, + skills, + args.metadata, + ); + return { + kind: "ndjson", + lines: reports, + summary: summarizeSkillVerification(reports), + ...(allSkillsVerified(reports) + ? {} + : { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }), + }; + } + result = { skills }; + } else if (args.method === "skills/get") { + if (!args.uri) { + throw new Error( + "URI is required for skills/get method. Use --uri to specify the skill URI.", + ); + } + const skill = await inspectorClient.getSkill(args.uri, args.metadata); + if (args.verify) { + const reports = await verifySkills( + inspectorClient, + [skill], + args.metadata, + ); + return { + kind: "ndjson", + lines: reports, + summary: summarizeSkillVerification(reports), + ...(allSkillsVerified(reports) + ? {} + : { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }), + }; + } + // The `{ skill }` envelope is restored here because it is what the wire + // carries: `GetSkillResultSchema` unwraps it for callers that want the + // entry, and a CLI whose contract is "print the result" must not quietly + // reshape one. + result = { skill }; + } else if (args.method === "resources/directory/read") { + if (!args.uri) { + throw new Error( + "URI is required for resources/directory/read. Use --uri to specify the directory URI.", + ); + } + // One page, not a walk. SEP-2640 says the listing is not recursive and + // clients descend by calling again on a child, so aggregating pages here + // would present a subtree as a directory — and the cursor is exposed as + // `--cursor` precisely so a script can do the descending. + result = await inspectorClient.readResourceDirectory( + args.uri, + args.cursor, + args.metadata, + ); } else if (args.method === "roots/list") { result = { roots: inspectorClient.getRoots() }; } else if (args.method === "roots/set") { @@ -318,6 +401,7 @@ export async function runMethod( managedResourcesState?.destroy(); managedResourceTemplatesState?.destroy(); managedPromptsState?.destroy(); + managedSkillsState?.destroy(); managedTasksState?.destroy(); } } diff --git a/clients/cli/src/handlers/skills-verify.ts b/clients/cli/src/handlers/skills-verify.ts new file mode 100644 index 0000000000..9ef59cd842 --- /dev/null +++ b/clients/cli/src/handlers/skills-verify.ts @@ -0,0 +1,36 @@ +/** + * `--verify`: the scriptable SEP-2640 conformance report (#2248). + * + * The Skills screen in the web client can verify a skill, but only by hand, one + * file at a time, in a browser. A server author wants the same verdict in CI, + * over the whole catalog, with an exit code — which is exactly the argument + * `--strict` makes for the tool-schema lint, so this follows that handler's + * shape rather than inventing a second one. + * + * The walk itself is `core/mcp/skillsVerification.ts`, shared with the TUI's + * Skills pane. What is left here is presentation: the one-line stderr summary, + * which is a CLI concern and nothing else's. + */ + +import type { SkillVerifyReport } from "@inspector/core/mcp/skillsVerification.js"; + +/** + * A one-line human summary for stderr, so a reader who piped stdout to `jq` + * still learns the verdict. + */ +export function summarizeSkillVerification( + reports: readonly SkillVerifyReport[], +): string { + const failed = reports.filter((report) => !report.ok).length; + const files = reports.reduce((sum, report) => sum + report.files.length, 0); + const mismatched = reports.reduce( + (sum, report) => + sum + report.files.filter((file) => file.status === "mismatch").length, + 0, + ); + const skillWord = reports.length === 1 ? "skill" : "skills"; + const fileWord = files === 1 ? "file" : "files"; + return failed === 0 + ? `Verified ${reports.length} ${skillWord} and ${files} ${fileWord}: no conformance errors.` + : `${failed} of ${reports.length} ${skillWord} failed verification (${mismatched} digest/size mismatch across ${files} ${fileWord}).`; +} diff --git a/clients/cli/tsup.config.ts b/clients/cli/tsup.config.ts index 99ffd89866..724317cc5d 100644 --- a/clients/cli/tsup.config.ts +++ b/clients/cli/tsup.config.ts @@ -55,6 +55,13 @@ export default defineConfig({ "atomically", "open", "zod", + // Newly on `core/`'s runtime import graph as of #2248: + // `core/mcp/skillFile.ts` parses a served SKILL.md's YAML frontmatter to + // check it against the entry the listing advertised (SEP-2640). Already a + // root `dependency` — it was reached from `test-servers/src` — so this + // adds no package, but a root-declared dependency `core/` imports must be + // named in all three `external` lists or tsup inlines it here. + "yaml", // Reached through `core/` but not through this client's own code today. // AGENTS.md requires every root-declared package `core/` imports at runtime // in ALL three lists regardless, because which client reaches one is a diff --git a/clients/tui/README.md b/clients/tui/README.md index ee7da50588..91be95e2f4 100644 --- a/clients/tui/README.md +++ b/clients/tui/README.md @@ -76,13 +76,14 @@ The TUI provides terminal-native tabs and panes for interacting with your MCP se - **Resources**: Browse and read resources exposed by the server. - **Prompts**: List and test prompts. - **Tools**: View available tools and execute them with form-like inputs. A tool whose advertised schema carries a portability problem is flagged in the list — red `!` for a construct a shipping MCP client refuses, yellow `?` for one handled unevenly — and the detail pane lists each finding under **Schema Portability** with the path, the problem, and a concrete fix. The verdict comes from [`core/json/schemaLint.ts`](../../core/json/schemaLint.ts), shared with the web Tools tab and the CLI's `--strict` report, so the three cannot disagree ([#1005](https://github.com/modelcontextprotocol/inspector/issues/1005)). +- **Skills**: Shown only when the connected server declares the SEP-2640 Skills extension (`io.modelcontextprotocol/skills`), since it is a *server* declaration and so only knowable after connecting. The list marks each skill with its structural verdict — `✓` conforms, `!` warnings only, `✗` an error — using a glyph as well as a colour, because this pane is read over ssh, in tmux and through `script(1)`. The detail pane shows the entry's URI, description, conformance findings and manifest. **Enter** verifies the selected skill: one `resources/read` per manifest file, each hashed against its advertised digest, plus the frontmatter cross-check that compares the served `SKILL.md`'s own frontmatter against the one the listing advertised. Verification is a gesture rather than a page load because SEP-2640 says hosts MUST NOT retrieve a skill's files ahead of need. The checks are the same ones the web Skills tab and the CLI's `--verify` run ([#2234](https://github.com/modelcontextprotocol/inspector/issues/2234), [#2248](https://github.com/modelcontextprotocol/inspector/issues/2248)). - **Protocol**: View JSON-RPC request/response/notification history (matches the web Protocol monitor). - **Network**: View HTTP fetch traffic for SSE / Streamable HTTP servers (matches the web Network monitor). - **Console**: View stdio stderr from the connected server process (matches the web Console monitor). ## Navigation -- Use the **Arrow Keys** (Left/Right) or **Tab** to switch between the main tabs (Resources, Tools, Prompts, etc.). +- Use the **Arrow Keys** (Left/Right) or **Tab** to switch between the main tabs (Resources, Tools, Prompts, Skills, etc.). - Use the **Arrow Keys** (Up/Down) to scroll through lists of items. - Press **Enter** to select an item, execute a tool, or fetch a resource. - Press **Escape** or `Ctrl+C` to exit the application. diff --git a/clients/tui/__tests__/App.test.tsx b/clients/tui/__tests__/App.test.tsx index 1d8faf7ced..f47cfaac58 100644 --- a/clients/tui/__tests__/App.test.tsx +++ b/clients/tui/__tests__/App.test.tsx @@ -24,6 +24,8 @@ const h = vi.hoisted(() => { resources: unknown[]; resourceTemplates: unknown[]; prompts: unknown[]; + skills: unknown[]; + skillsExtension: { directoryRead: boolean } | undefined; messages: unknown[]; fetchRequests: unknown[]; stderrLogs: unknown[]; @@ -39,6 +41,8 @@ const h = vi.hoisted(() => { resources: [], resourceTemplates: [], prompts: [], + skills: [], + skillsExtension: undefined as { directoryRead: boolean } | undefined, messages: [], fetchRequests: [], stderrLogs: [], @@ -160,6 +164,10 @@ const h = vi.hoisted(() => { | "sse" | "streamable-http", ); + // The Skills tab is gated on a SERVER declaration, so the default here is + // "not declared" — the tab is hidden unless a test opts in by pointing + // `ctrl.skillsExtension` at a declaration. + getSkillsExtension = vi.fn(() => ctrl.skillsExtension); authenticate = (...a: Parameters) => clientSpies.authenticate(...a); clearOAuthTokens = ( @@ -232,6 +240,11 @@ const h = vi.hoisted(() => { resourceTemplates: ctrl.resourceTemplates, })), useManagedPrompts: vi.fn(() => ({ prompts: ctrl.prompts })), + useManagedSkills: vi.fn(() => ({ + skills: ctrl.skills, + pageCount: ctrl.skills.length > 0 ? 1 : 0, + error: null, + })), useMessageLog: vi.fn(() => ({ messages: ctrl.messages })), useFetchRequestLog: vi.fn(() => ({ fetchRequests: ctrl.fetchRequests })), useStderrLog: vi.fn(() => ({ stderrLogs: ctrl.stderrLogs })), @@ -246,6 +259,7 @@ vi.mock("@inspector/core/mcp/state/index.js", () => ({ ManagedResourcesState: h.FakeManager, ManagedResourceTemplatesState: h.FakeManager, ManagedPromptsState: h.FakeManager, + ManagedSkillsState: h.FakeManager, MessageLogState: h.FakeManager, FetchRequestLogState: h.FakeManager, StderrLogState: h.FakeManager, @@ -271,6 +285,9 @@ vi.mock("@inspector/core/react/useManagedResources.js", () => ({ vi.mock("@inspector/core/react/useManagedResourceTemplates.js", () => ({ useManagedResourceTemplates: h.useManagedResourceTemplates, })); +vi.mock("@inspector/core/react/useManagedSkills.js", () => ({ + useManagedSkills: h.useManagedSkills, +})); vi.mock("@inspector/core/react/useManagedPrompts.js", () => ({ useManagedPrompts: h.useManagedPrompts, })); @@ -664,6 +681,8 @@ beforeEach(() => { resources: [], resourceTemplates: [], prompts: [], + skills: [], + skillsExtension: undefined as { directoryRead: boolean } | undefined, messages: [], fetchRequests: [], stderrLogs: [], @@ -751,6 +770,42 @@ describe("App (foundation)", () => { expect(h.connect).toHaveBeenCalled(); }); + it("hides the Skills tab until the server declares the extension", async () => { + // A *server*-declared extension (SEP-2640), so unlike the transport-derived + // tabs it is only knowable after connecting — and showing it against a + // server that never declared it would send `skills/list` to a server that + // answers -32601 (#2248). + h.ctrl.status = "connected"; + const r = await mount(oneStdio()); + await expectFrame(r, "Tools"); + expect(r.lastFrame() ?? "").not.toContain("Skills"); + }); + + it("shows the Skills tab, with its count, once the extension is declared", async () => { + h.ctrl.status = "connected"; + h.ctrl.skillsExtension = { directoryRead: false }; + h.ctrl.skills = [ + { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "d" }, + resources: [], + }, + ]; + const r = await mount(oneStdio()); + await expectFrame(r, "Skills (1)"); + }); + + it("opens the Skills tab with its 'k' accelerator", async () => { + // `k`, not `s` — the accelerator has to appear in the label and stay + // unique; see `tabsConfig.ts`. + h.ctrl.status = "connected"; + h.ctrl.skillsExtension = { directoryRead: true }; + const r = await mount(oneStdio()); + await expectFrame(r, "Skills"); + r.stdin.write("k"); + await expectFrame(r, "Select a skill to view details"); + }); + it("disconnects with 'd' when connected", async () => { h.ctrl.status = "connected"; const { stdin } = await mount(oneStdio()); diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx new file mode 100644 index 0000000000..ce642e0e18 --- /dev/null +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -0,0 +1,537 @@ +import React from "react"; +import { describe, it, expect, vi } from "vitest"; +import { render } from "./helpers/renderTui"; +import type { InspectorClient } from "@inspector/core/mcp/index.js"; +import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js"; +import { AuthRecoveryRequiredError } from "@inspector/core/auth/challenge.js"; +import { sha256Digest, textToBytes } from "@inspector/core/mcp/skills.js"; + +// MUST mock ink-scroll-view: the real ScrollView renders a placeholder minimap +// in the non-TTY test env and never mounts its children. +vi.mock("ink-scroll-view", () => import("./helpers/inkScrollViewMock.js")); + +import { SkillsTab } from "../src/components/SkillsTab.js"; + +const tick = async () => { + for (let i = 0; i < 8; i++) + await new Promise((resolve) => setTimeout(resolve, 4)); +}; + +const ESC = String.fromCharCode(27); +const UP = `${ESC}[A`; +const DOWN = `${ESC}[B`; +const PAGE_UP = `${ESC}[5~`; +const PAGE_DOWN = `${ESC}[6~`; +const ENTER = "\r"; + +const SKILL_MD = "---\nname: clean\ndescription: A clean skill\n---\n\n# C\n"; +// sha256 of SKILL_MD, so the clean fixture actually verifies. +const CLEAN_DIGEST = + "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + +const clean: SkillEntry = { + uri: "skill://clean/SKILL.md", + frontmatter: { name: "clean", description: "A clean skill" }, + resources: [ + { uri: "skill://clean/SKILL.md", digest: CLEAN_DIGEST, size: 51 }, + ], +}; +// A `name-path-mismatch`: the one structural invariant SEP-2640 states +// outright, so this row must carry the error mark. +const broken: SkillEntry = { + uri: "skill://wrong-folder/SKILL.md", + frontmatter: { name: "right-name", description: "Mismatched" }, + resources: [ + { uri: "skill://wrong-folder/SKILL.md", digest: CLEAN_DIGEST, size: 1 }, + ], +}; +// Legal but unverifiable — a WARNING, which must read differently from an error. +const dynamic: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen", description: "Generated" }, + resources: "dynamic", +}; +const noSize: SkillEntry = { + uri: "skill://nosize/SKILL.md", + frontmatter: { name: "nosize", description: "No declared size" }, + resources: [{ uri: "skill://nosize/SKILL.md", digest: CLEAN_DIGEST }], +}; + +const skills = [clean, broken, dynamic, noSize]; + +function mockClient( + readResource: unknown = vi.fn().mockResolvedValue({ + result: { contents: [{ uri: "skill://clean/SKILL.md", text: SKILL_MD }] }, + }), +): InspectorClient { + return { readResource } as unknown as InspectorClient; +} + +describe("SkillsTab (#2248)", () => { + it("renders the empty state when there are no skills", () => { + const { lastFrame } = render( + , + ); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Skills (0)"); + expect(frame).toContain("No skills available"); + expect(frame).toContain("Select a skill to view details"); + }); + + it("shows the page count only when the walk took more than one page", () => { + const one = render( + , + ); + expect(one.lastFrame() ?? "").toContain("Skills (4)"); + expect(one.lastFrame() ?? "").not.toContain("pages"); + const many = render( + , + ); + expect(many.lastFrame() ?? "").toContain("3 pages"); + }); + + it("renders the list error in place of the list", () => { + const { lastFrame } = render( + , + ); + expect(lastFrame() ?? "").toContain("walk failed"); + }); + + it("marks each row with its static conformance verdict", () => { + // The mark is a glyph, not only a colour: this pane is read over ssh, in + // tmux and through `script(1)`, where colour may not survive. + const { lastFrame } = render( + , + ); + const frame = lastFrame() ?? ""; + expect(frame).toContain("✓ clean"); + // `skillDisplayName` prefers the declared name over the URI segment. + expect(frame).toContain("✗ right-name"); + expect(frame).toContain("! gen"); + }); + + it("shows the selected skill's URI, description, findings and manifest", () => { + const { lastFrame } = render( + , + ); + const frame = lastFrame() ?? ""; + expect(frame).toContain("skill://clean/SKILL.md"); + expect(frame).toContain("A clean skill"); + expect(frame).toContain("Conformance: conforms"); + expect(frame).toContain("Manifest (1)"); + expect(frame).toContain("SKILL.md"); + expect(frame).toContain("(51 B)"); + expect(frame).toContain("[Enter to verify digests and frontmatter]"); + }); + + it("renders a dynamic skill's manifest as unadvertised rather than empty", async () => { + const { lastFrame, stdin } = render( + , + ); + stdin.write(DOWN); + await tick(); + stdin.write(DOWN); + await tick(); + const frame = lastFrame() ?? ""; + expect(frame).toContain('"dynamic" — no files advertised'); + expect(frame).toContain("integrity cannot be verified"); + }); + + it("omits the size caption when the manifest declares none", async () => { + const { lastFrame, stdin } = render( + , + ); + for (let i = 0; i < 3; i++) { + stdin.write(DOWN); + await tick(); + } + const frame = lastFrame() ?? ""; + expect(frame).toContain("Manifest (1)"); + expect(frame).not.toContain(" B)"); + }); + + it("moves selection with the arrow keys and stops at both boundaries", async () => { + const { lastFrame, stdin } = render( + , + ); + stdin.write(UP); + await tick(); + expect(lastFrame() ?? "").toContain("▶ ✓ clean"); + stdin.write(DOWN); + await tick(); + expect(lastFrame() ?? "").toContain("▶ ✗ right-name"); + for (let i = 0; i < 5; i++) { + stdin.write(DOWN); + await tick(); + } + // `nosize` omits a required `size`, so its row carries the error mark too + // — the mark tracks the checks, not the position. + expect(lastFrame() ?? "").toContain("▶ ✗ nosize"); + // …and back up from the bottom, which is the other direction of the same + // guard: the top boundary above never exercises the move itself. + stdin.write(UP); + await tick(); + expect(lastFrame() ?? "").toContain("▶ ! gen"); + }); + + it("scrolls the details pane without moving the selection", async () => { + const scrollBy = vi.fn(); + const { stdin } = render( + , + ); + stdin.write(UP); + stdin.write(DOWN); + stdin.write(PAGE_UP); + stdin.write(PAGE_DOWN); + await tick(); + // Nothing to assert on the mock beyond not crashing and not moving the + // selection — the ScrollView handle is stubbed by the shared mock. + expect(scrollBy).not.toHaveBeenCalled(); + }); + + it("ignores input entirely when a modal is open", async () => { + const { lastFrame, stdin } = render( + , + ); + stdin.write(DOWN); + await tick(); + expect(lastFrame() ?? "").toContain("▶ ✓ clean"); + }); + + it("verifies the selected skill on Enter and reports the outcome", async () => { + const readResource = vi.fn().mockResolvedValue({ + result: { contents: [{ uri: "skill://clean/SKILL.md", text: SKILL_MD }] }, + }); + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(readResource).toHaveBeenCalled(); + const frame = lastFrame() ?? ""; + // The fixture's advertised digest is all zeroes, so this is a mismatch — + // which is the outcome worth showing loudly. + expect(frame).toContain("Verification FAILED"); + expect(frame).toContain("✗ SKILL.md"); + }); + + it("surfaces the frontmatter cross-check after verifying", async () => { + const lying: SkillEntry = { + ...clean, + frontmatter: { name: "clean", description: "Something else entirely" }, + }; + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Frontmatter cross-check:"); + expect(frame).toContain("Something else entirely"); + }); + + it("reports an ordinary read failure as a failed verdict, not a crash", async () => { + // `verifySkills` records a plain read failure per file rather than + // throwing, so the pane shows the verdict rather than the error banner. + const readResource = vi.fn().mockRejectedValue(new Error("network down")); + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain("Verification FAILED"); + expect(lastFrame() ?? "").toContain("network down"); + }); + + it("hands an auth-recovery error to the callback instead of rendering it", async () => { + // The one error `verifySkills` re-throws: the session's authorization + // expired, and this callback is how the TUI offers to fix it. Rendered as + // a message instead, the user would be told the file could not be read and + // given no way to recover. + const err = new AuthRecoveryRequiredError( + new URL("https://auth.example/authorize"), + { reason: "expired" } as never, + ); + const onAuthRecoveryRequired = vi.fn(); + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(onAuthRecoveryRequired).toHaveBeenCalledWith(err); + expect(lastFrame() ?? "").not.toContain("Verification FAILED"); + }); + + it("shows the read failure's own reason under the file it happened on", async () => { + // A client missing `readResource` entirely fails every read; the walk + // records the reason per file rather than aborting, so the diagnosis lands + // beside the file it belongs to. + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain("is not a function"); + }); + + it("does nothing on Enter with no connected client", async () => { + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain( + "[Enter to verify digests and frontmatter]", + ); + }); + + it("reports a verified skill and re-verifies on a second Enter", async () => { + // The digest is computed from the very bytes the fake read returns, so the + // pass is real rather than a constant that happens to match. + const digest = await sha256Digest(textToBytes(SKILL_MD)); + const verifiable: SkillEntry = { + ...clean, + resources: [ + { + uri: "skill://clean/SKILL.md", + digest, + size: textToBytes(SKILL_MD).byteLength, + }, + ], + }; + const readResource = vi.fn().mockResolvedValue({ + result: { contents: [{ uri: "skill://clean/SKILL.md", text: SKILL_MD }] }, + }); + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain("Verified — Enter to re-verify"); + expect(lastFrame() ?? "").toContain("✓ SKILL.md"); + + stdin.write(ENTER); + await tick(); + expect(readResource).toHaveBeenCalledTimes(2); + }); + + it("shows a verifying state and ignores Enter while one is in flight", async () => { + // The guard is what stops a held Enter from opening a second walk over the + // same manifest on top of the first. + let release: ((value: unknown) => void) | undefined; + const readResource = vi.fn( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain("[Verifying…]"); + stdin.write(ENTER); + await tick(); + expect(readResource).toHaveBeenCalledTimes(1); + release?.({ + result: { contents: [{ uri: "skill://clean/SKILL.md", text: SKILL_MD }] }, + }); + await tick(); + }); + + it("falls back to the whole URI when a manifest entry has no path separator", async () => { + const odd: SkillEntry = { + uri: "skill://odd/SKILL.md", + frontmatter: { name: "odd", description: "d" }, + resources: [{ uri: "urn:opaque", digest: CLEAN_DIGEST, size: 1 }], + }; + const { lastFrame } = render( + , + ); + expect(lastFrame() ?? "").toContain("urn:opaque"); + }); + + it("keys a row by its index when the entry carries no URI", () => { + // A URI-less entry is a `malformed-uri` finding this pane reports, so it + // must still render a addressable row rather than colliding React keys. + const nameless = { + uri: "", + frontmatter: { name: "nameless", description: "d" }, + resources: [], + } as SkillEntry; + const { lastFrame } = render( + , + ); + expect(lastFrame() ?? "").toContain("nameless"); + }); + + it("shows the details footer only when the details pane is focused", () => { + const unfocused = render( + , + ); + expect(unfocused.lastFrame() ?? "").not.toContain("Enter to verify\n"); + const focused = render( + , + ); + expect(focused.lastFrame() ?? "").toContain( + "↑/↓ to scroll, Enter to verify", + ); + }); +}); diff --git a/clients/tui/__tests__/Tabs.test.tsx b/clients/tui/__tests__/Tabs.test.tsx index e15df18399..8854d12660 100644 --- a/clients/tui/__tests__/Tabs.test.tsx +++ b/clients/tui/__tests__/Tabs.test.tsx @@ -50,6 +50,38 @@ describe("Tabs", () => { expect(lastFrame() ?? "").toContain("Network"); }); + it("hides the skills tab by default and shows it when showSkills is true", () => { + // A *server-declared* extension (SEP-2640), unlike the transport-derived + // gates above — it is only knowable after connecting, so the default has + // to be hidden. + const hidden = render( + , + ); + expect(hidden.lastFrame() ?? "").not.toContain("Skills"); + const shown = render( + , + ); + expect(shown.lastFrame() ?? "").toContain("Skills"); + }); + + it("renders a count on the skills tab", () => { + const { lastFrame } = render( + , + ); + expect(lastFrame() ?? "").toContain("Skills (4)"); + }); + it("marks the active tab with the ▶ marker", () => { const { lastFrame } = render( , diff --git a/clients/tui/src/App.tsx b/clients/tui/src/App.tsx index 5b7d35f168..cdffc67b8b 100644 --- a/clients/tui/src/App.tsx +++ b/clients/tui/src/App.tsx @@ -27,6 +27,7 @@ import { ManagedResourcesState, ManagedResourceTemplatesState, ManagedPromptsState, + ManagedSkillsState, MessageLogState, FetchRequestLogState, StderrLogState, @@ -40,6 +41,7 @@ import { useManagedTools } from "@inspector/core/react/useManagedTools.js"; import { useManagedResources } from "@inspector/core/react/useManagedResources.js"; import { useManagedResourceTemplates } from "@inspector/core/react/useManagedResourceTemplates.js"; import { useManagedPrompts } from "@inspector/core/react/useManagedPrompts.js"; +import { useManagedSkills } from "@inspector/core/react/useManagedSkills.js"; import { useMessageLog } from "@inspector/core/react/useMessageLog.js"; import { useFetchRequestLog } from "@inspector/core/react/useFetchRequestLog.js"; import { useStderrLog } from "@inspector/core/react/useStderrLog.js"; @@ -79,6 +81,7 @@ import { InfoTab } from "./components/InfoTab.js"; import { AuthTab } from "./components/AuthTab.js"; import { ResourcesTab } from "./components/ResourcesTab.js"; import { PromptsTab } from "./components/PromptsTab.js"; +import { SkillsTab } from "./components/SkillsTab.js"; import { ToolsTab } from "./components/ToolsTab.js"; import { NotificationsTab } from "./components/NotificationsTab.js"; import { HistoryTab } from "./components/HistoryTab.js"; @@ -153,6 +156,7 @@ function App({ info?: number; resources?: number; prompts?: number; + skills?: number; tools?: number; messages?: number; requests?: number; @@ -244,6 +248,9 @@ function App({ const [managedPromptsStates, setManagedPromptsStates] = useState< Record >({}); + const [managedSkillsStates, setManagedSkillsStates] = useState< + Record + >({}); const [messageLogStates, setMessageLogStates] = useState< Record >({}); @@ -293,6 +300,7 @@ function App({ ManagedResourceTemplatesState > = {}; const newManagedPromptsStates: Record = {}; + const newManagedSkillsStates: Record = {}; const newMessageLogStates: Record = {}; const newFetchRequestLogStates: Record = {}; const newStderrLogStates: Record = {}; @@ -367,6 +375,7 @@ function App({ newManagedResourceTemplatesStates[serverName] = new ManagedResourceTemplatesState(client); newManagedPromptsStates[serverName] = new ManagedPromptsState(client); + newManagedSkillsStates[serverName] = new ManagedSkillsState(client); newMessageLogStates[serverName] = new MessageLogState(client); newFetchRequestLogStates[serverName] = new FetchRequestLogState(client); newStderrLogStates[serverName] = new StderrLogState(client); @@ -387,6 +396,10 @@ function App({ ...prev, ...newManagedPromptsStates, })); + setManagedSkillsStates((prev) => ({ + ...prev, + ...newManagedSkillsStates, + })); setMessageLogStates((prev) => ({ ...prev, ...newMessageLogStates })); setFetchRequestLogStates((prev) => ({ ...prev, @@ -420,6 +433,9 @@ function App({ Object.values(managedPromptsStates).forEach((manager) => { manager.destroy(); }); + Object.values(managedSkillsStates).forEach((manager) => { + manager.destroy(); + }); Object.values(messageLogStates).forEach((manager) => { manager.destroy(); }); @@ -441,6 +457,7 @@ function App({ managedResourcesStates, managedResourceTemplatesStates, managedPromptsStates, + managedSkillsStates, messageLogStates, fetchRequestLogStates, stderrLogStates, @@ -586,10 +603,28 @@ function App({ selectedInspectorClient, selectedManagedResourceTemplatesState, ); + const selectedManagedSkillsState = useMemo( + () => + selectedServer && managedSkillsStates[selectedServer] + ? managedSkillsStates[selectedServer] + : null, + [selectedServer, managedSkillsStates], + ); const { prompts: managedPrompts } = useManagedPrompts( selectedInspectorClient, selectedManagedPromptsState, ); + const { + skills: managedSkills, + pageCount: managedSkillsPageCount, + error: managedSkillsError, + } = useManagedSkills(selectedInspectorClient, selectedManagedSkillsState); + // A *server-declared* extension, so it is only knowable after connecting — + // unlike the transport-derived `showLoggingTab` / `showRequestsTab` above. + const showSkillsTab = + !!selectedServer && + !!selectedInspectorClient?.getSkillsExtension() && + inspectorStatus === "connected"; // Connect — on 401 or mid-session auth recovery, run OAuth then retry. type TuiOAuthRunResult = @@ -1347,6 +1382,7 @@ function App({ setTabCounts({ resources: managedResources.length || 0, prompts: managedPrompts.length || 0, + skills: managedSkills.length || 0, tools: managedTools.length || 0, messages: inspectorMessages.length || 0, requests: inspectorFetchRequests.length || 0, @@ -1356,6 +1392,7 @@ function App({ selectedServer, managedResources, managedPrompts, + managedSkills, managedTools, inspectorMessages, inspectorFetchRequests, @@ -1430,6 +1467,7 @@ function App({ if (tab.id === "auth" && !showAuthTab) return false; if (tab.id === "logging" && !showLoggingTab) return false; if (tab.id === "requests" && !showRequestsTab) return false; + if (tab.id === "skills" && !showSkillsTab) return false; return true; }) .map((tab: { id: TabType; label: string; accelerator: string }) => [ @@ -1517,6 +1555,7 @@ function App({ "auth", "resources", "prompts", + "skills", "tools", "messages", "requests", @@ -1526,6 +1565,7 @@ function App({ if (t === "auth" && !showAuthTab) return false; if (t === "logging" && !showLoggingTab) return false; if (t === "requests" && !showRequestsTab) return false; + if (t === "skills" && !showSkillsTab) return false; return true; }); const currentIndex = tabs.indexOf(activeTab); @@ -1763,6 +1803,7 @@ function App({ ? inspectorClients[selectedServer].getServerType() === "stdio" : false } + showSkills={showSkillsTab} showRequests={ selectedServer && inspectorClients[selectedServer] ? (() => { @@ -1967,6 +2008,34 @@ function App({ ) } /> + ) : activeTab === "skills" && + currentServerState?.status === "connected" && + selectedInspectorClient ? ( + ) : activeTab === "prompts" && currentServerState?.status === "connected" && selectedInspectorClient ? ( diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx new file mode 100644 index 0000000000..4fbc4a1e29 --- /dev/null +++ b/clients/tui/src/components/SkillsTab.tsx @@ -0,0 +1,416 @@ +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { Box, Text, useInput, type Key } from "ink"; +import { ScrollView, type ScrollViewRef } from "ink-scroll-view"; +import type { InspectorClient } from "@inspector/core/mcp/index.js"; +import { AuthRecoveryRequiredError } from "@inspector/core/auth/challenge.js"; +import { + checkSkillConformance, + skillDisplayName, + type SkillIssue, +} from "@inspector/core/mcp/skills.js"; +import { + DYNAMIC_RESOURCES, + type SkillEntry, +} from "@inspector/core/mcp/skillsSchemas.js"; +import { + verifySkills, + type SkillVerifyReport, +} from "@inspector/core/mcp/skillsVerification.js"; +import { useSelectableList } from "../hooks/useSelectableList.js"; + +interface SkillsTabProps { + skills: SkillEntry[]; + /** Pages the last `skills/list` walk took; shown so pagination is visible. */ + pageCount: number; + /** A failed list walk, rendered in place of the list. */ + loadError?: Error | null; + inspectorClient: InspectorClient | null; + width: number; + height: number; + focusedPane?: "list" | "details" | null; + onAuthRecoveryRequired?: (error: AuthRecoveryRequiredError) => void; + modalOpen?: boolean; +} + +/** + * The character that leads a finding line, by severity. A terminal pane cannot + * lean on colour alone — the Inspector is run over ssh, in tmux, and piped + * through `script(1)` — so severity is carried by a glyph as well as a colour. + */ +const ISSUE_MARK: Record = { + error: "✗", + warning: "!", +}; + +const ISSUE_COLOR: Record = { + error: "red", + warning: "yellow", +}; + +/** Per-file verification glyph, same reasoning as {@link ISSUE_MARK}. */ +const FILE_MARK: Record = { + verified: "✓", + mismatch: "✗", + unverifiable: "?", + error: "✗", + "read-error": "✗", +}; + +const FILE_COLOR: Record = { + verified: "green", + mismatch: "red", + unverifiable: "yellow", + error: "red", + "read-error": "red", +}; + +/** The file name a manifest URI ends in, for a list that must fit 40 columns. */ +function fileNameOf(uri: string): string { + const cut = uri.lastIndexOf("/"); + return cut === -1 ? uri : uri.slice(cut + 1); +} + +/** + * The Skills pane (SEP-2640, #2248): the catalog on the left, and on the right + * the selected skill's frontmatter, its conformance findings, and its manifest. + * + * **Enter verifies.** The static checks run on every render — they are a pure + * walk over a list already in memory — but digest verification needs the bytes, + * so it is one `resources/read` per manifest entry and must be asked for. That + * split is the same one the web screen makes and the same one SEP-2640 makes: + * hosts MUST NOT retrieve a skill's files ahead of need. + */ +export function SkillsTab({ + skills, + pageCount, + loadError = null, + inspectorClient, + width, + height, + focusedPane = null, + onAuthRecoveryRequired, + modalOpen = false, +}: SkillsTabProps) { + const visibleCount = Math.max(1, height - 7); + const { selectedIndex, firstVisible, setSelection } = useSelectableList( + skills.length, + visibleCount, + { resetWhen: skills }, + ); + const [error, setError] = useState(null); + const [verifying, setVerifying] = useState(false); + /** + * The last verification, keyed by the skill URI it was run for. Keyed rather + * than cleared on selection change so moving off a skill and back does not + * silently discard a verdict the user just paid a round trip for — and keyed + * by URI rather than index so a refresh that reorders the list cannot show + * one skill's verdict under another's name. + */ + const [report, setReport] = useState<{ + uri: string; + result: SkillVerifyReport; + } | null>(null); + const scrollViewRef = useRef(null); + + const selectedSkill = skills[selectedIndex] ?? null; + + const runVerify = useCallback( + (skill: SkillEntry) => { + if (!inspectorClient || verifying) return; + setVerifying(true); + setError(null); + // The IIFE catches everything it can throw, so there is no rejection for + // this key handler — which cannot await — to own. + void (async () => { + try { + const [result] = await verifySkills(inspectorClient, [skill]); + setReport({ uri: skill.uri, result }); + } catch (err) { + if (err instanceof AuthRecoveryRequiredError) { + onAuthRecoveryRequired?.(err); + return; + } + /* v8 ignore start -- `verifySkills` records an ordinary read failure + against the file it happened on and keeps walking, and it re-throws + exactly one error, handled directly above. So nothing the call + graph can produce reaches here; this is the guard that keeps a + future change from becoming an unhandled rejection instead of a + visible message. Exercising it would mean faking a throw the walk + cannot make, which tests the fake rather than the code. */ + setError( + err instanceof Error ? err.message : "Failed to verify skill", + ); + /* v8 ignore stop */ + } finally { + setVerifying(false); + } + })(); + }, + [inspectorClient, onAuthRecoveryRequired, verifying], + ); + + useInput( + (input: string, key: Key) => { + if (key.return && selectedSkill && inspectorClient) { + runVerify(selectedSkill); + return; + } + if (focusedPane === "list") { + if (key.upArrow && selectedIndex > 0) { + setSelection(selectedIndex - 1); + } else if (key.downArrow && selectedIndex < skills.length - 1) { + setSelection(selectedIndex + 1); + } + return; + } + if (focusedPane === "details") { + if (key.upArrow) { + scrollViewRef.current?.scrollBy(-1); + } else if (key.downArrow) { + scrollViewRef.current?.scrollBy(1); + } else if (key.pageUp) { + const viewportHeight = + scrollViewRef.current?.getViewportHeight() || 1; + scrollViewRef.current?.scrollBy(-viewportHeight); + } else if (key.pageDown) { + const viewportHeight = + scrollViewRef.current?.getViewportHeight() || 1; + scrollViewRef.current?.scrollBy(viewportHeight); + } + } + }, + { + isActive: + !modalOpen && (focusedPane === "list" || focusedPane === "details"), + }, + ); + + // Reset scroll when selection changes. A genuine synchronization with an + // external system (the ScrollView's imperative handle), not state derived + // from a prop — so an effect is the right tool here. + useEffect(() => { + scrollViewRef.current?.scrollTo(0); + }, [selectedIndex]); + + const listWidth = Math.floor(width * 0.4); + const detailWidth = width - listWidth; + const issues = selectedSkill ? checkSkillConformance(selectedSkill) : []; + const activeReport = + selectedSkill && report?.uri === selectedSkill.uri ? report.result : null; + const manifest = + selectedSkill && selectedSkill.resources !== DYNAMIC_RESOURCES + ? selectedSkill.resources + : []; + + return ( + + + + + Skills ({skills.length} + {pageCount > 1 ? `, ${pageCount} pages` : ""}) + + + {loadError ? ( + + {loadError.message} + + ) : skills.length === 0 ? ( + + No skills available + + ) : ( + + {skills + .slice(firstVisible, firstVisible + visibleCount) + .map((skill, i) => { + const index = firstVisible + i; + const isSelected = index === selectedIndex; + // The per-row mark is the static conformance verdict, which + // costs nothing — it is what makes a bad skill visible in the + // list rather than only after selecting it. + const rowIssues = checkSkillConformance(skill); + const worst = rowIssues.some((it) => it.severity === "error") + ? "error" + : rowIssues.length > 0 + ? "warning" + : null; + return ( + + + {isSelected ? "▶ " : " "} + {worst ? ( + + {ISSUE_MARK[worst]}{" "} + + ) : ( + ✓ + )} + {skillDisplayName(skill)} + + + ); + })} + + )} + + + + {selectedSkill ? ( + <> + + + {skillDisplayName(selectedSkill)} + + + + + + {selectedSkill.uri} + + {selectedSkill.frontmatter.description && ( + + {selectedSkill.frontmatter.description} + + )} + + + + Conformance{issues.length === 0 ? ": conforms" : ":"} + + + {issues.map((issue, idx) => ( + + + {ISSUE_MARK[issue.severity]} {issue.message} + + + ))} + + + + Manifest + {selectedSkill.resources === DYNAMIC_RESOURCES + ? ': "dynamic" — no files advertised' + : ` (${manifest.length})`} + + + {manifest.map((resource, idx) => { + const fileReport = activeReport?.files.find( + (file) => file.uri === resource.uri, + ); + return ( + + + {fileReport ? ( + + {FILE_MARK[fileReport.status] ?? "?"}{" "} + + ) : ( + · + )} + {fileNameOf(resource.uri)} + {resource.size !== undefined ? ( + ({resource.size} B) + ) : null} + + {fileReport?.reason && ( + + {fileReport.reason} + + )} + + ); + })} + + {activeReport && activeReport.frontmatter.length > 0 && ( + <> + + Frontmatter cross-check: + + {activeReport.frontmatter.map((issue, idx) => ( + + + {ISSUE_MARK[issue.severity]} {issue.message} + + + ))} + + )} + + {error && ( + + {error} + + )} + + + + {verifying + ? "[Verifying…]" + : activeReport + ? activeReport.ok + ? "[Verified — Enter to re-verify]" + : "[Verification FAILED — Enter to re-verify]" + : "[Enter to verify digests and frontmatter]"} + + + + + {focusedPane === "details" && ( + + + ↑/↓ to scroll, Enter to verify + + + )} + + ) : ( + + Select a skill to view details + + )} + + + ); +} diff --git a/clients/tui/src/components/Tabs.tsx b/clients/tui/src/components/Tabs.tsx index e61045dfc4..71e447dc0c 100644 --- a/clients/tui/src/components/Tabs.tsx +++ b/clients/tui/src/components/Tabs.tsx @@ -30,6 +30,7 @@ interface TabsProps { auth?: number; resources?: number; prompts?: number; + skills?: number; tools?: number; messages?: number; requests?: number; @@ -39,6 +40,13 @@ interface TabsProps { showAuth?: boolean; showLogging?: boolean; showRequests?: boolean; + /** + * The Skills tab is shown only when the connected server declared the + * SEP-2640 Skills extension — unlike Auth/Logging/Requests, which key off the + * transport, this one keys off a *server* declaration, so it can only be + * known after connecting. + */ + showSkills?: boolean; } export function Tabs({ @@ -49,6 +57,7 @@ export function Tabs({ showAuth = true, showLogging = true, showRequests = false, + showSkills = false, }: TabsProps) { let visibleTabs = tabs; if (!showAuth) { @@ -60,6 +69,9 @@ export function Tabs({ if (!showRequests) { visibleTabs = visibleTabs.filter((tab) => tab.id !== "requests"); } + if (!showSkills) { + visibleTabs = visibleTabs.filter((tab) => tab.id !== "skills"); + } return ( { if (uri === "skill://data-analysis/reference.md") return { text: REF_TEXT }; if (uri === "skill://tampered/notes.md") return { text: NOTES_TEXT }; - return { text: SELF_TEXT, mimeType: "text/markdown" }; + const owner = ALL_SKILLS.find((skill) => skill.uri === uri); + return { + text: owner ? skillMdFor(owner.frontmatter as Frontmatter) : SELF_TEXT, + mimeType: "text/markdown", + }; }); const baseProps: SkillsScreenProps = { @@ -576,15 +611,11 @@ describe("SkillsScreen", () => { { ...CLEAN_SKILL, resources: [ - { - uri: "skill://data-analysis/SKILL.md", - digest: SELF_DIGEST, - size: textToBytes(SELF_TEXT).byteLength, - }, + await selfEntry("skill://data-analysis/SKILL.md", CLEAN_FM), { uri: "skill://data-analysis/SKILL.md", digest: `sha256:${"d".repeat(64)}`, - size: textToBytes(SELF_TEXT).byteLength, + size: textToBytes(skillMdFor(CLEAN_FM)).byteLength, }, ], }, @@ -1717,3 +1748,416 @@ describe("SkillsScreen", () => { expect(screen.queryByText("too late")).not.toBeInTheDocument(); }); }); + +/** + * The Directory section (`resources/directory/read`, SEP-2640, #2248). + * + * Gated on the CALLBACK's presence, not on a boolean beside it: the SEP makes + * calling the method against a server that has not declared `directoryRead` a + * MUST NOT, and an absent callback is that rule expressed in the type. + */ +describe("SkillsScreen directory browsing (#2248)", () => { + const ROOT = "skill://data-analysis"; + const CHILD_FILE = { + uri: "skill://data-analysis/reference.md", + name: "reference.md", + mimeType: "text/markdown", + }; + const CHILD_DIR = { + uri: "skill://data-analysis/templates", + name: "templates", + mimeType: "inode/directory", + }; + const NESTED = { + uri: "skill://data-analysis/templates/invoice.md", + name: "invoice.md", + mimeType: "text/markdown", + }; + + function directoryReader( + pages: Record, + ) { + return vi.fn(async (uri: string, cursor?: string) => { + const page = pages[cursor === undefined ? uri : `${uri}#${cursor}`]; + if (!page) throw new Error(`no page for ${uri} ${cursor ?? ""}`); + return page as never; + }); + } + + it("renders no Directory section when the server did not declare directoryRead", async () => { + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + expect( + screen.queryByRole("button", { name: /Directory/ }), + ).not.toBeInTheDocument(); + }); + + it("reads on a click, never on selection", async () => { + // Every round trip on this screen is asked for — the same posture "Fetch + // entry" takes. + const user = userEvent.setup(); + const onReadResourceDirectory = directoryReader({ + [ROOT]: { resources: [CHILD_FILE] }, + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + expect(onReadResourceDirectory).not.toHaveBeenCalled(); + + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), + ); + expect(onReadResourceDirectory).toHaveBeenCalledWith(ROOT, undefined); + expect( + within(screen.getByTestId("skill-directory")).getByText( + "skill://data-analysis/reference.md", + ), + ).toBeInTheDocument(); + }); + + /** + * Open the Directory section on the clean skill and read its root. + * + * Shared because the descend/ascend assertions below would otherwise each + * repeat four sequential `userEvent` clicks inside one 5s budget — enough to + * make them the first thing to time out when the suite runs under load, + * which is a property of the test rather than of the screen. + */ + async function openRoot( + user: ReturnType, + reader: ReturnType, + ) { + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), + ); + } + + it("descends into a child directory", async () => { + const user = userEvent.setup(); + await openRoot( + user, + directoryReader({ + [ROOT]: { resources: [CHILD_FILE, CHILD_DIR] }, + [CHILD_DIR.uri]: { resources: [NESTED] }, + }), + ); + // A directory child is labelled as one and descends rather than opening in + // the viewer; the listing is not recursive, so this is a second call. + await user.click( + screen.getByRole("button", { name: `Open directory ${CHILD_DIR.uri}` }), + ); + await waitFor(() => + expect( + within(screen.getByTestId("skill-directory")).getByText(NESTED.uri), + ).toBeInTheDocument(), + ); + }); + + it("offers Up only below the skill root, and returns to it", async () => { + // Ascent is bounded by the root: this section browses the selected skill's + // tree, and walking above it would leave every other section's subject + // behind. + const user = userEvent.setup(); + await openRoot( + user, + directoryReader({ + [ROOT]: { resources: [CHILD_FILE, CHILD_DIR] }, + [CHILD_DIR.uri]: { resources: [NESTED] }, + }), + ); + expect( + screen.queryByRole("button", { name: "Up" }), + ).not.toBeInTheDocument(); + await user.click( + screen.getByRole("button", { name: `Open directory ${CHILD_DIR.uri}` }), + ); + await waitFor(() => + expect(screen.getByRole("button", { name: "Up" })).toBeInTheDocument(), + ); + await user.click(screen.getByRole("button", { name: "Up" })); + await waitFor(() => + expect( + within(screen.getByTestId("skill-directory")).getByText(CHILD_FILE.uri), + ).toBeInTheDocument(), + ); + expect( + screen.queryByRole("button", { name: "Up" }), + ).not.toBeInTheDocument(); + }); + + it("opens a file child in the viewer rather than descending", async () => { + const user = userEvent.setup(); + const onReadResourceDirectory = directoryReader({ + [ROOT]: { resources: [CHILD_FILE] }, + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), + ); + readFixtureFile.mockClear(); + await user.click( + screen.getByRole("button", { name: `View ${CHILD_FILE.uri}` }), + ); + await waitFor(() => + expect(readFixtureFile).toHaveBeenCalledWith(CHILD_FILE.uri), + ); + }); + + it("pages manually, accumulating children rather than replacing them", async () => { + // The cursor belongs to the client per the SEP, and this screen is what a + // server author uses to see their own pagination work — auto-walking it + // would hide the behaviour under test. + const user = userEvent.setup(); + const onReadResourceDirectory = directoryReader({ + [ROOT]: { resources: [CHILD_FILE], nextCursor: "1" }, + [`${ROOT}#1`]: { resources: [CHILD_DIR] }, + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Load more" }), + ).toBeInTheDocument(), + ); + await user.click(screen.getByRole("button", { name: "Load more" })); + await waitFor(() => { + const table = within(screen.getByTestId("skill-directory")); + expect(table.getByText(CHILD_FILE.uri)).toBeInTheDocument(); + expect(table.getByText(CHILD_DIR.uri)).toBeInTheDocument(); + }); + expect( + screen.queryByRole("button", { name: "Load more" }), + ).not.toBeInTheDocument(); + }); + + it("marks a child the manifest declares as listed", async () => { + const user = userEvent.setup(); + await openRoot( + user, + directoryReader({ [ROOT]: { resources: [CHILD_FILE] } }), + ); + const table = within(screen.getByTestId("skill-directory")); + expect(table.getByText("listed")).toBeInTheDocument(); + expect( + screen.queryByTestId("skill-directory-unlisted"), + ).not.toBeInTheDocument(); + }); + + it("flags a child the entry does not declare, without merging the two views", async () => { + // SEP-2640: a directory read is "a live observation" and hosts "MUST NOT + // treat the directory result as extending the manifest". The Inspector is + // not a host and does not refuse the read — what it must not do is present + // the child as one of the skill's files without saying where it came from. + const user = userEvent.setup(); + const STRAY = { + uri: "skill://data-analysis/added-later.md", + name: "added-later.md", + mimeType: "text/markdown", + }; + await openRoot( + user, + directoryReader({ [ROOT]: { resources: [CHILD_FILE, STRAY] } }), + ); + const table = within(screen.getByTestId("skill-directory")); + expect(table.getByText("listed")).toBeInTheDocument(); + expect(table.getByText("not listed")).toBeInTheDocument(); + const banner = screen.getByTestId("skill-directory-unlisted"); + expect(banner).toHaveTextContent(/1 file here that the held/); + // The recovery path the SEP names, rather than "read error". + expect(banner).toHaveTextContent(/skills\/get/); + }); + + it("gives a subdirectory no listed/unlisted verdict", async () => { + // A manifest lists files, so a directory is not a missing entry — a "not + // listed" chip on one would report a defect that is not there. + const user = userEvent.setup(); + await openRoot( + user, + directoryReader({ [ROOT]: { resources: [CHILD_DIR] } }), + ); + expect( + within(screen.getByTestId("skill-directory")).queryByText("not listed"), + ).not.toBeInTheDocument(); + expect( + screen.queryByTestId("skill-directory-unlisted"), + ).not.toBeInTheDocument(); + }); + + it("gives a dynamic skill's children no verdict either", async () => { + // `"dynamic"` advertises no manifest, so there is nothing for a child to be + // missing from — and a directory read is the only way its files are + // discoverable at all, which is the case the method exists for. + const user = userEvent.setup(); + const reader = directoryReader({ + "skill://dynamic-report": { + resources: [ + { + uri: "skill://dynamic-report/generated.md", + name: "generated.md", + mimeType: "text/markdown", + }, + ], + }, + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("dynamic-report")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), + ); + expect( + within(screen.getByTestId("skill-directory")).queryByText("not listed"), + ).not.toBeInTheDocument(); + expect( + screen.queryByTestId("skill-directory-unlisted"), + ).not.toBeInTheDocument(); + }); + + it("says an empty directory is empty", async () => { + const user = userEvent.setup(); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect(screen.getByText("This directory is empty.")).toBeInTheDocument(), + ); + }); + + it("renders a read failure without losing the section", async () => { + const user = userEvent.setup(); + const onReadResourceDirectory = vi.fn(async () => { + throw new Error("-32602 Not a directory resource"); + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await waitFor(() => + expect(screen.getByText(/Not a directory resource/)).toBeInTheDocument(), + ); + expect( + screen.getByRole("button", { name: /Directory/ }), + ).toBeInTheDocument(); + }); + + it("drops a listing when the selection changes mid-read", async () => { + // A read still in flight when the user switches skills must not land + // afterwards and paint one skill's tree under another's name. + const user = userEvent.setup(); + let release: ((value: unknown) => void) | undefined; + const onReadResourceDirectory = vi.fn( + () => + new Promise((resolve) => { + release = resolve; + }) as never, + ); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: "Read directory" })); + await user.click(screen.getByText("right-name")); + release?.({ resources: [CHILD_FILE] }); + await waitFor(() => + expect(screen.queryByTestId("skill-directory")).not.toBeInTheDocument(), + ); + }); + + it("renders no Directory section for a skill whose URI is malformed", async () => { + // There is no root to browse, and `malformed-uri` already reports it in + // Conformance. + const user = userEvent.setup(); + const odd: SkillEntry = { + uri: "not-a-uri", + frontmatter: { name: "odd", description: "d" }, + resources: [], + }; + renderWithMantine( + , + ); + await user.click(screen.getByText("odd")); + expect( + screen.queryByRole("button", { name: /Directory/ }), + ).not.toBeInTheDocument(); + }); +}); + +describe("SkillsScreen frontmatter cross-check (#2248)", () => { + it("reports a listing whose frontmatter disagrees with the served SKILL.md", async () => { + // The violation no digest can catch — the digest is over the bytes served + // and says nothing about whether the listing described them honestly. + const user = userEvent.setup(); + const lying: SkillEntry = { + ...CLEAN_SKILL, + frontmatter: { + name: "data-analysis", + description: "Not what the file says", + }, + }; + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Conformance/ })); + await waitFor(() => + expect( + screen.getByTestId("skill-frontmatter-issues"), + ).toBeInTheDocument(), + ); + // Scoped to the findings block: the skill's own description is rendered in + // the header too, so an unscoped match would pass on the wrong element. + expect( + within(screen.getByTestId("skill-frontmatter-issues")).getByText( + /Not what the file says/, + ), + ).toBeInTheDocument(); + }); + + it("reports nothing when the served frontmatter agrees", async () => { + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Conformance/ })); + await waitFor(() => + expect(screen.getByText("No structural issues")).toBeInTheDocument(), + ); + expect( + screen.queryByTestId("skill-frontmatter-issues"), + ).not.toBeInTheDocument(); + }); +}); diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 93a9ae7a4a..0a34fea6c9 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -19,17 +19,25 @@ import { import { MdSearch, MdVerifiedUser } from "react-icons/md"; import { RiArrowRightSLine } from "react-icons/ri"; import type { + DirectoryReadResult, SkillEntry, SkillResource, } from "@inspector/core/mcp/skillsSchemas.js"; -import { DYNAMIC_RESOURCES } from "@inspector/core/mcp/skillsSchemas.js"; +import { + DIRECTORY_MIME_TYPE, + DYNAMIC_RESOURCES, +} from "@inspector/core/mcp/skillsSchemas.js"; import { checkSkillConformance, + checkSkillFrontmatterMatch, skillDisplayName, + skillFileBytes, skillEntriesMatch, skillUriIdentity, + SKILL_FILE_SUFFIX, totalSkillBytes, verifySkillResource, + type SkillFileContents, type SkillIssue, type SkillVerification, } from "@inspector/core/mcp/skills.js"; @@ -37,11 +45,7 @@ import { CodeHighlight } from "../../elements/CodeHighlight/CodeHighlight"; import { ContentViewer } from "../../elements/ContentViewer/ContentViewer"; import { ListToggle } from "../../elements/ListToggle/ListToggle"; import { useValueChange } from "../../../hooks/useValueChange"; -import { - skillFileBytes, - type SkillFileContents, -} from "../../../utils/skillFileBytes"; -import { splitSkillFile } from "../../../utils/splitSkillFile"; +import { splitSkillFile } from "@inspector/core/mcp/skillFile.js"; import { inferMimeFromUri, isGenericMime, @@ -126,6 +130,38 @@ interface FetchedEntryState { message?: string; } +/** + * One child of a directory resource, as `resources/directory/read` returned it. + * Structurally the base protocol's `Resource`; only the three members this + * section renders are named. + */ +interface DirectoryChild { + uri: string; + name: string; + mimeType?: string; +} + +/** + * The directory browser's state: which directory is on screen, the children + * gathered so far, and the cursor for the next page. + * + * Pages **accumulate** rather than replacing, unlike a paged list elsewhere in + * the app, because a directory listing is one thing split across responses — a + * reader descending a tree wants the directory's contents, not page 2 of them. + * `key` invalidates it exactly as it does every other async slot here. + */ +interface DirectoryState { + key: string | null; + attempt?: number; + /** The directory being shown (or read). */ + uri?: string; + children?: DirectoryChild[]; + /** Cursor for the page after `children`, when the server sent one. */ + nextCursor?: string; + loading?: boolean; + message?: string; +} + export interface SkillsScreenProps { /** * Identity of the connected session. Part of the invalidation key below, so @@ -157,6 +193,20 @@ export interface SkillsScreenProps { * fresh read excuses. */ onGetSkill: (uri: string) => Promise; + /** + * One page of `resources/directory/read` (SEP-2640), or **`undefined` when + * the server did not declare `directoryRead`** — which is how the Directory + * section is gated. + * + * Gated by the prop's presence rather than by a boolean beside it, so the + * section cannot be rendered without a way to populate it: the SEP makes + * calling this method against a server that has not declared the sub-flag a + * MUST NOT, and an absent callback is that rule expressed in the type. + */ + onReadResourceDirectory?: ( + uri: string, + cursor?: string, + ) => Promise; } /** @@ -437,7 +487,13 @@ const SkillTitle = Text.withProps({ const SECTION_FLEX = "0 1 auto"; /** Every section this screen can render, in display order. */ -const ALL_SECTIONS = ["conformance", "resources", "frontmatter", "resource"]; +const ALL_SECTIONS = [ + "conformance", + "resources", + "directory", + "frontmatter", + "resource", +]; /** * The open set for the FIRST render. @@ -597,6 +653,7 @@ export function SkillsScreen({ onRefreshList, onReadSkillFile, onGetSkill, + onReadResourceDirectory, }: SkillsScreenProps) { const { selectedSkillUri, search } = ui; // Both slices carry the manifest key they belong to, and every async @@ -615,6 +672,7 @@ export function SkillsScreen({ const [fetchedEntry, setFetchedEntry] = useState({ key: null, }); + const [directory, setDirectory] = useState({ key: null }); // Every "Verify all" batch in flight, keyed by the manifest it belongs to. // // A **map**, not one slot, and the reason is a bug a single slot really had: @@ -716,6 +774,11 @@ export function SkillsScreen({ setVerification({ key: next, files: {} }); setPreviewState({ key: next }); setFetchedEntry({ key: next }); + // A directory listing is a live observation of a path under the *selected* + // skill, so it is invalidated with everything else — carrying one across a + // selection change would show the previous skill's tree under the new + // skill's name. + setDirectory({ key: next }); // Conformance tracks whether it has anything to say: an entry with no // errors and no warnings opens collapsed, because "0 error(s), 0 // warning(s)" on the header already carries the whole message and an @@ -940,6 +1003,86 @@ export function SkillsScreen({ showResource(selectedUri, manifestKey); }, [manifestKey, selectedUri, showResource]); + /** + * The skill's root directory: its entry URI with `/SKILL.md` removed. + * + * Computed from the NORMALIZED URI, like every containment decision in + * `core/mcp/skills.ts`, so a `..` segment cannot produce a root the resolved + * path does not carry. `undefined` for a malformed entry URI — there is no + * root to browse, and `malformed-uri` already reports that in Conformance. + */ + const skillRoot = useMemo(() => { + if (!selected) return undefined; + const normalized = skillUriIdentity(selected.uri); + return normalized.endsWith(SKILL_FILE_SUFFIX) + ? normalized.slice(0, -SKILL_FILE_SUFFIX.length) + : undefined; + }, [selected]); + + /** + * Read one page of a directory, replacing the listing (`cursor` omitted) or + * appending to it (`cursor` given). + * + * Keyed and attempt-stamped exactly as the other async slots here, so a read + * still in flight when the user descends into a different directory — or + * switches skills — cannot land afterwards and paint one directory's children + * under another's path. + */ + const readDirectory = useCallback( + (uri: string, key: string, cursor?: string) => { + if (!onReadResourceDirectory) return; + const attempt = (nextAttempt.current += 1); + const write = (next: Omit) => + setDirectory((prev) => { + if (prev.key !== null && prev.key !== key) return prev; + if (prev.attempt !== undefined && prev.attempt > attempt) return prev; + return { key, attempt, ...next }; + }); + // The path is claimed before the request goes out, so the header names + // the directory being read rather than continuing to announce the + // previous one for as long as the read takes. + setDirectory((prev) => + prev.key !== null && prev.key !== key + ? prev + : { + key, + attempt, + uri, + // Pages accumulate, so a "load more" keeps what is on screen; + // a fresh read of a different directory starts empty. + children: cursor === undefined ? undefined : prev.children, + loading: true, + }, + ); + // A click handler cannot await, and this chain terminates in its own + // `catch`, which surfaces the message in the section. + void onReadResourceDirectory(uri, cursor) + .then((page) => { + setDirectory((prev) => { + if (prev.key !== null && prev.key !== key) return prev; + if (prev.attempt !== undefined && prev.attempt > attempt) { + return prev; + } + const held = cursor === undefined ? [] : (prev.children ?? []); + return { + key, + attempt, + uri, + children: [...held, ...page.resources], + nextCursor: page.nextCursor, + }; + }); + }) + .catch((err: unknown) => { + write({ + uri, + message: err instanceof Error ? err.message : String(err), + }); + }); + }, + [onReadResourceDirectory], + ); + const fetchEntry = useCallback(() => { if (!selected) return; const key = manifestKey; @@ -1004,6 +1147,50 @@ export function SkillsScreen({ const batchRunning = batches.has(manifestKey); const previewCurrent = previewState.key === manifestKey; + /** + * The identities of every file the held entry's manifest lists. + * + * SEP-2640 is explicit that a directory read is *"a live observation that may + * run ahead of or behind"* the entry, and that **"Hosts MUST NOT treat the + * directory result as extending the manifest"** — a child the server lists + * but the entry does not is, to a host acting on the skill, a verification + * failure exactly as a digest mismatch is. The Inspector is not a host and + * does not refuse the read; what it must not do is present such a child as + * one of the skill's files without saying which view it came from. So the two + * views are labelled rather than merged. + * + * Compared on the normalized identity, like every other URI comparison here. + */ + const manifestIdentities = useMemo( + () => new Set(manifest.map((resource) => skillUriIdentity(resource.uri))), + [manifest], + ); + + // The directory slot, but only when it belongs to the current manifest — + // same guard every other async slot on this screen uses. + const directoryCurrent = directory.key === manifestKey; + const directoryUri = directoryCurrent ? directory.uri : undefined; + const directoryChildren = directoryCurrent ? directory.children : undefined; + const directoryError = directoryCurrent ? directory.message : undefined; + const directoryLoading = directoryCurrent && directory.loading === true; + const directoryNextCursor = directoryCurrent + ? directory.nextCursor + : undefined; + /** + * Children the directory listed that the held entry's manifest does not. + * Directories are excluded: a manifest lists files, so a directory is not a + * missing entry. So is a `"dynamic"` skill, which advertises no manifest for + * anything to be missing from. + */ + const unlistedChildren = useMemo(() => { + if (directoryChildren === undefined || isDynamic) return []; + return directoryChildren.filter( + (child) => + child.mimeType !== DIRECTORY_MIME_TYPE && + !manifestIdentities.has(skillUriIdentity(child.uri)), + ); + }, [directoryChildren, isDynamic, manifestIdentities]); + const preview = previewCurrent ? previewState.contents : undefined; const previewError = previewCurrent ? previewState.message : undefined; // The file the viewer is showing (or fetching). Falls back to the skill's own @@ -1089,10 +1276,13 @@ export function SkillsScreen({ () => [ "conformance", ...(isDynamic ? [] : ["resources"]), + ...(onReadResourceDirectory && skillRoot !== undefined + ? ["directory"] + : []), ...(previewParts?.frontmatter !== undefined ? ["frontmatter"] : []), "resource", ], - [isDynamic, previewParts], + [isDynamic, onReadResourceDirectory, previewParts, skillRoot], ); const allSectionsOpen = sectionIds.every((id) => openSections.includes(id)); @@ -1113,6 +1303,35 @@ export function SkillsScreen({ state.status === "done" && state.verification.status === "mismatch", ).length; + /** + * The SEP-2640 frontmatter cross-check, run against the file on screen — but + * **only when that file is the skill's own `SKILL.md`**. + * + * The obligation is that the entry's `frontmatter` match the frontmatter of + * the file the entry names; running it against a supporting file would report + * `frontmatter-absent` for every one of them, which is the tool inventing a + * defect. `showingSkillMd` is the same identity comparison the rest of this + * screen uses. + * + * It is deliberately **not** part of the static `issues` above: those are + * derived from the listing alone and are available the moment the list + * arrives, while this one needs a `resources/read` the user asked for. Folding + * them together would make the header badge's count change on its own the + * first time a file happened to be fetched. + */ + const frontmatterIssues = useMemo(() => { + if (!selected || !showingSkillMd || previewParts === undefined) return []; + // Reconstructed from the split rather than re-derived from the payload, so + // the check reads exactly the bytes the Frontmatter section displays. + if (previewParts.frontmatter === undefined) { + return checkSkillFrontmatterMatch(selected, previewParts.body); + } + return checkSkillFrontmatterMatch( + selected, + `---\n${previewParts.frontmatter}\n---\n\n${previewParts.body}`, + ); + }, [selected, showingSkillMd, previewParts]); + const errorCount = issues.filter((i) => i.severity === "error").length; const warningCount = issues.length - errorCount; @@ -1348,6 +1567,25 @@ export function SkillsScreen({ ))} )} + {/* The frontmatter cross-check renders in Conformance + rather than beside the Frontmatter section, because it + is a *finding about the entry* and every other finding + about the entry is here — a reader checking "does this + skill conform" must not have to know that one class of + violation is filed somewhere else. */} + {frontmatterIssues.length > 0 && ( + + {frontmatterIssues.map((issue, index) => ( + + {issue.message} + + ))} + + )} {manifest.map((resource, index) => { const state = fileStates[index]; if (state?.status === "done") { @@ -1597,6 +1835,197 @@ export function SkillsScreen({ )} + {/* Gated on the CALLBACK, which the parent supplies only when + the server declared `directoryRead`. SEP-2640 makes calling + `resources/directory/read` against a server that has not + declared it a MUST NOT, so an absent section is that rule + rather than a UI preference. */} + {onReadResourceDirectory && skillRoot !== undefined && ( + + + Directory + + + + {/* Read on a click, never on selection. A directory read + is a live round trip, and this screen's posture is + that every one of them is asked for — the same reason + "Fetch entry" is a button and not an effect. */} + + {directoryUri ?? skillRoot} + + {/* Ascending is bounded by the skill root: this + section browses the selected skill's tree, and + walking above it would leave the subject of every + other section on screen. */} + {directoryUri !== undefined && + directoryUri !== skillRoot && ( + + readDirectory( + directoryUri.slice( + 0, + directoryUri.lastIndexOf("/"), + ), + manifestKey, + ) + } + > + Up + + )} + + readDirectory(skillRoot, manifestKey) + } + > + {directoryChildren === undefined + ? "Read directory" + : "Reload root"} + + + + {directoryError !== undefined && ( + + {directoryError} + + )} + {/* Stated in prose the first time the two views + disagree, because the per-row chip alone does not say + why it matters — and "the skill changed and needs + re-approval" is what SEP-2640 asks a host to present + here, rather than a read error. */} + {unlistedChildren.length > 0 && ( + + The server is serving {unlistedChildren.length} file + {unlistedChildren.length === 1 ? "" : "s"} here that + the held skills/list entry does not + declare. A directory read is a live observation and + does not extend the manifest: to a + host acting on this skill, reading one of these is a + verification failure equivalent to a digest mismatch. + Re-fetch the entry with skills/get to see + whether the skill has changed. + + )} + {directoryChildren !== undefined && + (directoryChildren.length === 0 ? ( + This directory is empty. + ) : ( + + + + Name + URI + MIME type + In manifest + + + + {directoryChildren.map((child, index) => { + const isDir = + child.mimeType === DIRECTORY_MIME_TYPE; + // A directory is not a manifest entry in the + // first place — a manifest lists files — so it + // is neither listed nor unlisted and gets no + // verdict rather than a misleading "no". + const listed = manifestIdentities.has( + skillUriIdentity(child.uri), + ); + return ( + // Index-keyed for the same reason the + // manifest rows are: a server repeating a URI + // is a defect to display, not two rows to + // collapse into one. + + + + isDir + ? readDirectory( + child.uri, + manifestKey, + ) + : showResource( + child.uri, + manifestKey, + ) + } + > + {isDir ? `${child.name}/` : child.name} + + + + {child.uri} + + {child.mimeType ?? "—"} + + {isDir || isDynamic ? ( + + — + + ) : ( + + {listed ? "listed" : "not listed"} + + )} + + + ); + })} + + + ))} + {/* Paging is manual because the SEP gives the cursor to + the client and this screen is what a server author + uses to see their own pagination work. Auto-walking it + would hide exactly the behaviour under test. */} + {directoryNextCursor !== undefined && + directoryUri !== undefined && ( + + readDirectory( + directoryUri, + manifestKey, + directoryNextCursor, + ) + } + > + Load more + + )} + + + + )} + {/* Rendered ONLY when the file on display actually carries frontmatter. A skill's manifest files generally do not, and a section that lingered would be showing the previously selected diff --git a/clients/web/src/components/views/InspectorView/InspectorView.tsx b/clients/web/src/components/views/InspectorView/InspectorView.tsx index ff7d8a0cb0..9ee980f8cc 100644 --- a/clients/web/src/components/views/InspectorView/InspectorView.tsx +++ b/clients/web/src/components/views/InspectorView/InspectorView.tsx @@ -465,6 +465,7 @@ export function InspectorView({ onRefreshSkills, onReadSkillFile, onGetSkill, + onReadResourceDirectory, } = skillsPanel; const { tasks, @@ -1065,6 +1066,7 @@ export function InspectorView({ onRefreshList: onRefreshSkills, onReadSkillFile, onGetSkill, + onReadResourceDirectory, }; const tasksScreenProps = { tasks, diff --git a/clients/web/src/components/views/InspectorView/types.ts b/clients/web/src/components/views/InspectorView/types.ts index 1eff14681a..32d06bd143 100644 --- a/clients/web/src/components/views/InspectorView/types.ts +++ b/clients/web/src/components/views/InspectorView/types.ts @@ -61,7 +61,8 @@ import type { import type { LogsUiState } from "../../screens/LoggingScreen/LoggingScreen"; import type { SkillsUiState } from "../../screens/SkillsScreen/SkillsScreen"; import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js"; -import type { SkillFileContents } from "../../../utils/skillFileBytes"; +import type { SkillFileContents } from "@inspector/core/mcp/skills.js"; +import type { DirectoryReadResult } from "@inspector/core/mcp/skillsSchemas.js"; import type { TasksUiState } from "../../screens/TasksScreen/TasksScreen"; import type { ProtocolUiState } from "../../screens/ProtocolScreen/ProtocolScreen"; import type { NetworkUiState } from "../../screens/NetworkScreen/NetworkScreen"; @@ -330,6 +331,15 @@ export interface SkillsPanelProps { onReadSkillFile: (uri: string) => Promise; /** Re-fetch the selected entry through `skills/get`. */ onGetSkill: (uri: string) => Promise; + /** + * One page of `resources/directory/read`, or **`undefined` when the server + * did not declare `directoryRead`** — which is what gates the Skills screen's + * Directory section (SEP-2640 makes the call a MUST NOT otherwise). + */ + onReadResourceDirectory?: ( + uri: string, + cursor?: string, + ) => Promise; } /** The Tasks monitor: the task list, its progress map, and actions. */ diff --git a/clients/web/src/hooks/useServerCommands.test.tsx b/clients/web/src/hooks/useServerCommands.test.tsx index 004f0062fa..12d1fb97fa 100644 --- a/clients/web/src/hooks/useServerCommands.test.tsx +++ b/clients/web/src/hooks/useServerCommands.test.tsx @@ -946,6 +946,76 @@ describe("onReadSkillFile (#2234)", () => { }); }); +describe("onReadResourceDirectory (#2248)", () => { + const PAGE = { resources: [{ uri: "skill://demo/x.md", name: "x.md" }] }; + + it("passes the uri and cursor straight through to the client", async () => { + // Not aggregated across pages: SEP-2640 says the listing is not recursive + // and the client descends, so the cursor belongs to the caller. + const readResourceDirectory = vi.fn().mockResolvedValue(PAGE); + const h = harness({ client: client({ readResourceDirectory }) }); + await expect( + h.api().onReadResourceDirectory("skill://demo", "3"), + ).resolves.toBe(PAGE); + expect(readResourceDirectory).toHaveBeenCalledWith("skill://demo", "3"); + }); + + it("throws when there is no client", async () => { + await expect( + harness().api().onReadResourceDirectory("skill://demo"), + ).rejects.toThrow("Client is not connected"); + }); + + it("retries once after a satisfied recovery", async () => { + const recover = vi.fn().mockResolvedValue(true); + const readResourceDirectory = vi + .fn() + .mockRejectedValueOnce(authError()) + .mockResolvedValue(PAGE); + const h = harness({ + client: client({ readResourceDirectory }), + activeServerId: "a", + recovery: { handleCommandScopedAuthRecovery: recover }, + }); + await expect(h.api().onReadResourceDirectory("skill://demo")).resolves.toBe( + PAGE, + ); + expect(readResourceDirectory).toHaveBeenCalledTimes(2); + }); + + it("rethrows when the recovery was not satisfied", async () => { + const h = harness({ + client: client({ + readResourceDirectory: vi.fn().mockRejectedValue(authError()), + }), + activeServerId: "a", + recovery: { + handleCommandScopedAuthRecovery: vi.fn().mockResolvedValue(false), + }, + }); + await expect( + h.api().onReadResourceDirectory("skill://demo"), + ).rejects.toBeInstanceOf(AuthRecoveryRequiredError); + }); + + it("rethrows a non-auth failure untouched", async () => { + // The MUST NOT refusal for an undeclared `directoryRead` is raised by the + // client and must reach the caller unchanged, not be mistaken for auth. + const h = harness({ + client: client({ + readResourceDirectory: vi + .fn() + .mockRejectedValue(new Error("did not declare directoryRead")), + }), + activeServerId: "a", + recovery: { handleCommandScopedAuthRecovery: vi.fn() }, + }); + await expect( + h.api().onReadResourceDirectory("skill://demo"), + ).rejects.toThrow(/directoryRead/); + }); +}); + describe("onGetSkill (#2234)", () => { it("routes the uri through the client's skills/get", async () => { const getSkill = vi.fn().mockResolvedValue({ diff --git a/clients/web/src/hooks/useServerCommands.tsx b/clients/web/src/hooks/useServerCommands.tsx index 748101c425..2f02bec753 100644 --- a/clients/web/src/hooks/useServerCommands.tsx +++ b/clients/web/src/hooks/useServerCommands.tsx @@ -34,7 +34,8 @@ import type { } from "../components/screens/ToolsScreen/ToolsScreen"; import type { GetPromptState } from "../components/screens/PromptsScreen/PromptsScreen"; import type { ReadResourceState } from "../components/screens/ResourcesScreen/ResourcesScreen"; -import type { SkillFileContents } from "../utils/skillFileBytes"; +import type { SkillFileContents } from "@inspector/core/mcp/skills.js"; +import type { DirectoryReadResult } from "@inspector/core/mcp/skillsSchemas.js"; import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js"; import { normalizeSkillUri } from "@inspector/core/mcp/skills.js"; import { UrlElicitationErrorToastMessage } from "../components/elements/Toasts/UrlElicitationErrorToastMessage"; @@ -222,6 +223,11 @@ export interface ServerCommands { * over. */ onReadSkillFile: (uri: string) => Promise; + /** One page of `resources/directory/read` (SEP-2640). */ + onReadResourceDirectory: ( + uri: string, + cursor?: string, + ) => Promise; /** Re-fetch one skill entry through `skills/get` (SEP-2640). */ onGetSkill: (uri: string) => Promise; onSubscribeResource: (uri: string) => void; @@ -1026,6 +1032,37 @@ export function useServerCommands({ [inspectorClient, activeServerId, handleCommandScopedAuthRecovery], ); + /** + * One page of `resources/directory/read` (SEP-2640) — the direct children of + * a directory resource. + * + * Not aggregated across pages, unlike the skills walk: the SEP says the + * listing is not recursive and clients descend by calling again, so the + * cursor belongs to the caller doing the descending. `InspectorClient` refuses + * the call outright when the server did not declare `directoryRead`, which is + * the spec's MUST NOT — nothing here has to re-check it. + */ + const onReadResourceDirectory = useCallback( + async (uri: string, cursor?: string): Promise => { + if (!inspectorClient) throw new Error("Client is not connected"); + // Same shared auth recovery as every other server command (#2174). + const read = () => inspectorClient.readResourceDirectory(uri, cursor); + try { + return await read(); + } catch (err) { + if (err instanceof AuthRecoveryRequiredError && activeServerId) { + const satisfied = await handleCommandScopedAuthRecovery(err, { + serverId: activeServerId, + source: "resource", + }); + if (satisfied) return read(); + } + throw err; + } + }, + [inspectorClient, activeServerId, handleCommandScopedAuthRecovery], + ); + const onRefreshSkills = useCallback(() => { runCommandInBackground( () => refreshSkills(), @@ -1050,6 +1087,7 @@ export function useServerCommands({ onReadResource, onReadResourceContents, onReadSkillFile, + onReadResourceDirectory, onGetSkill, onSubscribeResource, onUnsubscribeResource, diff --git a/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts b/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts index 989a144850..8c2c1dcb18 100644 --- a/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts +++ b/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts @@ -228,6 +228,166 @@ describe("InspectorClient skills methods (#2234)", () => { expect(marked).toEqual([]); }); + /** Declare the extension with (or without) the `directoryRead` sub-flag. */ + function declareSkills(client: InspectorClient, directoryRead: boolean) { + internals(client).capabilities = { + extensions: { [SKILLS_EXTENSION_KEY]: { directoryRead } }, + } as ServerCapabilities; + } + + describe("readResourceDirectory (#2248)", () => { + const CHILD = { + uri: "skill://demo/ref.md", + name: "ref.md", + mimeType: "text/markdown", + }; + + it("throws when not connected, before the capability gate", async () => { + // The order matters: a disconnected client has no capabilities either, + // so checking the extension first would report every disconnected call + // as a missing `directoryRead` declaration. + const client = makeClient(); + declareSkills(client, true); + await expect( + client.readResourceDirectory("skill://demo"), + ).rejects.toThrow(/not connected/i); + }); + + it("refuses the call when the server did not declare directoryRead", async () => { + // SEP-2640 makes this a MUST NOT for the client, so it is refused + // locally rather than sent and answered -32601. A request we were never + // allowed to make must not appear in the Protocol log as a server fault. + const client = makeClient(); + const request = stubRequest(client, { resources: [] }); + declareSkills(client, false); + await expect( + client.readResourceDirectory("skill://demo"), + ).rejects.toThrow(/directoryRead/); + expect(request).not.toHaveBeenCalled(); + }); + + it("refuses the call when the extension is absent entirely", async () => { + const client = makeClient(); + stubRequest(client, { resources: [] }); + await expect( + client.readResourceDirectory("skill://demo"), + ).rejects.toThrow(/directoryRead/); + }); + + it("sends the uri and no cursor on the first page", async () => { + const client = makeClient(); + const request = stubRequest(client, { resources: [CHILD] }); + declareSkills(client, true); + const page = await client.readResourceDirectory("skill://demo"); + expect(request.mock.calls[0][0].method).toBe("resources/directory/read"); + expect(request.mock.calls[0][0].params.uri).toBe("skill://demo"); + expect(request.mock.calls[0][0].params).not.toHaveProperty("cursor"); + expect(page.resources).toEqual([CHILD]); + }); + + it("forwards a cursor and returns the server's nextCursor", async () => { + const client = makeClient(); + const request = stubRequest(client, { + resources: [], + nextCursor: "3", + }); + declareSkills(client, true); + const page = await client.readResourceDirectory("skill://demo", "2"); + expect(request.mock.calls[0][0].params.cursor).toBe("2"); + expect(page.nextCursor).toBe("3"); + }); + + it("forwards an empty-string cursor, which is a legal opaque value", async () => { + const client = makeClient(); + const request = stubRequest(client, { resources: [] }); + declareSkills(client, true); + await client.readResourceDirectory("skill://demo", ""); + expect(request.mock.calls[0][0].params.cursor).toBe(""); + }); + + it("stamps call metadata as _meta", async () => { + const client = makeClient(); + const request = stubRequest(client, { resources: [] }); + declareSkills(client, true); + await client.readResourceDirectory("skill://demo", undefined, { + progressToken: "p", + }); + expect(request.mock.calls[0][0].params._meta).toMatchObject({ + progressToken: "p", + }); + }); + + it("requires resultType on a modern connection", async () => { + const client = makeClient(); + internals(client).protocolEra = "modern"; + stubRequest(client, { resources: [] }); + declareSkills(client, true); + await expect( + client.readResourceDirectory("skill://demo"), + ).rejects.toBeDefined(); + }); + + it("accepts a legacy result without resultType", async () => { + const client = makeClient(); + stubRequest(client, { resources: [CHILD] }); + declareSkills(client, true); + await expect( + client.readResourceDirectory("skill://demo"), + ).resolves.toMatchObject({ resources: [CHILD] }); + }); + + it("attributes a rejected decode to the exchange it came from", async () => { + const client = makeClient(); + const marked: string[] = []; + ( + client as unknown as { + markResponseRejected: (m: string, r: string) => void; + } + ).markResponseRejected = (method) => { + marked.push(method); + }; + internals(client).client = { + request: async () => { + throw new SdkError( + SdkErrorCode.InvalidResult, + "Invalid result for resources/directory/read", + ); + }, + }; + declareSkills(client, true); + await expect( + client.readResourceDirectory("skill://demo"), + ).rejects.toBeDefined(); + expect(marked).toEqual(["resources/directory/read"]); + }); + + it("does NOT attribute a request that never produced a response", async () => { + // Marking here would stamp an earlier, successful exchange. + const client = makeClient(); + const marked: string[] = []; + ( + client as unknown as { + markResponseRejected: (m: string, r: string) => void; + } + ).markResponseRejected = (method) => { + marked.push(method); + }; + internals(client).client = { + request: async () => { + throw new SdkError( + SdkErrorCode.ConnectionClosed, + "Connection closed", + ); + }, + }; + declareSkills(client, true); + await expect( + client.readResourceDirectory("skill://demo"), + ).rejects.toThrow(); + expect(marked).toEqual([]); + }); + }); + it("rejects a skills/list result that is not a skills page", async () => { // The explicit result schema is the whole client-side mechanism for a // consumer-owned extension method, so a nonconforming result must fail diff --git a/clients/web/src/utils/splitSkillFile.test.ts b/clients/web/src/test/core/mcp/skillFile.test.ts similarity index 52% rename from clients/web/src/utils/splitSkillFile.test.ts rename to clients/web/src/test/core/mcp/skillFile.test.ts index fdd7e37bfa..fc7833b1e4 100644 --- a/clients/web/src/utils/splitSkillFile.test.ts +++ b/clients/web/src/test/core/mcp/skillFile.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect } from "vitest"; -import { splitSkillFile } from "./splitSkillFile"; +import { + parseSkillFrontmatter, + splitSkillFile, +} from "@inspector/core/mcp/skillFile.js"; describe("splitSkillFile", () => { it("separates a leading frontmatter fence from the body", () => { @@ -64,3 +67,51 @@ describe("splitSkillFile", () => { ).toEqual({ frontmatter: "name: a\ndescription: b", body: "Body\n" }); }); }); + +describe("parseSkillFrontmatter (#2248)", () => { + it("parses a mapping of fields", () => { + expect(parseSkillFrontmatter("name: demo\ndescription: A demo")).toEqual({ + fields: { name: "demo", description: "A demo" }, + }); + }); + + it("keeps non-string scalars as their YAML 1.2 core types", () => { + const parsed = parseSkillFrontmatter("n: 1\nb: true\nl: [1, 2]"); + expect(parsed).toEqual({ fields: { n: 1, b: true, l: [1, 2] } }); + }); + + it("leaves a timestamp-shaped value a string", () => { + // The other side of the comparison arrived over JSON-RPC and can only hold + // JSON types, so a `Date` here would report a conforming server as broken. + // This is the YAML 1.2 core schema doing its job — under 1.1 it would be a + // Date and the check would be wrong. + const parsed = parseSkillFrontmatter("when: 2001-12-14t21:59:43.10-05:00"); + expect(parsed).toEqual({ + fields: { when: "2001-12-14t21:59:43.10-05:00" }, + }); + }); + + it("reads an empty block as a mapping of no fields, not an error", () => { + expect(parseSkillFrontmatter("")).toEqual({ fields: {} }); + expect(parseSkillFrontmatter("# just a comment")).toEqual({ fields: {} }); + }); + + it("reports a scalar block as an error rather than as no fields", () => { + // `just a string` parses successfully as a scalar. Reporting it as an + // empty mapping would present a malformed file as one that merely omitted + // every field. + const parsed = parseSkillFrontmatter("just a string"); + expect(parsed).toEqual({ error: expect.stringContaining("mapping") }); + }); + + it("reports a sequence block as an error", () => { + expect(parseSkillFrontmatter("- one\n- two")).toEqual({ + error: expect.stringContaining("mapping"), + }); + }); + + it("reports invalid YAML with the parser's own message", () => { + const parsed = parseSkillFrontmatter("a: [1,"); + expect("error" in parsed && parsed.error.length > 0).toBe(true); + }); +}); diff --git a/clients/web/src/utils/skillFileBytes.test.ts b/clients/web/src/test/core/mcp/skillFileBytes.test.ts similarity index 95% rename from clients/web/src/utils/skillFileBytes.test.ts rename to clients/web/src/test/core/mcp/skillFileBytes.test.ts index be27497d64..0aab3c9dab 100644 --- a/clients/web/src/utils/skillFileBytes.test.ts +++ b/clients/web/src/test/core/mcp/skillFileBytes.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { skillFileBytes } from "./skillFileBytes"; +import { skillFileBytes } from "@inspector/core/mcp/skills.js"; describe("skillFileBytes", () => { it("encodes a text content block as UTF-8", () => { diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index f4ea62e26c..e346368165 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -7,6 +7,7 @@ import { SKILL_MAX_TOTAL_BYTES, base64ToBytes, checkSkillConformance, + checkSkillFrontmatterMatch, getSkillsExtension, isSkillsExtensionSupported, normalizeSkillUri, @@ -843,3 +844,137 @@ describe("verifySkillResource", () => { expect(result.expectedDigest).toBe("sha256:nope"); }); }); + +describe("checkSkillFrontmatterMatch (#2248)", () => { + const entry = (frontmatter: Record): SkillEntry => ({ + uri: "skill://demo/SKILL.md", + frontmatter, + resources: [], + }); + const file = (yaml: string, body = "# Demo\n") => + `---\n${yaml}\n---\n\n${body}`; + + it("reports nothing when every field agrees", () => { + expect( + checkSkillFrontmatterMatch( + entry({ name: "demo", description: "A demo" }), + file("name: demo\ndescription: A demo"), + ), + ).toEqual([]); + }); + + it("catches a listing that advertises a different description", () => { + // The violation no digest can catch: the digest is over the bytes the + // server served and says nothing about whether the listing described them + // honestly. + const issues = checkSkillFrontmatterMatch( + entry({ name: "demo", description: "Reads a spreadsheet" }), + file("name: demo\ndescription: Emails the spreadsheet"), + ); + expect(issues).toHaveLength(1); + expect(issues[0].code).toBe("frontmatter-mismatch"); + // Equivalent to a digest mismatch per the SEP, so it must be an error. + expect(issues[0].severity).toBe("error"); + // The diagnosis, not just the verdict — a server author has to be able to + // fix it from the message alone. + expect(issues[0].message).toContain("Reads a spreadsheet"); + expect(issues[0].message).toContain("Emails the spreadsheet"); + expect(issues[0].resourceUri).toBe("skill://demo/SKILL.md"); + }); + + it("reports one finding per differing field", () => { + const issues = checkSkillFrontmatterMatch( + entry({ name: "a", description: "x" }), + file("name: b\ndescription: y"), + ); + expect(issues).toHaveLength(2); + expect(issues.map((i) => i.message.match(/"(\w+)"/)?.[1])).toEqual([ + "description", + "name", + ]); + }); + + it("reports a field the file declares and the listing omits", () => { + const issues = checkSkillFrontmatterMatch( + entry({ name: "demo", description: "A demo" }), + file("name: demo\ndescription: A demo\nlicense: MIT"), + ); + expect(issues).toHaveLength(1); + expect(issues[0].message).toMatch(/declares "license".*omits it/); + }); + + it("reports a field the listing declares and the file omits", () => { + const issues = checkSkillFrontmatterMatch( + entry({ name: "demo", description: "A demo", license: "MIT" }), + file("name: demo\ndescription: A demo"), + ); + expect(issues).toHaveLength(1); + expect(issues[0].message).toMatch( + /listing declares "license".*served SKILL.md omits it/, + ); + }); + + it("treats a file with no frontmatter block as a violation", () => { + const issues = checkSkillFrontmatterMatch( + entry({ name: "demo" }), + "# Demo\n\nNo fence here.\n", + ); + expect(issues).toEqual([ + expect.objectContaining({ + code: "frontmatter-absent", + severity: "error", + }), + ]); + }); + + it("reports unparsable YAML as its own code, not as a mismatch", () => { + const issues = checkSkillFrontmatterMatch( + entry({ name: "demo" }), + file("a: [1,"), + ); + expect(issues).toEqual([ + expect.objectContaining({ + code: "frontmatter-unparsable", + severity: "error", + }), + ]); + }); + + it("compares nested mappings by content, not by key order", () => { + // Key order is not meaningful in JSON or YAML, so calling it a discrepancy + // would report a conforming server as broken. + expect( + checkSkillFrontmatterMatch( + entry({ meta: { b: 2, a: 1 } }), + file("meta:\n a: 1\n b: 2"), + ), + ).toEqual([]); + }); + + it("treats array ORDER as significant", () => { + // A YAML sequence is ordered, so two orderings are two different values. + const issues = checkSkillFrontmatterMatch( + entry({ tags: ["a", "b"] }), + file("tags: [b, a]"), + ); + expect(issues).toHaveLength(1); + expect(issues[0].code).toBe("frontmatter-mismatch"); + }); + + it("distinguishes an explicit null from an absent field", () => { + // `license:` with no value parses to null — a field that is present and + // holds null, which is not the same fact as a field that is not there. + const issues = checkSkillFrontmatterMatch( + entry({ license: null }), + file("license:"), + ); + expect(issues).toEqual([]); + expect( + checkSkillFrontmatterMatch(entry({}), file("license:")), + ).toHaveLength(1); + }); + + it("reports nothing for two empty frontmatters", () => { + expect(checkSkillFrontmatterMatch(entry({}), file(""))).toEqual([]); + }); +}); diff --git a/clients/web/src/test/core/mcp/skillsSchemas.test.ts b/clients/web/src/test/core/mcp/skillsSchemas.test.ts index 085e60b893..2bbe3407c8 100644 --- a/clients/web/src/test/core/mcp/skillsSchemas.test.ts +++ b/clients/web/src/test/core/mcp/skillsSchemas.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from "vitest"; import { + DIRECTORY_MIME_TYPE, + DirectoryReadResultSchema, + ModernDirectoryReadResultSchema, + RESOURCES_DIRECTORY_READ_METHOD, DYNAMIC_RESOURCES, GetSkillResultSchema, ListSkillsResultSchema, @@ -160,3 +164,123 @@ describe("GetSkillResultSchema", () => { ).toThrow(); }); }); + +describe("directory read schemas (#2248)", () => { + const CHILD = { + uri: "skill://demo/templates/invoice.md", + name: "invoice.md", + mimeType: "text/markdown", + }; + + it("names the method and the directory MIME type", () => { + expect(RESOURCES_DIRECTORY_READ_METHOD).toBe("resources/directory/read"); + expect(DIRECTORY_MIME_TYPE).toBe("inode/directory"); + }); + + it("parses the SEP's own worked example", () => { + // Verbatim from SEP-2640's `resources/directory/read` example, including a + // subdirectory child. If the schema cannot read the spec's own example it + // is wrong whatever else it accepts. + const example = { + resultType: "complete", + resources: [ + CHILD, + { + uri: "skill://demo/templates/regional", + name: "regional", + mimeType: "inode/directory", + }, + ], + }; + expect(DirectoryReadResultSchema.safeParse(example).success).toBe(true); + expect(ModernDirectoryReadResultSchema.safeParse(example).success).toBe( + true, + ); + }); + + it("accepts an empty directory", () => { + const parsed = DirectoryReadResultSchema.parse({ resources: [] }); + expect(parsed.resources).toEqual([]); + }); + + it("carries nextCursor through", () => { + const parsed = DirectoryReadResultSchema.parse({ + resources: [CHILD], + nextCursor: "7", + }); + expect(parsed.nextCursor).toBe("7"); + }); + + it("rejects a child that is not a base-protocol Resource", () => { + // `name` is required on `Resource`, and the SEP says a directory child IS + // one. Accepting a child here that `resources/list` would reject is the + // inconsistency the shared SDK schema exists to prevent. + expect( + DirectoryReadResultSchema.safeParse({ + resources: [{ uri: "skill://demo/x.md" }], + }).success, + ).toBe(false); + }); + + it("rejects a result whose resources member is not an array", () => { + expect( + DirectoryReadResultSchema.safeParse({ resources: "nope" }).success, + ).toBe(false); + }); + + it("requires resultType on the modern variant only", () => { + const legacyShape = { resources: [CHILD] }; + expect(DirectoryReadResultSchema.safeParse(legacyShape).success).toBe(true); + expect(ModernDirectoryReadResultSchema.safeParse(legacyShape).success).toBe( + false, + ); + }); + + it("does NOT require the caching attributes on the modern variant", () => { + // The deliberate asymmetry with `ModernListSkillsResultSchema`: SEP-2640 + // states `ttlMs`/`cacheScope` for a modern `skills/list` and says nothing + // of the kind for this method, whose only worked example omits them. + // Requiring them would fail a server that matched the spec's own example. + expect( + ModernDirectoryReadResultSchema.safeParse({ + resultType: "complete", + resources: [], + }).success, + ).toBe(true); + expect( + ModernListSkillsResultSchema.safeParse({ + resultType: "complete", + skills: [], + }).success, + ).toBe(false); + }); + + it("still accepts the caching attributes when a server sends them", () => { + // Permitted, not mandated — a schema is not the place to reject an extra + // member the spec leaves open. + expect( + ModernDirectoryReadResultSchema.safeParse({ + resultType: "complete", + resources: [], + ttlMs: 60, + cacheScope: "public", + }).success, + ).toBe(true); + }); +}); + +describe("GetSkillResultSchema caching attributes (#2248)", () => { + it("accepts a result with the caching attributes and one without", () => { + // SEP-2640 leaves the question open in as many words, so both are + // conforming and neither may be reported as a defect. + expect(GetSkillResultSchema.safeParse({ skill: ENTRY }).success).toBe(true); + expect( + GetSkillResultSchema.safeParse({ + skill: ENTRY, + resultType: "complete", + ttlMs: 0, + cacheScope: "public", + }).success, + ).toBe(true); + }); +}); diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts new file mode 100644 index 0000000000..57906b82d8 --- /dev/null +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -0,0 +1,329 @@ +import { describe, it, expect, vi } from "vitest"; +import type { InspectorClientProtocol } from "@inspector/core/mcp/inspectorClientProtocol.js"; +import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js"; +import { sha256Digest } from "@inspector/core/mcp/skills.js"; +import { AuthRecoveryRequiredError } from "@inspector/core/auth/challenge.js"; +import { + allSkillsVerified, + verifySkills, +} from "@inspector/core/mcp/skillsVerification.js"; + +/** + * `verifySkills` is the fetch-and-verify half of the SEP-2640 checks (#2248) — + * the part the pure checkers in `skills.ts` deliberately do not do. What these + * pin is the fetching policy and the failure handling, since the checks + * themselves are covered in `skills.test.ts`. + */ +describe("verifySkills (#2248)", () => { + const SKILL_MD = "---\nname: demo\ndescription: A demo\n---\n\n# Demo\n"; + const REF = "# Reference\n"; + + async function entry( + overrides: Partial = {}, + ): Promise { + return { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/SKILL.md", + digest: await sha256Digest(new TextEncoder().encode(SKILL_MD)), + size: new TextEncoder().encode(SKILL_MD).byteLength, + }, + { + uri: "skill://demo/ref.md", + digest: await sha256Digest(new TextEncoder().encode(REF)), + size: new TextEncoder().encode(REF).byteLength, + }, + ], + ...overrides, + }; + } + + /** A client whose `resources/read` answers from a URI → text map. */ + function clientServing(files: Record): { + client: InspectorClientProtocol; + readResource: ReturnType; + } { + const readResource = vi.fn(async (uri: string) => { + const served = files[uri]; + if (served === undefined) throw new Error(`unknown resource ${uri}`); + if (served instanceof Error) throw served; + return { result: { contents: [{ uri, text: served }] } }; + }); + return { + client: { readResource } as unknown as InspectorClientProtocol, + readResource, + }; + } + + it("verifies a clean skill and reports ok", async () => { + const skill = await entry(); + const { client } = clientServing({ + "skill://demo/SKILL.md": SKILL_MD, + "skill://demo/ref.md": REF, + }); + const [report] = await verifySkills(client, [skill]); + expect(report.ok).toBe(true); + expect(report.name).toBe("demo"); + expect(report.conformance).toEqual([]); + expect(report.frontmatter).toEqual([]); + expect(report.files.map((f) => f.status)).toEqual(["verified", "verified"]); + expect(allSkillsVerified([report])).toBe(true); + }); + + it("reads each manifest file exactly once", async () => { + // The entry's own SKILL.md is needed twice — for its digest and for the + // frontmatter cross-check — and reading it twice would both double the + // load and risk comparing two different snapshots. + const skill = await entry(); + const { client, readResource } = clientServing({ + "skill://demo/SKILL.md": SKILL_MD, + "skill://demo/ref.md": REF, + }); + await verifySkills(client, [skill]); + expect(readResource).toHaveBeenCalledTimes(2); + }); + + it("reports a digest mismatch and fails the skill", async () => { + const skill = await entry(); + const { client } = clientServing({ + "skill://demo/SKILL.md": SKILL_MD, + "skill://demo/ref.md": "different bytes entirely\n", + }); + const [report] = await verifySkills(client, [skill]); + expect(report.ok).toBe(false); + expect(report.files[1].status).toBe("mismatch"); + expect(allSkillsVerified([report])).toBe(false); + }); + + it("catches a listing whose frontmatter differs from the served file", async () => { + const skillMd = "---\nname: demo\ndescription: Something else\n---\n\n#\n"; + const bytes = new TextEncoder().encode(skillMd); + const skill: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/SKILL.md", + // The digest is over the bytes actually served, so it VERIFIES — + // which is the whole reason this check has to exist separately. + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + const { client } = clientServing({ "skill://demo/SKILL.md": skillMd }); + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("verified"); + expect(report.frontmatter).toHaveLength(1); + expect(report.ok).toBe(false); + }); + + it("records a read failure per file instead of aborting the report", async () => { + // A report that stopped at the first unreadable file would hide every + // finding after it, which defeats the point of running this in CI. + const skill = await entry(); + const { client } = clientServing({ + "skill://demo/SKILL.md": new Error("boom"), + "skill://demo/ref.md": REF, + }); + const [report] = await verifySkills(client, [skill]); + expect(report.files[0]).toMatchObject({ + status: "read-error", + reason: "boom", + }); + expect(report.files[1].status).toBe("verified"); + expect(report.ok).toBe(false); + }); + + it("reports a response with no content blocks as a read failure", async () => { + const skill = await entry(); + const readResource = vi.fn(async () => ({ result: { contents: [] } })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0]).toMatchObject({ status: "read-error" }); + expect(report.files[0].reason).toMatch(/no content blocks/); + }); + + it("reports a block carrying neither text nor blob as a read failure", async () => { + // Never as an empty file: an empty Uint8Array has a perfectly good + // SHA-256, so a silent fallback would report a confident, wrong mismatch. + const skill = await entry(); + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, mimeType: "text/markdown" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("read-error"); + expect(report.files[0].reason).toMatch(/neither text nor blob/); + }); + + it("still runs the frontmatter check for a dynamic skill", async () => { + // `"dynamic"` waives integrity, not the frontmatter identity requirement — + // the SKILL.md is still served and still has to match what was listed. + const skill: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen", description: "Listed" }, + resources: "dynamic", + }; + const { client, readResource } = clientServing({ + "skill://gen/SKILL.md": + "---\nname: gen\ndescription: Served\n---\n\n# Gen\n", + }); + const [report] = await verifySkills(client, [skill]); + expect(report.files).toEqual([]); + expect(readResource).toHaveBeenCalledWith( + "skill://gen/SKILL.md", + undefined, + ); + expect(report.frontmatter).toHaveLength(1); + expect(report.ok).toBe(false); + }); + + it("passes a dynamic skill whose served frontmatter agrees", async () => { + // The `dynamic-resources` finding is a WARNING, and a warning must not fail + // the report — a conforming generated skill would otherwise fail CI. + const skill: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen", description: "Same" }, + resources: "dynamic", + }; + const { client } = clientServing({ + "skill://gen/SKILL.md": + "---\nname: gen\ndescription: Same\n---\n\n# Gen\n", + }); + const [report] = await verifySkills(client, [skill]); + expect(report.conformance).toEqual([ + expect.objectContaining({ + code: "dynamic-resources", + severity: "warning", + }), + ]); + expect(report.ok).toBe(true); + }); + + const authError = () => + new AuthRecoveryRequiredError(new URL("https://auth.example/authorize"), { + reason: "expired", + } as never); + + it("re-throws an auth-recovery error instead of recording it per file", async () => { + // Not a property of the file in flight: the session's authorization + // expired, so every remaining read fails the same way. Absorbing it would + // produce N identical read failures AND swallow the one error a caller + // keys off to start a reauthorization. + const skill = await entry(); + const readResource = vi.fn(async () => { + throw authError(); + }); + const client = { readResource } as unknown as InspectorClientProtocol; + await expect(verifySkills(client, [skill])).rejects.toBeInstanceOf( + AuthRecoveryRequiredError, + ); + // Stops at the first read rather than walking the rest of the manifest. + expect(readResource).toHaveBeenCalledTimes(1); + }); + + it("re-throws an auth-recovery error from a dynamic skill's SKILL.md read", async () => { + // The other read site: a dynamic skill has no manifest, so its SKILL.md is + // fetched by the fallback below the loop, which has its own catch. + const skill: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen" }, + resources: "dynamic", + }; + const readResource = vi.fn(async () => { + throw authError(); + }); + const client = { readResource } as unknown as InspectorClientProtocol; + await expect(verifySkills(client, [skill])).rejects.toBeInstanceOf( + AuthRecoveryRequiredError, + ); + }); + + it("skips the frontmatter check when the SKILL.md cannot be read", async () => { + // The read failure is reported once, as a file result. Reporting it again + // as a phantom `frontmatter-absent` would invent a second defect. + const skill: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen" }, + resources: "dynamic", + }; + const readResource = vi.fn(async () => { + throw new Error("unreachable"); + }); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.frontmatter).toEqual([]); + }); + + it("fails a skill whose static conformance has an error", async () => { + const skill: SkillEntry = { + uri: "skill://wrong/SKILL.md", + frontmatter: { name: "right", description: "d" }, + resources: [], + }; + const { client } = clientServing({ + "skill://wrong/SKILL.md": + "---\nname: right\ndescription: d\n---\n\n# X\n", + }); + const [report] = await verifySkills(client, [skill]); + expect(report.conformance.map((i) => i.code)).toContain( + "name-path-mismatch", + ); + expect(report.ok).toBe(false); + }); + + it("accepts a canonicalized URI in the served content block", async () => { + // A server may answer with an RFC-equivalent spelling of the URI asked + // for; matching the block by URI would reject a conforming server. + const bytes = new TextEncoder().encode(SKILL_MD); + const skill: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/SKILL.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + const readResource = vi.fn(async () => ({ + result: { + contents: [{ uri: "skill://demo/%53KILL.md", text: SKILL_MD }], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("verified"); + }); + + it("reports every skill it was given, in order", async () => { + const a = await entry(); + const b = await entry({ uri: "skill://demo/SKILL.md" }); + const { client } = clientServing({ + "skill://demo/SKILL.md": SKILL_MD, + "skill://demo/ref.md": REF, + }); + const reports = await verifySkills(client, [a, b]); + expect(reports).toHaveLength(2); + }); + + it("forwards request metadata to every read", async () => { + const skill = await entry(); + const { client, readResource } = clientServing({ + "skill://demo/SKILL.md": SKILL_MD, + "skill://demo/ref.md": REF, + }); + await verifySkills(client, [skill], { progressToken: "p" }); + expect(readResource).toHaveBeenCalledWith("skill://demo/SKILL.md", { + progressToken: "p", + }); + }); + + it("allSkillsVerified is true for an empty report", async () => { + expect(allSkillsVerified([])).toBe(true); + }); +}); diff --git a/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts b/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts index 3b53d6a3eb..6e3df4ecf5 100644 --- a/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts +++ b/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts @@ -3,6 +3,10 @@ import { InspectorClient } from "@inspector/core/mcp/inspectorClient.js"; import { createTransportNode } from "@inspector/core/mcp/node/transport.js"; import { getSkillsExtension } from "@inspector/core/mcp/skills.js"; import { ManagedSkillsState } from "@inspector/core/mcp/state/managedSkillsState.js"; +import { + allSkillsVerified, + verifySkills, +} from "@inspector/core/mcp/skillsVerification.js"; import { createTestServerHttp, type TestServerHttp, @@ -96,9 +100,11 @@ describe("Skills extension over a real transport (#2234)", () => { it("advertises the extension in its capabilities", async () => { const started = await startSkillsServer(modern); const connected = await connect(started.url, modern); - // Bare, per the fixture: no `directoryRead` until phase 3 serves it. + // `directoryRead` is declared because the fixture now serves the + // method (#2248) — the declaration and the handler are one switch, so + // this can never report a sub-option the server does not answer. expect(getSkillsExtension(connected.getCapabilities())).toEqual({ - directoryRead: false, + directoryRead: true, }); }); @@ -115,13 +121,18 @@ describe("Skills extension over a real transport (#2234)", () => { // below — so a modern page missing the envelope surfaces here as a // rejection rather than as a missing property. // - // The fixture pages at two, so a client that stops here sees half. + // The fixture pages at two over five skills, so a client that stops + // here sees less than half. expect(first.skills).toHaveLength(2); expect(first.nextCursor).toBeDefined(); const second = await connected.listSkills(first.nextCursor); expect(second.skills).toHaveLength(2); - expect(second.nextCursor).toBeUndefined(); + expect(second.nextCursor).toBeDefined(); + + const third = await connected.listSkills(second.nextCursor); + expect(third.skills).toHaveLength(2); + expect(third.nextCursor).toBeUndefined(); }); it("walks every page through the managed store", async () => { @@ -134,9 +145,11 @@ describe("Skills extension over a real transport (#2234)", () => { "data-analysis", "tampered-notes", "dynamic-report", + "stale-manifest", + "lying-listing", "right-name", ]); - expect(store.getPagination()).toEqual({ pageCount: 2 }); + expect(store.getPagination()).toEqual({ pageCount: 3 }); } finally { store.destroy(); } @@ -171,6 +184,127 @@ describe("Skills extension over a real transport (#2234)", () => { expect("text" in block && block.text).toContain("Column rules"); }); + it("reads a directory and pages through its children", async () => { + // The whole `resources/directory/read` round trip against a real + // server: the client's `directoryRead` gate, the era-selected result + // schema, and the fixture's cursor. + const started = await startSkillsServer(modern); + const connected = await connect(started.url, modern); + const first = await connected.readResourceDirectory( + "skill://data-analysis", + ); + // Pages at one child, so a client ignoring `nextCursor` is visibly + // wrong here rather than merely lucky. + expect(first.resources).toHaveLength(1); + expect(first.nextCursor).toBeDefined(); + const second = await connected.readResourceDirectory( + "skill://data-analysis", + first.nextCursor, + ); + expect(second.nextCursor).toBeUndefined(); + expect( + [...first.resources, ...second.resources].map((r) => r.uri).sort(), + ).toEqual([ + "skill://data-analysis/SKILL.md", + "skill://data-analysis/reference.md", + ]); + }); + + it("lists a dynamic skill's files, which is what the method is for", async () => { + // `dynamic-report` advertises no manifest, so a directory read is the + // only way its files are discoverable at all — the case SEP-2640 says + // directory reading earns its place for. + const started = await startSkillsServer(modern); + const connected = await connect(started.url, modern); + const page = await connected.readResourceDirectory( + "skill://dynamic-report", + ); + expect(page.resources[0].uri).toBe("skill://dynamic-report/SKILL.md"); + }); + + it("lists a file the entry's manifest does not declare", async () => { + // The stale-snapshot case SEP-2640 governs: a directory read is "a + // live observation" that may run ahead of the held entry, and hosts + // MUST NOT treat it as extending the manifest. The entry itself is + // fully conforming — only the two views disagree — so nothing but this + // comparison can surface it. + const started = await startSkillsServer(modern); + const connected = await connect(started.url, modern); + const entry = await connected.getSkill( + "skill://stale-manifest/SKILL.md", + ); + const declared = new Set( + (entry.resources === "dynamic" ? [] : entry.resources).map( + (r) => r.uri, + ), + ); + expect(declared).toEqual(new Set(["skill://stale-manifest/SKILL.md"])); + + const first = await connected.readResourceDirectory( + "skill://stale-manifest", + ); + const second = await connected.readResourceDirectory( + "skill://stale-manifest", + first.nextCursor, + ); + const children = [...first.resources, ...second.resources].map( + (r) => r.uri, + ); + expect(children).toContain("skill://stale-manifest/added-later.md"); + expect(declared.has("skill://stale-manifest/added-later.md")).toBe( + false, + ); + + // And the entry still verifies clean — the disagreement is the whole + // defect, and no digest check can see it. + const [report] = await verifySkills(connected, [entry]); + expect(report.ok).toBe(true); + }); + + it("answers -32602 for a URI that is not a directory resource", async () => { + const started = await startSkillsServer(modern); + const connected = await connect(started.url, modern); + await expect( + connected.readResourceDirectory("skill://data-analysis/SKILL.md"), + ).rejects.toThrow(/Not a directory resource/); + }); + + it("verifies the whole catalog, failing exactly the three bad skills", async () => { + // End to end against the fixture: conformance, digests and the + // frontmatter cross-check, over a real transport. The three failures + // are one per violation class, and `dynamic-report` passing is the + // assertion that a warning does not fail a report. + const started = await startSkillsServer(modern); + const connected = await connect(started.url, modern); + const store = new ManagedSkillsState(connected); + try { + const skills = await store.refresh(); + const reports = await verifySkills(connected, skills); + expect(reports.filter((r) => !r.ok).map((r) => r.name)).toEqual([ + "tampered-notes", + "lying-listing", + "right-name", + ]); + expect(allSkillsVerified(reports)).toBe(false); + + const tampered = reports.find((r) => r.name === "tampered-notes")!; + expect(tampered.files.some((f) => f.status === "mismatch")).toBe( + true, + ); + + // The one violation only the frontmatter check can catch: its digest + // verifies, because the digest is over the bytes the server served. + const lying = reports.find((r) => r.name === "lying-listing")!; + expect(lying.files.every((f) => f.status === "verified")).toBe(true); + expect(lying.frontmatter[0].code).toBe("frontmatter-mismatch"); + + const dynamic = reports.find((r) => r.name === "dynamic-report")!; + expect(dynamic.ok).toBe(true); + } finally { + store.destroy(); + } + }); + it("still serves an ordinary resource — the wrapper delegates", async () => { // The one thing the `resources/read` wrap must not break. const started = await startSkillsServer(modern); diff --git a/clients/web/src/utils/skillFileBytes.ts b/clients/web/src/utils/skillFileBytes.ts deleted file mode 100644 index a1fd0e141d..0000000000 --- a/clients/web/src/utils/skillFileBytes.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Decoding a `resources/read` payload back to the bytes its digest was taken - * over (SEP-2640, #2234). - * - * A pure transform with no I/O and no subsystem ownership, so it belongs in - * `utils/` rather than `lib/` — the screen that verifies a skill file does the - * fetching; this only turns what came back into bytes. - */ - -import { base64ToBytes, textToBytes } from "@inspector/core/mcp/skills.js"; - -/** - * The content a `resources/read` returned for one skill file. Either `text` (a - * `TextResourceContents`) or `blob` (base64, a `BlobResourceContents`). - */ -export interface SkillFileContents { - text?: string; - blob?: string; - mimeType?: string; -} - -/** - * The raw bytes of a skill file, as fetched. - * - * Throws for a result carrying neither `text` nor `blob`. That is a server bug, - * and it must not be quietly treated as empty content: an empty `Uint8Array` - * has a perfectly good SHA-256, so a silent fallback would report a *digest - * mismatch* — a confident, wrong diagnosis — instead of "this response carried - * no content at all". Callers surface the throw as a per-file read failure. - */ -export function skillFileBytes(contents: SkillFileContents): Uint8Array { - if (typeof contents.text === "string") return textToBytes(contents.text); - if (typeof contents.blob === "string") return base64ToBytes(contents.blob); - throw new Error("resources/read returned neither text nor blob content."); -} diff --git a/clients/web/src/utils/splitSkillFile.ts b/clients/web/src/utils/splitSkillFile.ts deleted file mode 100644 index b60be956d1..0000000000 --- a/clients/web/src/utils/splitSkillFile.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Split a skill file into its YAML frontmatter and its body (#2263). - * - * A pure transform with no I/O and no subsystem of its own, so it lives in - * `utils/` rather than `lib/` — and in its own module rather than in - * `SkillsScreen.tsx`, because a component file that also exports a function - * defeats React Fast Refresh (`react-refresh/only-export-components`). - */ - -export interface SkillFileParts { - /** - * The raw YAML between the fences, fences excluded — `undefined` when the - * file has no frontmatter at all. Raw rather than parsed: this app carries no - * YAML parser, and showing the bytes the server actually served is the more - * useful answer for a conformance tool anyway. - */ - frontmatter?: string; - /** Everything after the closing fence, or the whole file when there is none. */ - body: string; -} - -/** - * Separate a leading YAML frontmatter fence from the rest of a skill file. - * - * The Skills screen renders the two halves in different places — the - * frontmatter in its own collapsible section, the body in the file viewer — and - * deriving both from **one** split is what stops them disagreeing: the section - * can never show one file's frontmatter while the viewer shows another's, and a - * file with no frontmatter cannot leave a stale section on screen. - * - * It also matters for rendering: the markdown renderer has no frontmatter - * support, so an un-split `---\nname: …\n---` is read as a setext heading and - * painted as a title above the document's real one. - * - * Two deliberate conservatisms, because this must never eat content: - * - * - Only a fence at the very **start** of the file counts. A `---` anywhere - * else is a horizontal rule and is left in the body. - * - A file that opens with `---` but never closes the fence is **not** - * frontmatter; the whole file is returned as the body rather than being - * truncated to nothing. - */ -export function splitSkillFile(text: string): SkillFileParts { - if (!/^---[ \t]*\r?\n/.test(text)) return { body: text }; - const rest = text.slice(text.indexOf("\n") + 1); - const close = rest.search(/^---[ \t]*\r?$/m); - if (close === -1) return { body: text }; - const frontmatter = rest.slice(0, close).replace(/\r?\n$/, ""); - const after = rest.slice(close); - const newline = after.indexOf("\n"); - if (newline === -1) return { frontmatter, body: "" }; - // Drop the blank line conventionally left between the fence and the body, so - // the document does not open with dead space. - return { frontmatter, body: after.slice(newline + 1).replace(/^\r?\n/, "") }; -} diff --git a/clients/web/tsup.runner.config.ts b/clients/web/tsup.runner.config.ts index 0b354d4222..49a7a7c44c 100644 --- a/clients/web/tsup.runner.config.ts +++ b/clients/web/tsup.runner.config.ts @@ -57,6 +57,13 @@ export default defineConfig({ // reached by a manifest edit rather than an omission. "ajv", "zod", + // Newly on `core/`'s runtime import graph as of #2248: + // `core/mcp/skillFile.ts` parses a served SKILL.md's YAML frontmatter to + // check it against the entry the listing advertised (SEP-2640). Already a + // root `dependency` — it was reached from `test-servers/src` — so this + // adds no package, but a root-declared dependency `core/` imports must be + // named in all three `external` lists or tsup inlines it here. + "yaml", // Reached through `core/` but not through this client's own code today. // AGENTS.md requires every root-declared package `core/` imports at runtime // in ALL three lists regardless, because which client reaches one is a diff --git a/core/mcp/inspectorClient.ts b/core/mcp/inspectorClient.ts index 4b9a7313cf..3eea39fb54 100644 --- a/core/mcp/inspectorClient.ts +++ b/core/mcp/inspectorClient.ts @@ -145,9 +145,14 @@ import { } from "./modernTaskSchemas.js"; import { buildClientExtensions } from "./extensions.js"; import { + DirectoryReadResultSchema, GetSkillResultSchema, ListSkillsResultSchema, + ModernDirectoryReadResultSchema, ModernListSkillsResultSchema, + RESOURCES_DIRECTORY_READ_METHOD, + SKILLS_EXTENSION_KEY, + type DirectoryReadResult, SKILLS_GET_METHOD, SKILLS_LIST_METHOD, type SkillEntry, @@ -5648,6 +5653,77 @@ export class InspectorClient extends InspectorClientEventTarget { } } + /** + * One page of `resources/directory/read` (SEP-2640): the direct children of + * a directory resource. + * + * **Gated on the server's own declaration, and the gate throws rather than + * asks.** SEP-2640 is explicit that *"clients MUST NOT call + * `resources/directory/read` against a server that has not declared + * `directoryRead: true`"*, so this refuses locally instead of sending a call + * the spec forbids and letting the server answer `-32601`. Refusing here also + * keeps the Protocol tab honest: a request we were never allowed to make + * should not appear in the exchange log as a server-side failure. + * + * Not recursive — the SEP says clients descend by calling the method again on + * a child directory, so the walking is the caller's, not this method's. + */ + async readResourceDirectory( + uri: string, + cursor?: string, + metadata?: RequestMetadata, + ): Promise { + if (!this.client) { + throw new Error("Client is not connected"); + } + const extension = this.getSkillsExtension(); + if (!extension?.directoryRead) { + throw new Error( + `Server did not declare directoryRead in ${SKILLS_EXTENSION_KEY}; ${RESOURCES_DIRECTORY_READ_METHOD} must not be called.`, + ); + } + const effectiveMeta = this.mergeMeta(metadata); + const params: Record = { + uri, + ...(effectiveMeta ? { _meta: effectiveMeta } : {}), + // `!== undefined` for the same reason `listSkills` uses it: a cursor is + // opaque and `""` is a legal value, so truthiness would silently re-ask + // for page one. + ...(cursor !== undefined ? { cursor } : {}), + }; + // Era-aware for the same reason `skills/list` is — the method is + // consumer-owned, so no SDK codec stamps or checks its envelope. The modern + // variant requires only `resultType`; see the schema for why it stops + // short of the caching attributes that `skills/list` requires. + const resultSchema = this.isModernEra() + ? ModernDirectoryReadResultSchema + : DirectoryReadResultSchema; + try { + return await this.invokeMcpClient( + () => + this.client!.request( + { method: RESOURCES_DIRECTORY_READ_METHOD, params }, + resultSchema, + this.getRequestOptions(this.progressTokenOf(metadata)), + ), + { method: RESOURCES_DIRECTORY_READ_METHOD }, + ); + } catch (err) { + // Same attribution `getSkill` does, and for the same reason: there is no + // managed store behind this method, so without this a rejected decode + // would render in the Protocol tab as a clean success while the caller + // showed an error. Only for a decode rejection — a request that never + // produced a response would otherwise stamp an earlier exchange. + if (isClientDecodeRejection(err)) { + this.markResponseRejected( + RESOURCES_DIRECTORY_READ_METHOD, + err instanceof Error ? err.message : String(err), + ); + } + throw err; + } + } + /** * Get a prompt by name * @param name Prompt name diff --git a/core/mcp/inspectorClientProtocol.ts b/core/mcp/inspectorClientProtocol.ts index 675bf6776a..2da4c4e617 100644 --- a/core/mcp/inspectorClientProtocol.ts +++ b/core/mcp/inspectorClientProtocol.ts @@ -42,6 +42,7 @@ import type { MalformedListItem } from "./listSalvage.js"; import type { InspectorClientEventTarget } from "./inspectorClientEventTarget.js"; import type { SkillEntry } from "./skillsSchemas.js"; import type { SkillsExtensionSupport } from "./skills.js"; +import type { DirectoryReadResult } from "./skillsSchemas.js"; import type { SamplingCreateMessage } from "./samplingCreateMessage.js"; import type { ElicitationCreateMessage } from "./elicitationCreateMessage.js"; @@ -122,6 +123,19 @@ export interface InspectorClientProtocol extends InspectorClientEventTarget { ): Promise<{ skills: SkillEntry[]; nextCursor?: string }>; /** One skill entry by URI (`skills/get`). */ getSkill(uri: string, metadata?: RequestMetadata): Promise; + /** + * One page of `resources/directory/read` — the direct children of a directory + * resource (#2248). Optional on this interface, unlike the two methods above: + * declaring the extension commits a server to `skills/list` and `skills/get`, + * while this one is separately gated on `directoryRead`, so a caller has to + * check for it anyway and the many test doubles that satisfy this interface + * should not all have to grow a method most of them never reach. + */ + readResourceDirectory?( + uri: string, + cursor?: string, + metadata?: RequestMetadata, + ): Promise; /** * Mark the response that most recently answered `method` as rejected by the diff --git a/core/mcp/skillFile.ts b/core/mcp/skillFile.ts new file mode 100644 index 0000000000..c49b7fe023 --- /dev/null +++ b/core/mcp/skillFile.ts @@ -0,0 +1,112 @@ +/** + * The `SKILL.md` file format: splitting a served file into its YAML + * frontmatter and body, and parsing that frontmatter into comparable JSON. + * + * Separate from `skills.ts` so the dependency runs one way. `skills.ts` owns the + * *checks* and needs both halves of this module; this module knows nothing about + * findings, severities, or `SkillEntry`, which is what lets it stay a pure text + * transform with a single import. + * + * ⚠️ **This is the one place in `core/` that imports a YAML parser**, and it is + * imported deliberately rather than hand-rolled. SEP-2640 requires a host to + * *parse* a `SKILL.md`'s frontmatter and compare it field by field against the + * entry's — see `checkSkillFrontmatterMatch` — and a regex approximation of YAML + * would report a conforming server as broken the first time a description + * carried a colon, a quoted string, or a multi-line block scalar. For a tool + * whose entire output is "does this server conform", a checker that is itself + * wrong is worse than no checker. `yaml` was already a repo-root **dependency** + * (reached from `test-servers/src/load-config.ts`), so this adds no new package + * to any manifest — but it does newly put `yaml` on `core/`'s runtime import + * graph, which is why it joins the three bundler `external` lists in the same + * change (see the dependency-placement rules in AGENTS.md). + */ + +import { parse as parseYaml } from "yaml"; + +export interface SkillFileParts { + /** + * The raw YAML between the fences, fences excluded — `undefined` when the + * file has no frontmatter at all. Kept raw as well as parsed because the two + * answer different questions: the parsed form is what the conformance check + * compares, and the bytes are what a reader needs to see when the two + * disagree. + */ + frontmatter?: string; + /** Everything after the closing fence, or the whole file when there is none. */ + body: string; +} + +/** + * Separate a leading YAML frontmatter fence from the rest of a skill file. + * + * The Skills screen renders the two halves in different places — the + * frontmatter in its own collapsible section, the body in the file viewer — and + * deriving both from **one** split is what stops them disagreeing: the section + * can never show one file's frontmatter while the viewer shows another's, and a + * file with no frontmatter cannot leave a stale section on screen. + * + * It also matters for rendering: the markdown renderer has no frontmatter + * support, so an un-split `---\nname: …\n---` is read as a setext heading and + * painted as a title above the document's real one. + * + * Two deliberate conservatisms, because this must never eat content: + * + * - Only a fence at the very **start** of the file counts. A `---` anywhere + * else is a horizontal rule and is left in the body. + * - A file that opens with `---` but never closes the fence is **not** + * frontmatter; the whole file is returned as the body rather than being + * truncated to nothing. + */ +export function splitSkillFile(text: string): SkillFileParts { + if (!/^---[ \t]*\r?\n/.test(text)) return { body: text }; + const rest = text.slice(text.indexOf("\n") + 1); + const close = rest.search(/^---[ \t]*\r?$/m); + if (close === -1) return { body: text }; + const frontmatter = rest.slice(0, close).replace(/\r?\n$/, ""); + const after = rest.slice(close); + const newline = after.indexOf("\n"); + if (newline === -1) return { frontmatter, body: "" }; + // Drop the blank line conventionally left between the fence and the body, so + // the document does not open with dead space. + return { frontmatter, body: after.slice(newline + 1).replace(/^\r?\n/, "") }; +} + +/** Outcome of parsing a frontmatter block. Exactly one member is set. */ +export type ParsedFrontmatter = + | { fields: Record } + | { error: string }; + +/** + * Parse a frontmatter block into a field map. + * + * Two non-obvious decisions: + * + * - **YAML 1.2 core schema**, which is `yaml`'s default and is what the Agent + * Skills format assumes. It resolves only JSON's own types, so a timestamp + * stays the string the server wrote rather than becoming a `Date` — which + * matters because the other side of the comparison arrived over JSON-RPC and + * can only ever hold JSON types. Under YAML 1.1 the two would differ for a + * date-shaped value that is in fact identical on the wire. + * - **A non-mapping is an error, not an empty map.** `---\njust a string\n---` + * parses successfully as the scalar `"just a string"`, and reporting that as + * "no fields" would present a malformed file as one that merely omitted + * everything. An *empty* block (`fields: {}`) is a different fact and is + * reported as a successful parse of nothing. + */ +export function parseSkillFrontmatter(yamlText: string): ParsedFrontmatter { + let parsed: unknown; + try { + parsed = parseYaml(yamlText); + } catch (err) { + return { error: err instanceof Error ? err.message : String(err) }; + } + // `null` is what an empty (or comment-only) block parses to — a real, if + // degenerate, mapping of no fields rather than a malformed one. + if (parsed === null || parsed === undefined) return { fields: {} }; + if (typeof parsed !== "object" || Array.isArray(parsed)) { + return { + error: "Frontmatter is not a YAML mapping of fields.", + }; + } + return { fields: parsed as Record }; +} diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index 5d4883744e..30f785b87d 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -23,15 +23,15 @@ * machinery (activation, per-skill consent, content-bound approval) is * implemented here. Surface and verify. * - * ⚠️ **One SEP-2640 obligation is deliberately NOT checked here: that an entry's - * `frontmatter` matches the fetched `SKILL.md`'s frontmatter field by field.** - * The digest check does not cover it — a digest is taken over the bytes the - * server served, so it proves the file was not tampered with in transit and - * says nothing about whether the *listing* described that file honestly. A - * server can therefore advertise one description, serve a different one, and - * pass every check in this module. Closing it needs a YAML parser, which is a - * new runtime dependency and a placement decision of its own, so it is tracked - * on #2248 rather than half-done here. + * **The frontmatter cross-check closes the gap the digest cannot** (#2248). + * SEP-2640 requires that an entry's `frontmatter` match the fetched `SKILL.md`'s + * frontmatter field by field, and no digest can establish that: a digest is + * taken over the bytes the server served, so it proves the file was not altered + * in transit and says nothing about whether the *listing* described that file + * honestly. A server could advertise one description, serve another, and pass + * every other check in this module. {@link checkSkillFrontmatterMatch} is the + * check; it needs a real YAML parser, and `core/mcp/skillFile.ts` explains why + * that dependency is imported rather than approximated. */ import type { ServerCapabilities } from "@modelcontextprotocol/client"; @@ -42,6 +42,7 @@ import { type SkillResource, } from "./skillsSchemas.js"; import { sha256Bytes } from "./sha256.js"; +import { parseSkillFrontmatter, splitSkillFile } from "./skillFile.js"; /** Maximum resource entries a single skill may declare (SEP-2640). */ export const SKILL_MAX_RESOURCE_ENTRIES = 512; @@ -243,7 +244,10 @@ export type SkillIssueCode = | "resource-outside-skill-root" | "manifest-missing-self" | "resource-limit-exceeded" - | "size-limit-exceeded"; + | "size-limit-exceeded" + | "frontmatter-absent" + | "frontmatter-unparsable" + | "frontmatter-mismatch"; /** * `error` marks a **MUST** of SEP-2640 that the server broke, so a manifest @@ -646,6 +650,32 @@ export function base64ToBytes(blob: string): Uint8Array { return bytes; } +/** + * The content a `resources/read` returned for one skill file. Either `text` (a + * `TextResourceContents`) or `blob` (base64, a `BlobResourceContents`). + */ +export interface SkillFileContents { + text?: string; + blob?: string; + mimeType?: string; +} + +/** + * The raw bytes of a skill file, as fetched — the bytes its digest was taken + * over. + * + * Throws for a result carrying neither `text` nor `blob`. That is a server bug, + * and it must not be quietly treated as empty content: an empty `Uint8Array` + * has a perfectly good SHA-256, so a silent fallback would report a *digest + * mismatch* — a confident, wrong diagnosis — instead of "this response carried + * no content at all". Callers surface the throw as a per-file read failure. + */ +export function skillFileBytes(contents: SkillFileContents): Uint8Array { + if (typeof contents.text === "string") return textToBytes(contents.text); + if (typeof contents.blob === "string") return base64ToBytes(contents.blob); + throw new Error("resources/read returned neither text nor blob content."); +} + /** * Verify one fetched skill file against its manifest entry. * @@ -704,3 +734,105 @@ export async function verifySkillResource( : {}), }; } + +/** + * Compare the fetched `SKILL.md`'s own frontmatter against the frontmatter the + * entry advertised, field by field — the SEP-2640 obligation a digest cannot + * discharge (#2248). + * + * SEP-2640: *"hosts MUST parse its YAML frontmatter and compare it + * field-by-field against the entry's `frontmatter`. Any discrepancy MUST be + * treated as a verification failure equivalent to a digest mismatch"*. So every + * finding here is an `error`, matching what a digest mismatch reports — the + * spec makes them equivalent and the report must not rank one below the other. + * + * **One finding per differing field, not one per file.** "Frontmatter does not + * match" is unactionable for the server author who has to fix it; "listing says + * `description: A`, file says `description: B`" is the whole diagnosis. The + * union of both sides' keys is walked, so a field present on only one side is + * reported as such rather than silently skipped. + * + * ⚠️ **Only call this with the bytes of the entry's own `SKILL.md`.** The check + * is meaningless against a supporting file, which has no frontmatter to match, + * and would report every one of them as `frontmatter-absent`. Callers select + * the file; this function cannot tell which one it was handed. + * + * Values are compared as **canonical JSON**, so a frontmatter field holding a + * nested mapping compares equal when the two sides agree on content and differ + * only in key order — which is not a discrepancy in either JSON or YAML. Array + * order *is* significant and is preserved, because a YAML sequence is ordered. + */ +export function checkSkillFrontmatterMatch( + entry: SkillEntry, + skillFileText: string, +): SkillIssue[] { + const { frontmatter } = splitSkillFile(skillFileText); + if (frontmatter === undefined) { + return [ + { + code: "frontmatter-absent", + severity: "error", + message: + "The served SKILL.md carries no YAML frontmatter block, so the listing's frontmatter cannot be the file's.", + resourceUri: entry.uri, + }, + ]; + } + const parsed = parseSkillFrontmatter(frontmatter); + if ("error" in parsed) { + return [ + { + code: "frontmatter-unparsable", + severity: "error", + message: `The served SKILL.md's frontmatter is not valid YAML: ${parsed.error}`, + resourceUri: entry.uri, + }, + ]; + } + const issues: SkillIssue[] = []; + // Sorted so the report is stable across runs — `Object.keys` order follows + // insertion, which is the wire order on one side and the file order on the + // other, and those need not agree even when the content does. + const fields = [ + ...new Set([ + ...Object.keys(entry.frontmatter), + ...Object.keys(parsed.fields), + ]), + ].sort(); + for (const field of fields) { + const listed = entry.frontmatter[field]; + const served = parsed.fields[field]; + // `undefined` is the only way "absent" reaches here: JSON has no undefined + // value, and a YAML key written with an empty value parses to `null`, which + // is a present field holding null and compares as one. + if (listed === undefined) { + issues.push({ + code: "frontmatter-mismatch", + severity: "error", + message: `The served SKILL.md declares "${field}" but the listing's frontmatter omits it.`, + resourceUri: entry.uri, + }); + continue; + } + if (served === undefined) { + issues.push({ + code: "frontmatter-mismatch", + severity: "error", + message: `The listing declares "${field}" but the served SKILL.md omits it.`, + resourceUri: entry.uri, + }); + continue; + } + const listedJson = JSON.stringify(canonicalize(listed)); + const servedJson = JSON.stringify(canonicalize(served)); + if (listedJson !== servedJson) { + issues.push({ + code: "frontmatter-mismatch", + severity: "error", + message: `Field "${field}" differs: the listing says ${listedJson} but the served SKILL.md says ${servedJson}.`, + resourceUri: entry.uri, + }); + } + } + return issues; +} diff --git a/core/mcp/skillsSchemas.ts b/core/mcp/skillsSchemas.ts index bb3a1160dd..e13a6fcce5 100644 --- a/core/mcp/skillsSchemas.ts +++ b/core/mcp/skillsSchemas.ts @@ -29,6 +29,7 @@ */ import { z } from "zod/v4"; +import { ResourceSchema } from "@modelcontextprotocol/core"; /** SEP-2133 extension identifier for the Skills extension (SEP-2640). */ export const SKILLS_EXTENSION_KEY = "io.modelcontextprotocol/skills"; @@ -143,6 +144,22 @@ export const ModernListSkillsResultSchema = ListSkillsResultSchema.extend({ /** * The `skills/get` result envelope: the entry wrapped under `skill`. * + * ⚠️ **Not era-aware, and that is settled rather than pending (#2248).** The + * obvious symmetry would be a modern variant requiring the caching attributes + * the way {@link ModernListSkillsResultSchema} does. SEP-2640 forecloses it in + * as many words, under `skills/get`: *"whether the result should also carry the + * base protocol's caching attributes (`ttlMs` and `cacheScope`, per SEP-2549), + * as `resources/read` results do, is **left open**"*. + * + * So there is no requirement to enforce, and inventing one would do real harm + * rather than none: a server that reasonably reads "left open" as "not + * required" would be reported as non-conforming by the tool whose job is to + * tell it whether it conforms. `looseObject` means a server that *does* send + * them still parses, which is the right handling for an attribute the spec + * permits and does not mandate. Revisit only if a later revision closes the + * question — and note that the same sentence is why `skills/get` carries no + * `nextCursor` handling either: "a single entry is not a list". + * * Required, not one of two accepted shapes. An earlier revision of this module * also accepted a bare entry at the top level, on the reading that the SEP * settled the entry but not its wrapper. It does settle the wrapper, and @@ -165,13 +182,70 @@ export const GetSkillResultSchema = GetSkillEnvelopeSchema.transform( export type GetSkillResult = SkillEntry; /** - * ⚠️ **No `resources/directory/read` result schema here yet, on purpose.** - * - * The method name and the directory MIME type above are stated in SEP-2640; - * the shape of the result it returns is not something this PR verified against - * the normative text, and the Inspector does not call the method (phase 3, - * #2248). Declaring a guessed schema would put an unverified claim in the one - * module that is supposed to be the authority on the wire format — and one - * nothing exercises, so it could be wrong indefinitely without failing - * anything. Phase 3 adds it against the spec, alongside the call that uses it. + * One `resources/directory/read` child: **the SDK's own `Resource`**, not a + * shape restated here. + * + * SEP-2640 defines the result as carrying "the same `Resource` objects that + * `resources/list` returns, with the same `nextCursor` pagination contract", so + * the schema that already decodes `resources/list` in this app is the literal + * statement of that sentence — and one the SDK, not this module, keeps current. + * Restating it would let the two drift, at which point a directory child and a + * listed resource could disagree about what a `Resource` is while both claimed + * to be one. + * + * ⚠️ It is **stricter than everything else in this module**, and that is the + * deliberate exception rather than an oversight. `ResourceSchema` requires + * `name` and strips unknown members, so one child missing `name` rejects the + * whole page instead of being reported as a per-child finding — the opposite of + * the posture the entry schemas above take. The reason the trade goes the other + * way here is ownership: the skills entry types are consumer-owned and nothing + * else validates them, so this module has to be the reporter; `Resource` is + * base-protocol and already validated exactly this strictly on the + * `resources/list` path, where {@link listSalvage} is the answer to a single bad + * entry. Being *more* permissive here would mean a URI that fails as a listed + * resource succeeds as a directory child, which is a worse inconsistency than + * an all-or-nothing page. + */ +export const DirectoryChildSchema = ResourceSchema; + +/** + * `resources/directory/read` result on a **legacy** connection: the directory's + * direct children plus the opaque cursor. + * + * `looseObject`, matching every other result schema here: a server that also + * sends the caching attributes is not wrong for doing so, and a schema is not + * the place to reject an extra member. */ +export const DirectoryReadResultSchema = z.looseObject({ + resources: z.array(DirectoryChildSchema), + nextCursor: z.string().optional(), +}); + +export type DirectoryReadResult = z.infer; + +/** + * `resources/directory/read` result on a **modern** (2026-07-28+) connection: + * the page plus `resultType`, and deliberately **not** `ttlMs` / `cacheScope`. + * + * That asymmetry with {@link ModernListSkillsResultSchema} is the one judgement + * call in this module, so it is written down rather than left to be re-derived: + * + * - For `skills/list` the SEP states the requirement outright — *"In protocol + * versions 2026-07-28 and later, the result also carries … `ttlMs` and + * `cacheScope`"* — so requiring them is quoting the spec. + * - For `resources/directory/read` it states **nothing of the kind**, and its + * one worked example of the result carries `resultType: "complete"` and no + * caching attributes at all. Requiring them here would fail a server that + * matched the SEP's own example, which is the failure direction this module + * works hardest to avoid. + * + * `resultType` is required because it is the base protocol's, not this + * extension's: SEP-2322 makes it a member of every modern result, the SEP's + * example carries it, and `skills/*` being consumer-owned means the SDK codec + * validates none of it — so if this schema does not, nothing does. + * + * ⚠️ Picked by `InspectorClient.readResourceDirectory` from the negotiated era. + */ +export const ModernDirectoryReadResultSchema = DirectoryReadResultSchema.extend( + { resultType: z.literal("complete") }, +); diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts new file mode 100644 index 0000000000..0ac9423f78 --- /dev/null +++ b/core/mcp/skillsVerification.ts @@ -0,0 +1,221 @@ +/** + * Fetch-and-verify: run every SEP-2640 check that needs the *bytes* of a skill's + * files, over a whole set of entries (#2248). + * + * Separate from `skills.ts`, which is deliberately I/O-free — + * `checkSkillConformance` reads a manifest, `verifySkillResource` compares bytes + * it is handed, `checkSkillFrontmatterMatch` compares text it is handed, and + * none of them knows how to obtain a file. This module is the part that does, + * and keeping it in its own file is what lets the pure checks stay testable with + * no client at all. + * + * It lives in `core/` because **two** clients drive it: the CLI's `--verify` + * turns it into an NDJSON report with an exit code, and the TUI's Skills pane + * runs it for one selected skill. Only the web screen does something different + * — it fetches lazily, per click, because a browser user is reading one file at + * a time rather than asking a yes/no question about a catalog. That difference + * is about *when* to fetch, not *how* to check, so it does not belong here. + * + * ⚠️ Reads are **sequential, deliberately.** A conforming manifest may declare + * 512 entries, and a parallel walk over one would open 512 `resources/read` + * calls against a server whose whole purpose here is to be tested — the hazard + * the web screen bounds with a concurrency limit. Sequential also makes the + * report deterministic: entries come back in manifest order on every run, so a + * CI diff of two reports shows what changed rather than what raced. + */ + +import { AuthRecoveryRequiredError } from "../auth/challenge.js"; +import type { InspectorClientProtocol } from "./inspectorClientProtocol.js"; +import type { RequestMetadata } from "./types.js"; +import { + checkSkillConformance, + checkSkillFrontmatterMatch, + skillDisplayName, + skillFileBytes, + verifySkillResource, + type SkillIssue, + type SkillVerification, +} from "./skills.js"; +import { DYNAMIC_RESOURCES, type SkillEntry } from "./skillsSchemas.js"; + +/** One manifest entry's outcome. `read-error` means the fetch itself failed. */ +export type SkillFileStatus = SkillVerification["status"] | "read-error"; + +export interface SkillFileReport { + uri: string; + status: SkillFileStatus; + expectedDigest?: string; + actualDigest?: string; + expectedSize?: number; + actualSize?: number; + reason?: string; +} + +/** One skill's full verdict, and the unit of the NDJSON stream. */ +export interface SkillVerifyReport { + uri: string; + name: string; + /** Structural findings against the entry as listed. */ + conformance: SkillIssue[]; + /** + * Findings from comparing the served `SKILL.md`'s own frontmatter against the + * listed one. Empty when the file could not be read — the read failure is + * reported once, as a file result, rather than a second time as a phantom + * frontmatter discrepancy. + */ + frontmatter: SkillIssue[]; + /** One entry per manifest file, in manifest order. Empty for `"dynamic"`. */ + files: SkillFileReport[]; + /** + * False when anything the SEP makes a MUST was broken: an error-severity + * finding, a digest or size mismatch, or a file that could not be read. + * + * A `warning` does **not** clear it — a `"dynamic"` manifest is legal, and a + * report that failed CI for it would be telling server authors their + * conforming skill is broken. + */ + ok: boolean; +} + +/** Result shape of one `resources/read`, narrowed to what a digest needs. */ +interface ReadContents { + text?: string; + blob?: string; + mimeType?: string; +} + +/** + * The first content block of a `resources/read` result. + * + * `contents[0]` rather than a search by URI: a server may legitimately answer + * with a canonicalized spelling of the URI we asked for, and matching on the + * string would reject it. A result with no blocks is a read failure and is + * reported as one. + */ +function firstContents(result: unknown): ReadContents | undefined { + const contents = (result as { contents?: unknown })?.contents; + if (!Array.isArray(contents) || contents.length === 0) return undefined; + const first: unknown = contents[0]; + if (typeof first !== "object" || first === null) return undefined; + return first as ReadContents; +} + +/** + * Verify every skill in `entries` against the connected server. + * + * Never throws for a single skill or a single file: a report that aborted on + * the first unreadable file would hide every finding after it, and finding + * everything wrong in one pass is the entire value of running this in CI. + * + * ⚠️ **`AuthRecoveryRequiredError` is the deliberate exception and is re-thrown.** + * It is not a property of the file that happened to be in flight — it says the + * session's authorization expired, so every remaining read would fail the same + * way. Recording it per file would produce a report of N identical read + * failures and, worse, would swallow the one error a caller keys off to start a + * reauthorization: the TUI pane hands it to its recovery callback and the web + * commands retry after it. Absorbed here, the user is simply told the files + * could not be read, with no way offered to fix it. + */ +export async function verifySkills( + client: InspectorClientProtocol, + entries: readonly SkillEntry[], + metadata?: RequestMetadata, +): Promise { + const reports: SkillVerifyReport[] = []; + for (const entry of entries) { + // The entry's own SKILL.md, read once and used twice — for its digest and + // for the frontmatter cross-check. Reading it twice would double the load + // on the server and, worse, could compare a digest against one snapshot + // and frontmatter against another. + let entryText: string | undefined; + const files: SkillFileReport[] = []; + + const manifest = + entry.resources === DYNAMIC_RESOURCES ? [] : entry.resources; + for (const resource of manifest) { + let contents: ReadContents | undefined; + try { + const invocation = await client.readResource(resource.uri, metadata); + contents = firstContents(invocation.result); + } catch (err) { + if (err instanceof AuthRecoveryRequiredError) throw err; + files.push({ + uri: resource.uri, + status: "read-error", + reason: err instanceof Error ? err.message : String(err), + }); + continue; + } + if (!contents) { + files.push({ + uri: resource.uri, + status: "read-error", + reason: "resources/read returned no content blocks.", + }); + continue; + } + if (resource.uri === entry.uri && typeof contents.text === "string") { + entryText = contents.text; + } + let bytes: Uint8Array; + try { + bytes = skillFileBytes(contents); + } catch (err) { + files.push({ + uri: resource.uri, + status: "read-error", + reason: err instanceof Error ? err.message : String(err), + }); + continue; + } + const verification = await verifySkillResource(resource, bytes); + files.push({ uri: resource.uri, ...verification }); + } + + // A `"dynamic"` skill has no manifest, so the loop above read nothing — + // but its SKILL.md is still served and still has to match the frontmatter + // the listing advertised. That obligation is not waived by the file set + // being unenumerable; only integrity is. + if (entryText === undefined) { + try { + const invocation = await client.readResource(entry.uri, metadata); + const contents = firstContents(invocation.result); + if (typeof contents?.text === "string") entryText = contents.text; + } catch (err) { + // Left undefined: the frontmatter check is skipped below. When the + // manifest listed this file the failure is already reported there, and + // when it did not, `manifest-missing-self` is the finding that matters. + // An expired authorization is not that case — see the note above. + if (err instanceof AuthRecoveryRequiredError) throw err; + } + } + + const conformance = checkSkillConformance(entry); + const frontmatter = + entryText === undefined + ? [] + : checkSkillFrontmatterMatch(entry, entryText); + const hasError = [...conformance, ...frontmatter].some( + (issue) => issue.severity === "error", + ); + const fileFailed = files.some( + (file) => file.status === "mismatch" || file.status === "read-error", + ); + reports.push({ + uri: entry.uri, + name: skillDisplayName(entry), + conformance, + frontmatter, + files, + ok: !hasError && !fileFailed, + }); + } + return reports; +} + +/** True when every skill in the report passed. */ +export function allSkillsVerified( + reports: readonly SkillVerifyReport[], +): boolean { + return reports.every((report) => report.ok); +} diff --git a/core/mcp/state/managedSkillsState.ts b/core/mcp/state/managedSkillsState.ts index 0429e821d5..c0d75bf8a2 100644 --- a/core/mcp/state/managedSkillsState.ts +++ b/core/mcp/state/managedSkillsState.ts @@ -16,6 +16,33 @@ * reason the request is a plain `client.request`: the SDK has no high-level verb * for a consumer-owned extension method, so there is no cache-aware wrapper to * delegate to and no `cacheMode` to honor. + * + * ⚠️ **There is deliberately no `PagedSkillsState`, so the `paginatedLists` + * server setting does not apply to Skills (#2248).** Tools, prompts and + * resources each have a paged counterpart that setting switches them to; skills + * does not, and the reason is not that the walk is cheap. + * + * It is that **every consumer of this list is a whole-catalog verdict.** The + * Skills screen's conformance summary, and the CLI's `--verify` exit code, are + * statements about the catalog: "this server's skills conform". Computed over + * page one of three, that statement is *wrong* — it reports a clean catalog + * while the tampered digest sits on page three, and reports it with exactly the + * confidence of a real pass. Paging the other lists costs a reader some rows; + * paging this one would make the tool's own output untrue. The setting exists + * to let a user watch a server's pagination work, and this list's page count is + * surfaced instead (`getPagination`, rendered by both the web screen and the + * TUI pane), which serves that purpose without staking a verdict on a partial + * read. + * + * The cost argument, which is the one #2248 asked about, points the same way + * and is secondary: SEP-2640 makes a listing entry a *complete* manifest — + * verbatim frontmatter and the full `resources` set with digests — precisely so + * that "a host that pages through the listing has, in that one pass, everything + * it needs … there is no second round-trip per skill". A full walk is the + * access pattern the wire format was designed for. Revisit if a real server + * turns up whose catalog makes the walk painful; the guards this walk already + * carries (`SKILLS_MAX_PAGES`, the repeated-cursor check) are what bound it + * until then. */ import type { InspectorClientProtocol } from "../inspectorClientProtocol.js"; diff --git a/docs/test-servers.md b/docs/test-servers.md index 521435ec55..71c1dacd19 100644 --- a/docs/test-servers.md +++ b/docs/test-servers.md @@ -60,17 +60,21 @@ as a missing capability rather than an error. | `subscriptions-never-acknowledged-http.json` **(modern era)** | A `subscriptions/listen` answered with a bare result | [#2097](https://github.com/modelcontextprotocol/inspector/issues/2097) | | `tasks-{legacy,modern}-http.json` **(era per file)** | Tasks, both eras | [#1631](https://github.com/modelcontextprotocol/inspector/issues/1631) | | `cancellation-modern-http.json` **(modern era)** | Cancelling a call by closing its response stream | [#2140](https://github.com/modelcontextprotocol/inspector/issues/2140) | -| `skills-http.json` **(either era)** | Skills tab: `skills/list`, digest verification, and the non-conforming cases | [#2234](https://github.com/modelcontextprotocol/inspector/issues/2234) | +| `skills-http.json` **(either era)** | Skills tab: `skills/list`, `resources/directory/read`, digest verification, the frontmatter cross-check, and the non-conforming cases | [#2234](https://github.com/modelcontextprotocol/inspector/issues/2234), [#2248](https://github.com/modelcontextprotocol/inspector/issues/2248) | ## Skills (SEP-2640) -`skills-http.json` sets `"skills": true` and serves four skills over two -`skills/list` pages. The extension is advertised **bare**: there is deliberately -no `directoryRead` option to turn on, because nothing here serves -`resources/directory/read` and a config that advertised it would produce exactly -the false capability this fixture helps catch — Connection Info reporting a -sub-option supported while the method answers `-32601`. Both come back in -phase 3 ([#2248](https://github.com/modelcontextprotocol/inspector/issues/2248)). +`skills-http.json` sets `"skills": true` and serves six skills over three +`skills/list` pages. Since +[#2248](https://github.com/modelcontextprotocol/inspector/issues/2248) that one +flag also declares **`directoryRead: true`** and registers the +`resources/directory/read` handler. The declaration and the handler are one +switch on purpose: the sub-flag's whole hazard is advertising a method nothing +answers — Connection Info reporting a sub-option "Supported" while the method +returns `-32601` — and a config that cannot express the declaration without the +handler cannot reach it. To exercise the *undeclared* case, connect to any +config **without** `"skills"`, where the Inspector must refuse to send the call +locally rather than letting the server answer it. Every result carries the modern base envelope (`resultType` / `ttlMs` / `cacheScope`). `skills/*` are consumer-owned, so the SDK stamps nothing for @@ -84,7 +88,7 @@ It works on **either era**: `skills/list`, `skills/get` and era codec defines, so the SDK's era gate skips them entirely — which is why this fixture, unlike the tasks ones, needs no per-era variant. -Three of the four skills are deliberately awkward, because the checks the Skills +Four of the six skills are deliberately awkward, because the checks the Skills tab runs are untestable without them. Only two are actual violations — the `"dynamic"` form is **conforming**, and is here because "legal but unverifiable" is the case most easily buried: @@ -95,12 +99,44 @@ is the case most easily buried: | `tampered-notes` | An advertised digest that does not match the bytes served, so verification reports a **digest mismatch** with both digests shown. | | `dynamic-report` | `resources: "dynamic"` — a **legal** form for generated content. No manifest is advertised, so integrity cannot be verified at all; reported as a warning, not an error. | | `wrong-folder` | A URI path segment (`wrong-folder`) that disagrees with `frontmatter.name` (`right-name`), the one structural invariant SEP-2640 states outright. | +| `stale-manifest` | A skill that **serves and directory-lists a file its `resources` manifest does not declare**. Its entry is otherwise fully conforming and verifies clean, so the disagreement between the two views is the only defect — and only a directory read can see it. SEP-2640 calls a directory result "a live observation" and says hosts MUST NOT treat it as extending the manifest, so the Directory section marks the extra child **not listed** rather than showing it as one of the skill's files. | +| `lying-listing` | A `skills/list` entry advertising one `description` while the served `SKILL.md` carries another. **Its digest verifies** — a digest is taken over the bytes the server served and says nothing about whether the listing described them honestly — so this is the one violation only the frontmatter cross-check can catch. | Connection Info's **Skills Extension Options** section shows the `directoryRead` -sub-flag — against this fixture, a red ✗. The Inspector surfaces the flag but -does not call `resources/directory/read` yet, and the wire schema for that -result is deliberately absent from `core/mcp/skillsSchemas.ts` too: phase 3 adds -it against the normative text rather than shipping a guess nothing exercises. +sub-flag — against this fixture, a green ✓. The Skills screen then renders a +**Directory** section for the selected skill: press *Read directory* to list the +skill root's children, click a directory row to descend, *Up* to come back, and +*Load more* to page. Pages are one child each here, so a client that ignores +`nextCursor` is visibly wrong rather than merely lucky. `dynamic-report` is the +case the method actually exists for — it advertises no manifest, so a directory +read is the only way its files are discoverable at all. + +From the CLI, the same catalog reports itself: + +```sh +mcp-inspector --cli --server-url http://127.0.0.1:3230/mcp --transport http \ + --method skills/list --verify +``` + +One JSON report per skill on stdout, a one-line summary on stderr, and exit **7** +when any skill fails — which it does here, on `tampered-notes` (digest), +`wrong-folder` (name) and `lying-listing` (frontmatter). The TUI's **Skills** +pane runs the same checks for one selected skill on Enter. + +### Why these five shapes + +They are the client-side obligations SEP-2640 makes testable from a hostile +server, which is how the +[`modelcontextprotocol/conformance`](https://github.com/modelcontextprotocol/conformance) +harness grades a *client*: it stands up a server and watches what the client +does. Four of its five skills scenarios map onto a fixture here — a digest +mismatch (`tampered-notes`), a size mismatch, a frontmatter mismatch +(`lying-listing`), and a read of a file the manifest does not list +(`stale-manifest`). The fifth, **no-prefetch**, is a negative: it passes only if +connecting and calling `skills/list` produces *no* `resources/read` at all. The +Inspector satisfies it structurally — nothing is fetched until a user selects a +skill or presses Verify, which is why every round trip on the Skills screen is a +button rather than an effect. ## Cancelling a call diff --git a/test-servers/src/composable-test-server.ts b/test-servers/src/composable-test-server.ts index 7123873fbf..10e895d91b 100644 --- a/test-servers/src/composable-test-server.ts +++ b/test-servers/src/composable-test-server.ts @@ -603,11 +603,15 @@ export interface ServerConfig { * `skills/get` plus the `skill://` files those entries name. The fixture set * deliberately includes non-conforming skills — see `skills.ts`. * - * There is deliberately **no `directoryRead` option**. The flag would - * advertise `resources/directory/read`, which nothing here serves, so a - * config could produce exactly the false capability this fixture exists to - * help catch — Connection Info reporting "supported" for a method that - * answers `-32601`. It comes back in phase 3 (#2248) with the handler. + * Turning this on also declares **`directoryRead: true`** and serves + * `resources/directory/read` over the same `skill://` tree (#2248). The two + * are one switch rather than two on purpose: the sub-flag's whole hazard is + * advertising a method nothing answers — Connection Info reporting + * "Supported" for a call that returns `-32601` — and a config that cannot + * express the declaration without the handler cannot reach it. A fixture for + * the *undeclared* case is still available and is the more useful one: + * any config without `skills` at all, against which the Inspector must + * refuse to send the call locally. */ skills?: boolean; /** @@ -878,13 +882,14 @@ export function createMcpServer(config: ServerConfig): McpServer { }; } - // Skills extension (SEP-2640): a server-declared extension, advertised bare. - // See `ServerConfig.skills` for why there is no `directoryRead` sub-option - // to turn on. + // Skills extension (SEP-2640): a server-declared extension. `directoryRead` + // is declared because `wireSkillsHandlers` registers the handler for it in + // the same `config.skills` branch below — see `ServerConfig.skills` for why + // the declaration and the handler are one switch. if (config.skills) { capabilities.extensions = { ...(capabilities.extensions ?? {}), - [SKILLS_EXTENSION_KEY]: {}, + [SKILLS_EXTENSION_KEY]: { directoryRead: true }, }; // Skill files are fetched through ordinary `resources/read`, so the // resources capability has to be advertised even when the config registers diff --git a/test-servers/src/load-config.ts b/test-servers/src/load-config.ts index 4f08e232ad..3b32862f05 100644 --- a/test-servers/src/load-config.ts +++ b/test-servers/src/load-config.ts @@ -67,8 +67,8 @@ export interface ConfigFile { * and wire its handlers + `modern_task` / `modern_input_task` tools. Pair with * `transport.modern`. */ tasksExtension?: boolean; - /** Advertise the Skills extension (SEP-2640) and serve its fixture skills. - * No `directoryRead` sub-option — see {@link ServerConfig.skills}. */ + /** Advertise the Skills extension (SEP-2640) and serve its fixture skills, + * including `directoryRead` — see {@link ServerConfig.skills}. */ skills?: boolean; /** Advertise the MCP Apps `io.modelcontextprotocol/ui` extension with the nested * `elicitation` setting — the server half of app-rendered form elicitation diff --git a/test-servers/src/skills.ts b/test-servers/src/skills.ts index c1d977e35c..82679e5355 100644 --- a/test-servers/src/skills.ts +++ b/test-servers/src/skills.ts @@ -1,9 +1,10 @@ /** * Skills extension test fixture — SEP-2640 (`io.modelcontextprotocol/skills`). * - * Serves `skills/list` (paginated) and `skills/get`, plus `resources/read` for - * the `skill://` URIs those entries name, so an Inspector connected here can - * exercise the whole flow: enumerate, fetch a file, and verify its digest. + * Serves `skills/list` (paginated), `skills/get` and + * `resources/directory/read`, plus `resources/read` for the `skill://` URIs + * those entries name, so an Inspector connected here can exercise the whole + * flow: enumerate, descend the tree, fetch a file, and verify its digest. * * **The awkward skills are the point.** A fixture that only served a clean * skill would leave every verification and conformance path in the Inspector @@ -19,6 +20,18 @@ * serves — a genuine violation. * - `wrong-folder` has a URI path segment that disagrees with * `frontmatter.name` — the other genuine violation. + * - `stale-manifest` serves — and directory-lists — a file its `resources` + * manifest does not declare. SEP-2640 calls a directory result "a live + * observation" and says hosts MUST NOT treat it as extending the manifest, + * so this is the fixture for that rule: the Inspector must show the extra + * child as *not listed* rather than as one of the skill's files (#2248). + * - `lying-listing` advertises one `description` in its `skills/list` entry + * and serves a different one in its `SKILL.md` — the violation no digest can + * catch, because the digest is over the bytes the server served and says + * nothing about whether the *listing* described them honestly (#2248). It is + * the only fixture whose `SKILL.md` is deliberately NOT derived from its + * listed frontmatter, and the exception is what makes the frontmatter + * cross-check demonstrable at all. * * `skills/list` and `skills/get` are registered through the **public** * `setRequestHandler`, which accepts a consumer-owned method name as long as @@ -100,6 +113,16 @@ interface FixtureSkill { frontmatter: Frontmatter; /** `"dynamic"` for a generated skill with no enumerable manifest. */ files: FixtureFile[] | "dynamic"; + /** + * Files this skill **serves and directory-lists but does NOT declare** in its + * manifest — the stale-snapshot case SEP-2640 governs, where a server has + * added a file since the entry was fetched. + * + * Deliberately excluded from `toEntry`, so the entry stays otherwise + * conforming: the only thing wrong is the disagreement between the two views, + * which is exactly what a consumer must not paper over. + */ + unlistedFiles?: FixtureFile[]; } interface Frontmatter { @@ -227,11 +250,39 @@ const MISMATCHED_FM: Frontmatter = { description: "A skill whose URI path segment disagrees with its frontmatter name", }; +const STALE_FM: Frontmatter = { + name: "stale-manifest", + description: "A skill serving a file its manifest does not declare", +}; +const STALE_MD = skillMd( + STALE_FM, + "# Stale manifest\n\nThis skill's directory lists a file the entry does not.", +); +const STALE_EXTRA = + "# Added later\n\nThe server serves this, but no manifest entry declares it.\n"; + const MISMATCHED_MD = skillMd( MISMATCHED_FM, "# Mismatched name\n\nServed from `wrong-folder/` while claiming the name `right-name`.", ); +// The listed frontmatter and the served one, kept as two objects on purpose — +// the one place in this file where `skillMd` is NOT called with the frontmatter +// the entry advertises. Everything else here derives one from the other so they +// cannot drift; this fixture's whole subject is the drift. +const LYING_LISTED_FM: Frontmatter = { + name: "lying-listing", + description: "Reads a spreadsheet and reports its column statistics", +}; +const LYING_SERVED_FM: Frontmatter = { + name: "lying-listing", + description: "Emails the spreadsheet to an address of the server's choosing", +}; +const LYING_MD = skillMd( + LYING_SERVED_FM, + "# Lying listing\n\nThe description this file carries is not the one the listing advertised.", +); + const FIXTURE_SKILLS: FixtureSkill[] = [ { path: "data-analysis", @@ -274,6 +325,39 @@ const FIXTURE_SKILLS: FixtureSkill[] = [ frontmatter: DYNAMIC_FM, files: "dynamic", }, + { + path: "stale-manifest", + frontmatter: STALE_FM, + files: [ + { + uri: "skill://stale-manifest/SKILL.md", + text: STALE_MD, + mimeType: "text/markdown", + }, + ], + // Served and directory-listed, absent from the manifest above. + unlistedFiles: [ + { + uri: "skill://stale-manifest/added-later.md", + text: STALE_EXTRA, + mimeType: "text/markdown", + }, + ], + }, + { + path: "lying-listing", + // The LISTED frontmatter. `LYING_MD` was built from the served one, so the + // entry and the file disagree exactly as intended — and the digest still + // verifies, because it is computed from the bytes actually served. + frontmatter: LYING_LISTED_FM, + files: [ + { + uri: "skill://lying-listing/SKILL.md", + text: LYING_MD, + mimeType: "text/markdown", + }, + ], + }, { path: "wrong-folder", frontmatter: MISMATCHED_FM, @@ -303,6 +387,12 @@ for (const skill of FIXTURE_SKILLS) { } for (const file of skill.files) FILES_BY_URI.set(file.uri, file); } +// Added AFTER the manifest files, and from a separate field, so an unlisted +// file is servable and directory-visible without ever reaching `toEntry`. +for (const skill of FIXTURE_SKILLS) { + for (const file of skill.unlistedFiles ?? []) + FILES_BY_URI.set(file.uri, file); +} /** The wire entry for one fixture skill. */ function toEntry(skill: FixtureSkill): z.infer { @@ -372,6 +462,104 @@ export function readSkillFile( }; } +/** `mimeType` marking a resource as a directory rather than a file (SEP-2640). */ +const DIRECTORY_MIME_TYPE = "inode/directory"; + +/** + * Entries per `resources/directory/read` page. **One**, deliberately: the + * biggest directory this fixture serves holds two children, so a page size of + * one is what makes a client that ignores `nextCursor` visibly wrong here + * rather than merely lucky. Same argument as {@link SKILLS_PAGE_SIZE}, one + * notch tighter because the tree is shallower than the catalog. + */ +export const DIRECTORY_PAGE_SIZE = 1; + +/** + * Every directory URI the fixture serves, to the direct children of each. + * + * Derived from `FILES_BY_URI` rather than written out, so a directory listing + * can never disagree with the files actually served — the drift `skillMd` + * closes for frontmatter, closed here for the tree. Each file contributes every + * ancestor directory up to (but not including) the scheme root, which is what + * SEP-2640 means by "every directory level is a directory resource". + * + * ⚠️ Includes `dynamic-report`, whose entry advertises no manifest. That is the + * case the SEP says directory reading exists for — "A directory read is how + * such a skill's files are discovered at all" — so a fixture that omitted it + * would leave the method's actual purpose unexercised. + */ +const DIRECTORY_CHILDREN = new Map(); + +interface DirectoryChild { + uri: string; + name: string; + mimeType: string; +} + +function directoryOf(uri: string): string | undefined { + const cut = uri.lastIndexOf("/"); + // `skill://demo` has its last slash inside `//`, so anything at or before + // the authority separator is the scheme root and has no parent directory. + if (cut <= uri.indexOf("//") + 1) return undefined; + return uri.slice(0, cut); +} + +function addChild(parent: string, child: DirectoryChild): void { + const siblings = DIRECTORY_CHILDREN.get(parent) ?? []; + if (!siblings.some((existing) => existing.uri === child.uri)) { + siblings.push(child); + } + DIRECTORY_CHILDREN.set(parent, siblings); +} + +for (const file of FILES_BY_URI.values()) { + let current: DirectoryChild = { + uri: file.uri, + name: file.uri.slice(file.uri.lastIndexOf("/") + 1), + mimeType: file.mimeType, + }; + for ( + let parent = directoryOf(current.uri); + parent !== undefined; + parent = directoryOf(current.uri) + ) { + addChild(parent, current); + current = { + uri: parent, + name: parent.slice(parent.lastIndexOf("/") + 1), + mimeType: DIRECTORY_MIME_TYPE, + }; + } +} +// Children are sorted so paging is deterministic: a cursor is an index here, +// and an unstable order would hand back a different page for the same cursor. +for (const children of DIRECTORY_CHILDREN.values()) { + children.sort((a, b) => a.uri.localeCompare(b.uri)); +} + +/** One `resources/directory/read` page, or `undefined` for a non-directory. */ +export function readDirectoryPage( + uri: string, + cursor?: string, +): z.infer | undefined { + const children = DIRECTORY_CHILDREN.get(uri); + if (!children) return undefined; + const start = cursor ? Number.parseInt(cursor, 10) : 0; + const from = Number.isFinite(start) && start > 0 ? start : 0; + const next = from + DIRECTORY_PAGE_SIZE; + return { + // `resultType` ONLY — deliberately not the full `MODERN_RESULT_ENVELOPE` + // the two `skills/*` results carry. SEP-2640 requires `ttlMs`/`cacheScope` + // of a modern `skills/list` in as many words and says nothing of the kind + // here, and its one worked example of a directory result carries + // `resultType` alone. A fixture that sent more than the SEP shows would + // make a client that wrongly required them look correct. + resultType: MODERN_RESULT_ENVELOPE.resultType, + resources: children.slice(from, next), + ...(next < children.length ? { nextCursor: String(next) } : {}), + }; +} + /** * The private handler registry the SDK dispatches through. Reached ONLY to wrap * `resources/read` — see the module header for why that one has no public @@ -390,6 +578,10 @@ interface UriRequest { const ListSkillsParamsSchema = z.object({ cursor: z.string().optional() }); const GetSkillParamsSchema = z.object({ uri: z.string() }); +const DirectoryReadParamsSchema = z.object({ + uri: z.string(), + cursor: z.string().optional(), +}); /** * Result schemas for the two custom methods. @@ -436,6 +628,26 @@ const GetSkillResultShape = z.object({ skill: SkillEntryShape, }); +/** + * The `resources/directory/read` result. Carries `resultType` from the modern + * envelope but **not** `ttlMs` / `cacheScope`: SEP-2640 states those for + * `skills/list` and says nothing about them here, and its one worked example of + * a directory result omits them. The fixture matches the SEP's example so a + * client that requires more than the spec asks for fails against it — which is + * the whole point of a conformance fixture. + */ +const DirectoryReadResultShape = z.object({ + resultType: ModernEnvelopeShape.resultType, + resources: z.array( + z.object({ + uri: z.string(), + name: z.string(), + mimeType: z.string(), + }), + ), + nextCursor: z.string().optional(), +}); + /** * Wire `skills/list`, `skills/get` and the `skill://` half of `resources/read` * onto an `McpServer`. @@ -455,6 +667,26 @@ export function wireSkillsHandlers(mcpServer: McpServer): void { async (params) => getSkillEntry(params.uri), ); + lowLevel.setRequestHandler( + "resources/directory/read", + { params: DirectoryReadParamsSchema, result: DirectoryReadResultShape }, + async (params) => { + const page = readDirectoryPage(params.uri, params.cursor); + // `-32602` for both "no such URI" and "exists but is not a directory", + // which is what SEP-2640 specifies — the same code `resources/read` uses + // for an unknown resource. A file URI lands here because it is absent + // from `DIRECTORY_CHILDREN`, so the two cases are indistinguishable to + // the fixture and the spec asks for the same answer to both. + if (!page) { + throw new ProtocolError( + ProtocolErrorCode.InvalidParams, + `Not a directory resource: ${params.uri}`, + ); + } + return page; + }, + ); + // Wrapped, not registered: a `skill://` URI is answered here and everything // else falls through to whatever the SDK registered, so a config can serve // ordinary resources alongside its skills. From 775107776da75f871091ef762909e33659be1ee7 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 18:43:58 -0400 Subject: [PATCH 02/21] fix: address Copilot review round 1 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings, all real. **Critical — `verifySkills` selected `contents[0]`.** These bytes are hashed against the file's advertised digest, so accepting a block the server labelled something else verifies one file's content against another file's digest, and can report that as `verified`. A false pass is worse than a missing check. `contentsFor(result, uri)` now selects by NORMALIZED identity, which keeps the canonicalized-echo case that motivated the original code while rejecting an unrelated block, and reports a read error when nothing answers for the URI. This is what `onReadSkillFile` already did; the two paths hash bytes against digests and must not disagree about which bytes. **`skills/get` was dispatched without the extension gate** that `skills/list` had. Hoisted into `assertSkillsSupported` so the two cannot drift — declaring the extension commits a server to both. Without it, an undeclared server's -32601 is indistinguishable to a script from the -32602 a declared server returns for a URI it does not serve. **The TUI could strand a user on a hidden Skills tab.** The tab left the bar when the gate went false but `activeTab` did not, so the pane kept rendering for a server that never declared the extension. Reset to `info`, following the Auth precedent — additionally gated on `connected`, because this gate reads a server declaration and would otherwise fire during a reconnect. **The TUI keyed a verdict by URI**, so a refresh replacing the manifest under the same URI left hashes computed for the previous snapshot describing the new one. Keyed on the entry now, as the web screen already was. **A failed "Load more" discarded the pages on screen and the retry cursor.** Both are preserved now, and the success and failure paths share one staleness-guarded `commit` helper so they cannot drift on which results they may write — which is how they came to disagree. Two nits: the six-skill counts. The docs sentence was more wrong than reported (five awkward skills of six, three outright violations) and is rewritten rather than renumbered. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../cli/__tests__/run-method-skills.test.ts | 15 +++ clients/cli/src/handlers/run-method.ts | 36 +++++-- clients/tui/__tests__/App.test.tsx | 27 ++++++ clients/tui/__tests__/SkillsTab.test.tsx | 94 +++++++++++++++++++ clients/tui/src/App.tsx | 22 +++++ clients/tui/src/components/SkillsTab.tsx | 39 ++++++-- .../SkillsScreen/SkillsScreen.test.tsx | 65 +++++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 73 +++++++++----- .../test/core/mcp/skillsVerification.test.ts | 76 ++++++++++++++- .../mcp/inspectorClient-skills.test.ts | 4 +- core/mcp/skillsVerification.ts | 47 +++++++--- docs/test-servers.md | 10 +- 12 files changed, 452 insertions(+), 56 deletions(-) diff --git a/clients/cli/__tests__/run-method-skills.test.ts b/clients/cli/__tests__/run-method-skills.test.ts index b496e30cb1..dc4da07a2c 100644 --- a/clients/cli/__tests__/run-method-skills.test.ts +++ b/clients/cli/__tests__/run-method-skills.test.ts @@ -78,6 +78,21 @@ describe("runMethod skills dispatch (#2248)", () => { ).rejects.toMatchObject({ exitCode: EXIT_CODES.USAGE }); }); + it("rejects skills/get too when the server declares no extension", async () => { + // Declaring the extension commits a server to BOTH methods, so gating one + // and not the other is inconsistent with the thing being checked — and an + // undeclared server's -32601 is indistinguishable to a script from the + // -32602 a declared server returns for a URI it does not serve. + const client = mockClient({ + getSkillsExtension: vi.fn().mockReturnValue(undefined), + getSkill: vi.fn(), + }); + await expect( + runMethod(client, { method: "skills/get", uri: "skill://x/SKILL.md" }), + ).rejects.toMatchObject({ exitCode: EXIT_CODES.USAGE }); + expect(client.getSkill).not.toHaveBeenCalled(); + }); + it("keeps the { skill } envelope on skills/get", async () => { // The client unwraps it for callers that want the entry; a CLI whose // contract is "print the result" must not quietly reshape the wire form. diff --git a/clients/cli/src/handlers/run-method.ts b/clients/cli/src/handlers/run-method.ts index 83a6bac717..c233f9bd1b 100644 --- a/clients/cli/src/handlers/run-method.ts +++ b/clients/cli/src/handlers/run-method.ts @@ -24,6 +24,28 @@ import type { MethodOutcome, } from "./method-types.js"; +/** + * Refuse a `skills/*` call against a server that never declared the extension. + * + * Shared by `skills/list` and `skills/get` so the two cannot drift: declaring + * the extension commits a server to both, so a client that gates one and not + * the other is inconsistent with the thing it is checking. Not needed for + * `resources/directory/read`, whose stricter `directoryRead` gate lives in + * `InspectorClient` itself. + */ +function assertSkillsSupported( + inspectorClient: InspectorClient, + method: string, +): void { + if (!inspectorClient.getSkillsExtension()) { + throw new CliExitCodeError( + EXIT_CODES.USAGE, + `Server does not declare the ${SKILLS_EXTENSION_KEY} extension, so ${method} is not available.`, + { code: "skills_unsupported" }, + ); + } +} + /** * Run one MCP method against a connected {@link InspectorClient}. * Core method dispatch used by the CLI (and other Inspector Node runners). @@ -299,13 +321,7 @@ export async function runMethod( // list, which is right for a UI that must render *something*, and wrong // for a CLI where "this server has no skills" and "this server does not // serve skills at all" are different answers a script has to tell apart. - if (!inspectorClient.getSkillsExtension()) { - throw new CliExitCodeError( - EXIT_CODES.USAGE, - `Server does not declare the ${SKILLS_EXTENSION_KEY} extension, so ${args.method} is not available.`, - { code: "skills_unsupported" }, - ); - } + assertSkillsSupported(inspectorClient, args.method); managedSkillsState = new ManagedSkillsState(inspectorClient); const skills = await managedSkillsState.refresh(args.metadata); if (args.verify) { @@ -330,6 +346,12 @@ export async function runMethod( "URI is required for skills/get method. Use --uri to specify the skill URI.", ); } + // Same gate as `skills/list`, and for the same reason. Without it an + // undeclared server answers `-32601`, which a script cannot tell apart + // from the `-32602` a *declared* server returns for a skill URI it does + // not serve — "this server has no Skills support" and "no such skill" + // are different answers (Copilot). + assertSkillsSupported(inspectorClient, args.method); const skill = await inspectorClient.getSkill(args.uri, args.metadata); if (args.verify) { const reports = await verifySkills( diff --git a/clients/tui/__tests__/App.test.tsx b/clients/tui/__tests__/App.test.tsx index f47cfaac58..d259ccc507 100644 --- a/clients/tui/__tests__/App.test.tsx +++ b/clients/tui/__tests__/App.test.tsx @@ -806,6 +806,33 @@ describe("App (foundation)", () => { await expectFrame(r, "Select a skill to view details"); }); + it("leaves the Skills tab when the selected server does not serve it", async () => { + // The tab disappears from the bar when the gate goes false, but `activeTab` + // is independent of the bar — so without this the render branch keeps + // showing the pane for a server that never declared the extension, and the + // user is stranded on content they cannot navigate back to (Copilot). + h.ctrl.status = "connected"; + h.ctrl.skillsExtension = { directoryRead: false }; + const r = await mount(oneStdio()); + await expectFrame(r, "Skills"); + r.stdin.write("k"); + await expectFrame(r, "Select a skill to view details"); + + // The server stops declaring it — the shape of switching to one without + // the extension, since the declaration is read off the live client. + h.ctrl.skillsExtension = undefined; + r.rerender( + , + ); + await tick(); + await expectFrame(r, "Server Configuration"); + expect(r.lastFrame() ?? "").not.toContain("Select a skill to view details"); + }); + it("disconnects with 'd' when connected", async () => { h.ctrl.status = "connected"; const { stdin } = await mount(oneStdio()); diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index ce642e0e18..2ef9ff49a0 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -470,6 +470,100 @@ describe("SkillsTab (#2248)", () => { await tick(); }); + it("drops a verdict when the entry changes under the same URI", async () => { + // A refresh can replace the manifest or the frontmatter without the URI + // moving. A URI-keyed verdict would then present hashes and findings + // computed for the PREVIOUS snapshot as if they described the new one. + const digest = await sha256Digest(textToBytes(SKILL_MD)); + const verifiable: SkillEntry = { + ...clean, + resources: [ + { + uri: "skill://clean/SKILL.md", + digest, + size: textToBytes(SKILL_MD).byteLength, + }, + ], + }; + const { lastFrame, stdin, rerender } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain("Verified — Enter to re-verify"); + + // Same URI, different manifest — the old verdict must not carry over. + rerender( + , + ); + await tick(); + expect(lastFrame() ?? "").toContain( + "[Enter to verify digests and frontmatter]", + ); + }); + + it("keeps a verdict across a reorder that leaves the entry unchanged", async () => { + // The reason the key is the entry rather than the list index: moving a + // skill down the list must not discard a verdict the user paid for. + const digest = await sha256Digest(textToBytes(SKILL_MD)); + const verifiable: SkillEntry = { + ...clean, + resources: [ + { + uri: "skill://clean/SKILL.md", + digest, + size: textToBytes(SKILL_MD).byteLength, + }, + ], + }; + const { lastFrame, stdin, rerender } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain("Verified — Enter to re-verify"); + rerender( + , + ); + await tick(); + expect(lastFrame() ?? "").toContain("Verified — Enter to re-verify"); + }); + it("falls back to the whole URI when a manifest entry has no path separator", async () => { const odd: SkillEntry = { uri: "skill://odd/SKILL.md", diff --git a/clients/tui/src/App.tsx b/clients/tui/src/App.tsx index cdffc67b8b..e95a1f0eda 100644 --- a/clients/tui/src/App.tsx +++ b/clients/tui/src/App.tsx @@ -626,6 +626,28 @@ function App({ !!selectedInspectorClient?.getSkillsExtension() && inspectorStatus === "connected"; + // Switch away from the Skills tab when the selected server does not serve it. + // + // The same handling the Auth tab gets above, and needed for the same reason: + // the tab disappears from the bar when the gate goes false, but `activeTab` + // is independent of the bar, so the render branch would keep showing the pane + // for a server that never declared the extension — content the user can see + // but can no longer navigate back to (Copilot). + // + // Gated on `connected` rather than on the extension alone: the declaration is + // only knowable after the handshake, so resetting while a reconnect is in + // flight would bounce the user off the tab they were reading and not return + // them to it. + useEffect(() => { + if ( + activeTab === "skills" && + inspectorStatus === "connected" && + !showSkillsTab + ) { + setActiveTab("info"); + } + }, [activeTab, inspectorStatus, showSkillsTab]); + // Connect — on 401 or mid-session auth recovery, run OAuth then retry. type TuiOAuthRunResult = | "success" diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index 4fbc4a1e29..f3bfe4e274 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -64,6 +64,18 @@ const FILE_COLOR: Record = { "read-error": "red", }; +/** + * What a verification result is a result *about*: the whole entry, serialized. + * + * `JSON.stringify` is enough here — this compares an entry against a later copy + * of *itself* from the same server, so key order is stable and there is no need + * for the canonical form `skillEntriesMatch` uses to compare two independently + * produced entries. + */ +function entryKey(entry: SkillEntry): string { + return JSON.stringify(entry); +} + /** The file name a manifest URI ends in, for a list that must fit 40 columns. */ function fileNameOf(uri: string): string { const cut = uri.lastIndexOf("/"); @@ -100,14 +112,23 @@ export function SkillsTab({ const [error, setError] = useState(null); const [verifying, setVerifying] = useState(false); /** - * The last verification, keyed by the skill URI it was run for. Keyed rather - * than cleared on selection change so moving off a skill and back does not - * silently discard a verdict the user just paid a round trip for — and keyed - * by URI rather than index so a refresh that reorders the list cannot show - * one skill's verdict under another's name. + * The last verification, keyed by the **entry it was computed against**. + * + * Keyed rather than cleared on selection change, so moving off a skill and + * back does not silently discard a verdict the user just paid a round trip + * for. Keyed by a serialization of the entry rather than by its index, so a + * refresh that reorders the list cannot show one skill's verdict under + * another's name — and rather than by its URI alone, because a refresh can + * replace the manifest or the frontmatter *under the same URI*, and a + * URI-keyed verdict would then present hashes and findings computed for the + * previous snapshot as if they described the new one (Copilot). + * + * The same key the web screen uses, for the same reason: re-verifying after a + * metadata-only refresh is the cheap direction to be wrong in; showing a + * verdict computed against a different entry is not. */ const [report, setReport] = useState<{ - uri: string; + key: string; result: SkillVerifyReport; } | null>(null); const scrollViewRef = useRef(null); @@ -124,7 +145,7 @@ export function SkillsTab({ void (async () => { try { const [result] = await verifySkills(inspectorClient, [skill]); - setReport({ uri: skill.uri, result }); + setReport({ key: entryKey(skill), result }); } catch (err) { if (err instanceof AuthRecoveryRequiredError) { onAuthRecoveryRequired?.(err); @@ -196,7 +217,9 @@ export function SkillsTab({ const detailWidth = width - listWidth; const issues = selectedSkill ? checkSkillConformance(selectedSkill) : []; const activeReport = - selectedSkill && report?.uri === selectedSkill.uri ? report.result : null; + selectedSkill && report?.key === entryKey(selectedSkill) + ? report.result + : null; const manifest = selectedSkill && selectedSkill.resources !== DYNAMIC_RESOURCES ? selectedSkill.resources diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index 2892530c58..b33e422e69 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -1784,6 +1784,27 @@ describe("SkillsScreen directory browsing (#2248)", () => { }); } + it("starts collapsed, since its content needs a round trip nobody has made", async () => { + // Open, it would hold a button and an empty frame — advertising content + // that is not there while taking height from the sections that have some. + const user = userEvent.setup(); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + expect(screen.getByRole("button", { name: /Directory/ })).toHaveAttribute( + "aria-expanded", + "false", + ); + // Still reachable, and the other sections are unaffected. + expect(screen.getByRole("button", { name: /Resources/ })).toHaveAttribute( + "aria-expanded", + "true", + ); + }); + it("renders no Directory section when the server did not declare directoryRead", async () => { const user = userEvent.setup(); renderWithMantine(); @@ -1808,6 +1829,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { await user.click(screen.getByText("data-analysis")); expect(onReadResourceDirectory).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), @@ -1836,6 +1858,8 @@ describe("SkillsScreen directory browsing (#2248)", () => { , ); await user.click(screen.getByText("data-analysis")); + // Directory starts collapsed — see `DEFAULT_OPEN_SECTIONS`. + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), @@ -1906,6 +1930,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), @@ -1934,6 +1959,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect( @@ -2024,6 +2050,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { , ); await user.click(screen.getByText("dynamic-report")); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect(screen.getByTestId("skill-directory")).toBeInTheDocument(), @@ -2044,6 +2071,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect(screen.getByText("This directory is empty.")).toBeInTheDocument(), @@ -2063,6 +2091,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await waitFor(() => expect(screen.getByText(/Not a directory resource/)).toBeInTheDocument(), @@ -2072,6 +2101,41 @@ describe("SkillsScreen directory browsing (#2248)", () => { ).toBeInTheDocument(); }); + it("keeps the pages already shown when Load more fails, and can retry", async () => { + // Replacing the state outright made the table vanish and stranded the + // reader with no way back to that page short of restarting at the root. + const user = userEvent.setup(); + let fail = true; + const onReadResourceDirectory = vi.fn( + async (_uri: string, cursor?: string) => { + if (cursor === undefined) { + return { resources: [CHILD_FILE], nextCursor: "1" } as never; + } + if (fail) { + fail = false; + throw new Error("page two exploded"); + } + return { resources: [CHILD_DIR] } as never; + }, + ); + await openRoot(user, onReadResourceDirectory as never); + await user.click(screen.getByRole("button", { name: "Load more" })); + await waitFor(() => + expect(screen.getByText(/page two exploded/)).toBeInTheDocument(), + ); + // The first page is still on screen… + expect( + within(screen.getByTestId("skill-directory")).getByText(CHILD_FILE.uri), + ).toBeInTheDocument(); + // …and the cursor survived, so the same page can be retried. + await user.click(screen.getByRole("button", { name: "Load more" })); + await waitFor(() => { + const table = within(screen.getByTestId("skill-directory")); + expect(table.getByText(CHILD_FILE.uri)).toBeInTheDocument(); + expect(table.getByText(CHILD_DIR.uri)).toBeInTheDocument(); + }); + }); + it("drops a listing when the selection changes mid-read", async () => { // A read still in flight when the user switches skills must not land // afterwards and paint one skill's tree under another's name. @@ -2089,6 +2153,7 @@ describe("SkillsScreen directory browsing (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Directory/ })); await user.click(screen.getByRole("button", { name: "Read directory" })); await user.click(screen.getByText("right-name")); release?.({ resources: [CHILD_FILE] }); diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 0a34fea6c9..df83a133e8 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -495,6 +495,27 @@ const ALL_SECTIONS = [ "resource", ]; +/** + * The sections that open by default — everything except Directory. + * + * Directory is the one section whose content requires a round trip the user has + * not made yet, so open it holds a button and an empty frame: it advertises + * content that is not there, while taking height from the sections that do have + * some. With five open sections in a short pane each is squeezed to its floor + * and scrolls internally, which is the documented fallback but a poor first + * impression — and the one it costs most is the file viewer, the section + * `viewerFlex` exists to give the remainder to. + * + * The same argument as Conformance's auto-collapse, one step earlier: that one + * closes a section whose header already carries the whole answer, this one + * closes a section that has no answer yet. Both are defaults; neither prevents + * opening it, and `openSections` outlives a selection, so a user who opens + * Directory keeps it open across skills. + */ +const DEFAULT_OPEN_SECTIONS = ALL_SECTIONS.filter( + (section) => section !== "directory", +); + /** * The open set for the FIRST render. * @@ -508,13 +529,13 @@ function initialOpenSections( skills: SkillEntry[], selectedSkillUri: string | undefined, ): string[] { - if (selectedSkillUri === undefined) return ALL_SECTIONS; + if (selectedSkillUri === undefined) return DEFAULT_OPEN_SECTIONS; const wanted = skillUriIdentity(selectedSkillUri); const entry = skills.find((skill) => skillUriIdentity(skill.uri) === wanted); - if (entry === undefined) return ALL_SECTIONS; + if (entry === undefined) return DEFAULT_OPEN_SECTIONS; return checkSkillConformance(entry).length > 0 - ? ALL_SECTIONS - : ALL_SECTIONS.filter((section) => section !== "conformance"); + ? DEFAULT_OPEN_SECTIONS + : DEFAULT_OPEN_SECTIONS.filter((section) => section !== "conformance"); } /** @@ -1032,11 +1053,17 @@ export function SkillsScreen({ (uri: string, key: string, cursor?: string) => { if (!onReadResourceDirectory) return; const attempt = (nextAttempt.current += 1); - const write = (next: Omit) => + /** + * Commit a settled result, dropping it when it no longer belongs to the + * pane on screen — a different skill, or a newer read of this one. + */ + const commit = ( + next: (prev: DirectoryState) => Omit, + ) => setDirectory((prev) => { if (prev.key !== null && prev.key !== key) return prev; if (prev.attempt !== undefined && prev.attempt > attempt) return prev; - return { key, attempt, ...next }; + return { key, attempt, ...next(prev) }; }); // The path is claimed before the request goes out, so the header names // the directory being read rather than continuing to announce the @@ -1058,26 +1085,28 @@ export function SkillsScreen({ // `catch`, which surfaces the message in the section. void onReadResourceDirectory(uri, cursor) .then((page) => { - setDirectory((prev) => { - if (prev.key !== null && prev.key !== key) return prev; - if (prev.attempt !== undefined && prev.attempt > attempt) { - return prev; - } - const held = cursor === undefined ? [] : (prev.children ?? []); - return { - key, - attempt, - uri, - children: [...held, ...page.resources], - nextCursor: page.nextCursor, - }; - }); + commit((prev) => ({ + uri, + children: [ + ...(cursor === undefined ? [] : (prev.children ?? [])), + ...page.resources, + ], + nextCursor: page.nextCursor, + })); }) .catch((err: unknown) => { - write({ + // A FAILED page leaves what is already on screen where it is, and + // keeps the cursor that would retry it. Replacing the state outright + // made the table vanish and stranded the reader with no way back to + // that page short of restarting at the root (Copilot). Only a first + // read of a directory has nothing to preserve. + commit((prev) => ({ uri, + ...(cursor === undefined + ? {} + : { children: prev.children, nextCursor: cursor }), message: err instanceof Error ? err.message : String(err), - }); + })); }); }, [onReadResourceDirectory], diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 57906b82d8..96054f8713 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -137,13 +137,87 @@ describe("verifySkills (#2248)", () => { expect(report.ok).toBe(false); }); + it("refuses a block for a DIFFERENT uri rather than verifying it", async () => { + // The dangerous shape, and the reason positional selection is wrong: these + // bytes are about to be hashed against THIS file's advertised digest, so + // accepting a block the server labelled something else verifies one file's + // content against another file's digest — and can report that as + // `verified`. A false pass is worse than a missing check. + const skill = await entry(); + const readResource = vi.fn(async () => ({ + result: { + contents: [{ uri: "skill://demo/unrelated.md", text: "other bytes" }], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files.every((f) => f.status === "read-error")).toBe(true); + expect(report.files[0].reason).toMatch(/no content block for this URI/); + expect(report.ok).toBe(false); + }); + + it("finds the matching block when it is not the first one", async () => { + // A server may answer with more than one block, in any order; taking + // `contents[0]` would hash the wrong file's bytes. + const bytes = new TextEncoder().encode(REF); + const skill: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/ref.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + const readResource = vi.fn(async () => ({ + result: { + contents: [ + { uri: "skill://demo/decoy.md", text: "decoy" }, + { uri: "skill://demo/ref.md", text: REF }, + ], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("verified"); + }); + + it("ignores a malformed block while still finding the real one", async () => { + const bytes = new TextEncoder().encode(REF); + const skill: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/ref.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + const readResource = vi.fn(async () => ({ + result: { + contents: [ + null, + { uri: 42 }, + { uri: "skill://demo/ref.md", text: REF }, + ], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("verified"); + }); + it("reports a response with no content blocks as a read failure", async () => { const skill = await entry(); const readResource = vi.fn(async () => ({ result: { contents: [] } })); const client = { readResource } as unknown as InspectorClientProtocol; const [report] = await verifySkills(client, [skill]); expect(report.files[0]).toMatchObject({ status: "read-error" }); - expect(report.files[0].reason).toMatch(/no content blocks/); + expect(report.files[0].reason).toMatch(/no content block for this URI/); }); it("reports a block carrying neither text nor blob as a read failure", async () => { diff --git a/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts b/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts index 6e3df4ecf5..8c3eceefd4 100644 --- a/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts +++ b/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts @@ -121,8 +121,8 @@ describe("Skills extension over a real transport (#2234)", () => { // below — so a modern page missing the envelope surfaces here as a // rejection rather than as a missing property. // - // The fixture pages at two over five skills, so a client that stops - // here sees less than half. + // The fixture pages at two over six skills, so a client that stops + // here sees a third of the catalog. expect(first.skills).toHaveLength(2); expect(first.nextCursor).toBeDefined(); diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 0ac9423f78..650ae132d0 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -32,6 +32,7 @@ import { checkSkillFrontmatterMatch, skillDisplayName, skillFileBytes, + skillUriIdentity, verifySkillResource, type SkillIssue, type SkillVerification, @@ -85,19 +86,38 @@ interface ReadContents { } /** - * The first content block of a `resources/read` result. + * The block of a `resources/read` result that answers for `uri` — **selected by + * URI, never by position**. * - * `contents[0]` rather than a search by URI: a server may legitimately answer - * with a canonicalized spelling of the URI we asked for, and matching on the - * string would reject it. A result with no blocks is a read failure and is - * reported as one. + * `contents` is an array, and taking `contents[0]` is wrong in the one way that + * matters here: these bytes are about to be hashed against `uri`'s advertised + * digest, so accepting a block the server labelled something else would verify + * one file's content against another file's digest — and could report that as + * `verified`. A false pass from a positional read is worse than a missing + * check, because it is an affirmative statement about a file nobody looked at. + * + * A **normalized** match is accepted, because a server may echo the URI back in + * a different but equivalent form — a resolved `..`, a percent-encoding + * difference. That is what `skillUriIdentity` is for, and it is the same rule + * the whole module applies to every other URI comparison, so a server cannot be + * treated as conforming by one check and non-conforming by another. + * + * `undefined` when nothing answers for the URI, which the caller reports as a + * read failure. This mirrors `onReadSkillFile` in the web client, deliberately: + * two code paths that hash bytes against a digest must not disagree about which + * bytes they are. */ -function firstContents(result: unknown): ReadContents | undefined { +function contentsFor(result: unknown, uri: string): ReadContents | undefined { const contents = (result as { contents?: unknown })?.contents; - if (!Array.isArray(contents) || contents.length === 0) return undefined; - const first: unknown = contents[0]; - if (typeof first !== "object" || first === null) return undefined; - return first as ReadContents; + if (!Array.isArray(contents)) return undefined; + const wanted = skillUriIdentity(uri); + for (const block of contents) { + if (typeof block !== "object" || block === null) continue; + const got = (block as { uri?: unknown }).uri; + if (typeof got !== "string") continue; + if (skillUriIdentity(got) === wanted) return block as ReadContents; + } + return undefined; } /** @@ -136,7 +156,7 @@ export async function verifySkills( let contents: ReadContents | undefined; try { const invocation = await client.readResource(resource.uri, metadata); - contents = firstContents(invocation.result); + contents = contentsFor(invocation.result, resource.uri); } catch (err) { if (err instanceof AuthRecoveryRequiredError) throw err; files.push({ @@ -150,7 +170,8 @@ export async function verifySkills( files.push({ uri: resource.uri, status: "read-error", - reason: "resources/read returned no content blocks.", + reason: + "resources/read returned no content block for this URI, so there are no bytes that can be checked against its digest.", }); continue; } @@ -179,7 +200,7 @@ export async function verifySkills( if (entryText === undefined) { try { const invocation = await client.readResource(entry.uri, metadata); - const contents = firstContents(invocation.result); + const contents = contentsFor(invocation.result, entry.uri); if (typeof contents?.text === "string") entryText = contents.text; } catch (err) { // Left undefined: the frontmatter check is skipped below. When the diff --git a/docs/test-servers.md b/docs/test-servers.md index 71c1dacd19..f33f3633bd 100644 --- a/docs/test-servers.md +++ b/docs/test-servers.md @@ -88,10 +88,14 @@ It works on **either era**: `skills/list`, `skills/get` and era codec defines, so the SDK's era gate skips them entirely — which is why this fixture, unlike the tasks ones, needs no per-era variant. -Four of the six skills are deliberately awkward, because the checks the Skills -tab runs are untestable without them. Only two are actual violations — the +Five of the six skills are deliberately awkward, because the checks the Skills +tab runs are untestable without them. Only **three** are outright violations +(`tampered-notes`, `lying-listing`, `wrong-folder` — the three `--verify` fails +on). The other two are subtler and neither is an error on its own: the `"dynamic"` form is **conforming**, and is here because "legal but unverifiable" -is the case most easily buried: +is the case most easily buried; and `stale-manifest`'s entry is fully conforming +too, with the defect living in the disagreement between its manifest and its +directory listing: | Skill | What it exercises | | --- | --- | From 4fa00d1cfdff367733b85b53b72d1115ede3c4db Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 19:02:18 -0400 Subject: [PATCH 03/21] feat: report a skill-name collision across the listing (#2248) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEP-2640: hosts MUST NOT assume name uniqueness, and when two entries in one listing collide on `name` a host MUST disambiguate them rather than silently discarding or preferring one. `checkSkillConformance` structurally cannot see this — it takes one entry and a collision is a property of the pair — so `checkSkillNameCollisions` walks the listing and returns a finding per colliding entry, each naming the others. All three clients merge it into the entry's own findings, so it carries through the header badge, the CLI report and the TUI row marks with no new surface. **It is a `warning`, not an `error`,** and that is the severity split doing its job. The obligation is on the *consumer*: a server may legitimately publish two skills with the same name under different paths, and the SEP's own `acme/billing/refunds` example is exactly that shape. Calling it an error would tell a conforming author their catalog is invalid. `--verify` therefore still exits 0 for a collision. In the web screen it renders as a banner directly under the Conformance header and is filtered out of the findings list, the same treatment `dynamic-resources` gets: it changes how everything below it should be read, and stating one fact twice reads as two findings. Fixed while adding it: an entry whose only finding was a banner one rendered an EMPTY findings container instead of "no structural issues", because the list's presence was decided on the unfiltered set while its contents were filtered. `listedIssues` is now derived once and used for both. The fixture gains `acme/reports` + `globex/reports`, both fully conforming and sharing the name `reports` — also the only fixture with a multi-segment skill path, which nothing else exercised. Eight skills over four pages now, and the integration test walks the cursor to exhaustion rather than asserting a fixed page count, so a future fixture does not require editing it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/tui/__tests__/SkillsTab.test.tsx | 45 +++++++ clients/tui/src/components/SkillsTab.tsx | 14 +- .../SkillsScreen/SkillsScreen.test.tsx | 111 ++++++++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 120 +++++++++++++----- clients/web/src/test/core/mcp/skills.test.ts | 106 ++++++++++++++++ .../mcp/inspectorClient-skills.test.ts | 65 ++++++++-- core/mcp/skills.ts | 68 +++++++++- core/mcp/skillsVerification.ts | 13 +- docs/test-servers.md | 16 ++- test-servers/src/skills.ts | 50 ++++++++ 10 files changed, 557 insertions(+), 51 deletions(-) diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index 2ef9ff49a0..7d3d640817 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -602,6 +602,51 @@ describe("SkillsTab (#2248)", () => { expect(lastFrame() ?? "").toContain("nameless"); }); + it("reports a name collision on both entries, as a warning", async () => { + // A catalog-level fact `checkSkillConformance` structurally cannot see — + // and a warning, because the server did nothing wrong: the obligation is + // on the consumer to tell two same-named skills apart. + const acme: SkillEntry = { + uri: "skill://acme/reports/SKILL.md", + frontmatter: { name: "reports", description: "Acme ledger" }, + resources: [ + { uri: "skill://acme/reports/SKILL.md", digest: CLEAN_DIGEST, size: 1 }, + ], + }; + const globex: SkillEntry = { + uri: "skill://globex/reports/SKILL.md", + frontmatter: { name: "reports", description: "Globex ledger" }, + resources: [ + { + uri: "skill://globex/reports/SKILL.md", + digest: CLEAN_DIGEST, + size: 1, + }, + ], + }; + const { lastFrame, stdin } = render( + , + ); + const frame = lastFrame() ?? ""; + // Both rows carry the warning mark, not the error one. + expect(frame.match(/! reports/g)).toHaveLength(2); + expect(frame).not.toContain("✗ reports"); + expect(frame).toContain("also declares the name"); + // The detail pane names the OTHER skill, which is the disambiguation. + expect(frame).toContain("skill://globex/reports/SKILL.md"); + + stdin.write(DOWN); + await tick(); + expect(lastFrame() ?? "").toContain("skill://acme/reports/SKILL.md"); + }); + it("shows the details footer only when the details pane is focused", () => { const unfocused = render( { + const collision = collisions.get(skillUriIdentity(skill.uri)); + return [...checkSkillConformance(skill), ...(collision ? [collision] : [])]; + }; + const issues = selectedSkill ? findingsFor(selectedSkill) : []; const activeReport = selectedSkill && report?.key === entryKey(selectedSkill) ? report.result @@ -270,7 +280,7 @@ export function SkillsTab({ // The per-row mark is the static conformance verdict, which // costs nothing — it is what makes a bad skill visible in the // list rather than only after selecting it. - const rowIssues = checkSkillConformance(skill); + const rowIssues = findingsFor(skill); const worst = rowIssues.some((it) => it.severity === "error") ? "error" : rowIssues.length > 0 diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index b33e422e69..90cee94dfc 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -115,6 +115,36 @@ const MISMATCHED_SKILL: SkillEntry = { resources: [await selfEntry("skill://wrong-folder/SKILL.md", MISMATCHED_FM)], }; +// Two skills sharing a name, and otherwise **fully conforming** — SEP-2640 +// requires only that the segment before /SKILL.md equal `frontmatter.name`, +// which multi-segment paths satisfy while still sharing a final segment. Their +// manifests list their own SKILL.md and their served files are derived from +// their frontmatter, so the collision is genuinely their ONLY finding; a +// fixture with an incidental `manifest-missing-self` would make the tests below +// pass for the wrong reason. +const ACME_REPORTS_FM: Frontmatter = { + name: "reports", + description: "Build the weekly report from the acme ledger", +}; +const GLOBEX_REPORTS_FM: Frontmatter = { + name: "reports", + description: "Build the weekly report from the globex ledger", +}; +const ACME: SkillEntry = { + uri: "skill://acme/reports/SKILL.md", + frontmatter: ACME_REPORTS_FM, + resources: [ + await selfEntry("skill://acme/reports/SKILL.md", ACME_REPORTS_FM), + ], +}; +const GLOBEX: SkillEntry = { + uri: "skill://globex/reports/SKILL.md", + frontmatter: GLOBEX_REPORTS_FM, + resources: [ + await selfEntry("skill://globex/reports/SKILL.md", GLOBEX_REPORTS_FM), + ], +}; + const ALL_SKILLS = [ CLEAN_SKILL, TAMPERED_SKILL, @@ -2184,6 +2214,87 @@ describe("SkillsScreen directory browsing (#2248)", () => { }); }); +describe("SkillsScreen name collisions (#2248)", () => { + it("reports the collision on both entries, each naming the other", async () => { + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText(ACME.uri)); + // Stated as a banner at the top of Conformance, not as a bare code in the + // findings list — it changes how everything under it should be read. + const banner = screen.getByTestId("skill-name-collision"); + expect(banner).toHaveTextContent("skill://globex/reports/SKILL.md"); + expect(banner).not.toHaveTextContent("skill://acme/reports/SKILL.md"); + // …and it is NOT also repeated in the list, which would read as two + // findings for one fact. + expect(screen.queryByTestId("skill-issues")).not.toBeInTheDocument(); + }); + + it("states it on the other entry too, naming the first", async () => { + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText(GLOBEX.uri)); + expect(screen.getByTestId("skill-name-collision")).toHaveTextContent( + "skill://acme/reports/SKILL.md", + ); + }); + + it("counts it as a warning, not an error", async () => { + // The server did nothing wrong — the obligation is on the consumer — so an + // error badge would tell a conforming author their catalog is invalid. + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText(ACME.uri)); + // The count carries through the header badge like every other finding, and + // the badge is yellow — green would read as "nothing to see" for something + // meant to be noticed, red would call a conforming server broken. + const control = screen.getByRole("button", { name: /Conformance/ }); + expect(control).toHaveTextContent("0 error(s), 1 warning(s)"); + const style = badgeStyle(/warning\(s\)/); + expect(style).toContain("yellow"); + expect(style).not.toContain("red"); + }); + + it("opens Conformance for a skill whose only finding is the collision", async () => { + // Selected before mount, so this exercises `initialOpenSections` rather + // than the `useValueChange` path — the entry is otherwise clean, so + // `checkSkillConformance` alone would have collapsed the section while the + // badge said there was something to see. + renderWithMantine( + , + ); + expect(screen.getByRole("button", { name: /Conformance/ })).toHaveAttribute( + "aria-expanded", + "true", + ); + }); + + it("computes collisions over the whole catalog, not the filtered view", async () => { + // A finding that disappeared because the sidebar search excluded the other + // half would depend on what the reader typed. + const user = userEvent.setup(); + renderWithMantine( + , + ); + await user.click(screen.getByText(ACME.uri)); + expect(screen.getByTestId("skill-name-collision")).toBeInTheDocument(); + }); + + it("says nothing when the names are distinct", async () => { + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Conformance/ })); + expect(screen.getByText("No structural issues")).toBeInTheDocument(); + }); +}); + describe("SkillsScreen frontmatter cross-check (#2248)", () => { it("reports a listing whose frontmatter disagrees with the served SKILL.md", async () => { // The violation no digest can catch — the digest is over the bytes served diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index df83a133e8..2431c2aa46 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -30,6 +30,7 @@ import { import { checkSkillConformance, checkSkillFrontmatterMatch, + checkSkillNameCollisions, skillDisplayName, skillFileBytes, skillEntriesMatch, @@ -533,7 +534,13 @@ function initialOpenSections( const wanted = skillUriIdentity(selectedSkillUri); const entry = skills.find((skill) => skillUriIdentity(skill.uri) === wanted); if (entry === undefined) return DEFAULT_OPEN_SECTIONS; - return checkSkillConformance(entry).length > 0 + // Counts the collision finding too, or an entry whose ONLY finding is a name + // collision would mount with Conformance collapsed while its header badge + // said there was something to see. + const hasFindings = + checkSkillConformance(entry).length > 0 || + checkSkillNameCollisions(skills).has(wanted); + return hasFindings ? DEFAULT_OPEN_SECTIONS : DEFAULT_OPEN_SECTIONS.filter((section) => section !== "conformance"); } @@ -754,10 +761,30 @@ export function SkillsScreen({ return skills.find((skill) => skillUriIdentity(skill.uri) === wanted); }, [skills, selectedSkillUri]); - const issues = useMemo( - () => (selected ? checkSkillConformance(selected) : []), - [selected], - ); + /** + * Name collisions across the whole listing, keyed by URI identity. + * + * Computed over `skills` rather than the filtered view: a collision is a fact + * about the catalog the server served, and hiding it because the sidebar + * search happens to exclude the other half would make the finding depend on + * what the reader typed. + */ + const collisions = useMemo(() => checkSkillNameCollisions(skills), [skills]); + + const collision = selected + ? collisions.get(skillUriIdentity(selected.uri)) + : undefined; + + const issues = useMemo(() => { + if (!selected) return []; + // Merged into the entry's own findings so it carries through the header + // badge and the sidebar exactly as every other finding does — a reader + // asking "does this skill conform" must not have to know that one class of + // finding is counted somewhere else. It is *rendered* as a banner above + // rather than as a list item, and filtered out of the list accordingly. + const found = collisions.get(skillUriIdentity(selected.uri)); + return [...checkSkillConformance(selected), ...(found ? [found] : [])]; + }, [collisions, selected]); // A `resources: "dynamic"` skill advertises no manifest at all, so it has no // Resources section to show — the fact is a conformance statement, and it is @@ -1361,6 +1388,24 @@ export function SkillsScreen({ ); }, [selected, showingSkillMd, previewParts]); + /** + * The findings rendered as list items — everything except the two that are + * stated in prose above the list. + * + * Derived once and used for BOTH the "is there a list" decision and the list + * itself. Deciding on `issues` while rendering the filtered set is how an + * entry whose only finding is a banner one ended up showing an empty findings + * container instead of "no structural issues". + */ + const listedIssues = useMemo( + () => + issues.filter( + (issue) => + issue.code !== "dynamic-resources" && issue.code !== "duplicate-name", + ), + [issues], + ); + const errorCount = issues.filter((i) => i.severity === "error").length; const warningCount = issues.length - errorCount; @@ -1559,7 +1604,25 @@ export function SkillsScreen({ integrity cannot be verified. )} - {issues.length === 0 ? ( + {/* A name collision is a fact about the LISTING rather + than about this entry, so it is stated in prose at the + top of the section and filtered out of the findings + list below — the same treatment, and the same reason, as + `dynamic-resources`: the same fact twice, once as a + banner and once as a bare code, reads as two findings. + It leads the section because it changes how everything + under it should be read — these are the findings for + ONE of two skills the server named the same thing. */} + {collision && ( + + {collision.message} + + )} + {listedIssues.length === 0 ? ( // Titled for the check it actually summarises. Now that // every verdict renders in this one section, an // unqualified "Conforms" sits directly above a red digest @@ -1570,30 +1633,27 @@ export function SkillsScreen({ ) : ( - {issues - // The banner above already states this one, in prose. - .filter((issue) => issue.code !== "dynamic-resources") - .map((issue, index) => ( - - - {issue.message} - {issue.resourceUri && ( - {issue.resourceUri} - )} - - - ))} + {listedIssues.map((issue, index) => ( + + + {issue.message} + {issue.resourceUri && ( + {issue.resourceUri} + )} + + + ))} )} {/* The frontmatter cross-check renders in Conformance diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index e346368165..6b2aa06961 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -8,6 +8,7 @@ import { base64ToBytes, checkSkillConformance, checkSkillFrontmatterMatch, + checkSkillNameCollisions, getSkillsExtension, isSkillsExtensionSupported, normalizeSkillUri, @@ -978,3 +979,108 @@ describe("checkSkillFrontmatterMatch (#2248)", () => { expect(checkSkillFrontmatterMatch(entry({}), file(""))).toEqual([]); }); }); + +describe("checkSkillNameCollisions (#2248)", () => { + const at = (uri: string, name?: string): SkillEntry => ({ + uri, + frontmatter: name === undefined ? {} : { name, description: "d" }, + resources: [], + }); + + it("reports nothing when every name is distinct", () => { + expect( + checkSkillNameCollisions([ + at("skill://a/SKILL.md", "a"), + at("skill://b/SKILL.md", "b"), + ]).size, + ).toBe(0); + }); + + it("flags both entries of a collision, each naming the other", () => { + // SEP-2640's own shape: two conforming skills whose paths differ but whose + // final segment — and so their name — is the same. + const collisions = checkSkillNameCollisions([ + at("skill://acme/reports/SKILL.md", "reports"), + at("skill://globex/reports/SKILL.md", "reports"), + ]); + expect(collisions.size).toBe(2); + const acme = collisions.get("skill://acme/reports/SKILL.md"); + const globex = collisions.get("skill://globex/reports/SKILL.md"); + expect(acme?.message).toContain("skill://globex/reports/SKILL.md"); + expect(acme?.message).not.toContain("skill://acme/reports/SKILL.md"); + expect(globex?.message).toContain("skill://acme/reports/SKILL.md"); + }); + + it("is a WARNING, because the server did nothing wrong", () => { + // The obligation is on the consumer, not the server. Reporting an error + // would tell a conforming server author their catalog is invalid. + const [issue] = [ + ...checkSkillNameCollisions([ + at("skill://a/reports/SKILL.md", "reports"), + at("skill://b/reports/SKILL.md", "reports"), + ]).values(), + ]; + expect(issue.code).toBe("duplicate-name"); + expect(issue.severity).toBe("warning"); + }); + + it("names every other colliding entry when three share a name", () => { + const collisions = checkSkillNameCollisions([ + at("skill://a/r/SKILL.md", "r"), + at("skill://b/r/SKILL.md", "r"), + at("skill://c/r/SKILL.md", "r"), + ]); + expect(collisions.size).toBe(3); + const first = collisions.get("skill://a/r/SKILL.md"); + expect(first?.message).toContain("skill://b/r/SKILL.md"); + expect(first?.message).toContain("skill://c/r/SKILL.md"); + }); + + it("does not report the SAME skill listed twice as a collision", () => { + // A repeated entry is a different defect from two skills sharing a name, + // and calling it this one would be a wrong diagnosis rather than a missing + // one. Compared on normalized identity, like every other URI comparison. + expect( + checkSkillNameCollisions([ + at("skill://a/r/SKILL.md", "r"), + at("skill://a/x/../r/SKILL.md", "r"), + ]).size, + ).toBe(0); + }); + + it("ignores entries with no name, which is already its own finding", () => { + // Two entries that both omit a name are not "colliding on a name" — there + // is no name — and saying so would bury `missing-name` under a derived + // finding. + expect( + checkSkillNameCollisions([ + at("skill://a/SKILL.md"), + at("skill://b/SKILL.md"), + ]).size, + ).toBe(0); + expect( + checkSkillNameCollisions([ + at("skill://a/SKILL.md", " "), + at("skill://b/SKILL.md", " "), + ]).size, + ).toBe(0); + }); + + it("does not treat names differing only in case as colliding", () => { + // The Agent Skills grammar is lowercase already; normalizing more than the + // grammar does would report a collision the spec considers two names. + expect( + checkSkillNameCollisions([ + at("skill://a/r/SKILL.md", "reports"), + at("skill://b/R/SKILL.md", "Reports"), + ]).size, + ).toBe(0); + }); + + it("reports nothing for an empty or single-entry listing", () => { + expect(checkSkillNameCollisions([]).size).toBe(0); + expect(checkSkillNameCollisions([at("skill://a/SKILL.md", "a")]).size).toBe( + 0, + ); + }); +}); diff --git a/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts b/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts index 8c3eceefd4..7bbf03c20f 100644 --- a/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts +++ b/clients/web/src/test/integration/mcp/inspectorClient-skills.test.ts @@ -121,18 +121,25 @@ describe("Skills extension over a real transport (#2234)", () => { // below — so a modern page missing the envelope surfaces here as a // rejection rather than as a missing property. // - // The fixture pages at two over six skills, so a client that stops - // here sees a third of the catalog. + // The fixture pages at two over eight skills, so a client that stops + // here sees a quarter of the catalog. expect(first.skills).toHaveLength(2); expect(first.nextCursor).toBeDefined(); - const second = await connected.listSkills(first.nextCursor); - expect(second.skills).toHaveLength(2); - expect(second.nextCursor).toBeDefined(); - - const third = await connected.listSkills(second.nextCursor); - expect(third.skills).toHaveLength(2); - expect(third.nextCursor).toBeUndefined(); + // Walked to the end rather than asserting a fixed page count, so + // adding a fixture does not require editing this test — what it pins + // is that the cursor terminates and every page is full but the last. + let cursor = first.nextCursor; + let pages = 1; + let total = first.skills.length; + while (cursor !== undefined) { + const page = await connected.listSkills(cursor); + total += page.skills.length; + pages += 1; + cursor = page.nextCursor; + } + expect(pages).toBe(4); + expect(total).toBe(8); }); it("walks every page through the managed store", async () => { @@ -147,9 +154,13 @@ describe("Skills extension over a real transport (#2234)", () => { "dynamic-report", "stale-manifest", "lying-listing", + // Two skills, one name — the collision case. The walk must keep + // both; collapsing them is the thing SEP-2640 forbids. + "reports", + "reports", "right-name", ]); - expect(store.getPagination()).toEqual({ pageCount: 3 }); + expect(store.getPagination()).toEqual({ pageCount: 4 }); } finally { store.destroy(); } @@ -261,6 +272,40 @@ describe("Skills extension over a real transport (#2234)", () => { expect(report.ok).toBe(true); }); + it("reports a name collision without failing either skill", async () => { + // Both entries are fully conforming: SEP-2640 requires only that the + // segment before /SKILL.md equal the name, which multi-segment paths + // satisfy while sharing a final segment. The obligation is on the + // consumer, so this is a warning and `ok` stays true. + const started = await startSkillsServer(modern); + const connected = await connect(started.url, modern); + const store = new ManagedSkillsState(connected); + try { + const skills = await store.refresh(); + const colliding = skills.filter( + (s) => s.frontmatter.name === "reports", + ); + expect(colliding.map((s) => s.uri).sort()).toEqual([ + "skill://acme/reports/SKILL.md", + "skill://globex/reports/SKILL.md", + ]); + + const reports = await verifySkills(connected, skills); + for (const uri of colliding.map((s) => s.uri)) { + const report = reports.find((r) => r.uri === uri)!; + expect(report.conformance).toEqual([ + expect.objectContaining({ + code: "duplicate-name", + severity: "warning", + }), + ]); + expect(report.ok).toBe(true); + } + } finally { + store.destroy(); + } + }); + it("answers -32602 for a URI that is not a directory resource", async () => { const started = await startSkillsServer(modern); const connected = await connect(started.url, modern); diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index 30f785b87d..525ecd8ac0 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -247,7 +247,8 @@ export type SkillIssueCode = | "size-limit-exceeded" | "frontmatter-absent" | "frontmatter-unparsable" - | "frontmatter-mismatch"; + | "frontmatter-mismatch" + | "duplicate-name"; /** * `error` marks a **MUST** of SEP-2640 that the server broke, so a manifest @@ -836,3 +837,68 @@ export function checkSkillFrontmatterMatch( } return issues; } + +/** + * Findings that can only be computed over the **whole listing**, keyed by the + * entry they belong to (its normalized URI identity). + * + * Today that is exactly one: two entries in a single `skills/list` colliding on + * `frontmatter.name`. {@link checkSkillConformance} structurally cannot report + * it — it sees one entry at a time, and a collision is a property of the pair. + * + * SEP-2640: *"Hosts MUST NOT assume name uniqueness"*, and *"When two entries + * in one listing collide on `name`, hosts MUST disambiguate them — for example + * by their distinguishing path segments — rather than silently discarding or + * preferring one."* + * + * ⚠️ **A collision is a `warning`, not an `error`, and the distinction is the + * whole point of the severity split.** The obligation here is on the *host*, + * not the server: a server may legitimately publish two skills with the same + * name under different paths, and the SEP's own example + * (`acme/billing/refunds`) is exactly that. Reporting it as an error would tell + * a conforming server author their catalog is invalid. What the warning says is + * that a consumer must not collapse the two — which is why the Inspector shows + * each skill's URI beside its name, and now says so rather than leaving the + * reader to notice. + * + * Names are compared **raw**, not trimmed or case-folded. The Agent Skills + * grammar is lowercase already, and a checker that normalized more than the + * grammar does would report a collision between two names the spec considers + * distinct. + */ +export function checkSkillNameCollisions( + entries: readonly SkillEntry[], +): Map { + const byName = new Map(); + for (const entry of entries) { + const name = entry.frontmatter.name; + // An absent name is `missing-name`, reported per entry. Two entries that + // both omit one are not "colliding on a name" — there is no name — and + // saying so would bury the real finding under a derived one. + if (typeof name !== "string" || name.trim() === "") continue; + const group = byName.get(name); + if (group) group.push(entry); + else byName.set(name, [entry]); + } + + const issues = new Map(); + for (const [name, group] of byName) { + // Deduplicated by URI identity first: the SAME skill appearing twice in a + // listing is a repeated entry, not two skills sharing a name, and + // `skills/list` returning it twice is a different defect from the one this + // function reports. + const identities = new Set(group.map((e) => skillUriIdentity(e.uri))); + if (identities.size < 2) continue; + const uris = [...identities].sort(); + for (const entry of group) { + const self = skillUriIdentity(entry.uri); + const others = uris.filter((uri) => uri !== self); + issues.set(self, { + code: "duplicate-name", + severity: "warning", + message: `Another skill in this listing also declares the name "${name}" (${others.join(", ")}). This is legal — a consumer must tell them apart by their URIs rather than collapsing or preferring one.`, + }); + } + } + return issues; +} diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 650ae132d0..a7ee2aac57 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -30,6 +30,7 @@ import type { RequestMetadata } from "./types.js"; import { checkSkillConformance, checkSkillFrontmatterMatch, + checkSkillNameCollisions, skillDisplayName, skillFileBytes, skillUriIdentity, @@ -141,6 +142,12 @@ export async function verifySkills( entries: readonly SkillEntry[], metadata?: RequestMetadata, ): Promise { + // Computed once over the whole set, because a name collision is a property + // of the listing rather than of an entry — `checkSkillConformance` sees one + // at a time and structurally cannot report it. Note this is scoped to the + // entries passed in, so `--method skills/get --verify` on a single skill + // reports no collision: there is no listing to collide within. + const collisions = checkSkillNameCollisions(entries); const reports: SkillVerifyReport[] = []; for (const entry of entries) { // The entry's own SKILL.md, read once and used twice — for its digest and @@ -211,7 +218,11 @@ export async function verifySkills( } } - const conformance = checkSkillConformance(entry); + const collision = collisions.get(skillUriIdentity(entry.uri)); + const conformance = [ + ...checkSkillConformance(entry), + ...(collision ? [collision] : []), + ]; const frontmatter = entryText === undefined ? [] diff --git a/docs/test-servers.md b/docs/test-servers.md index f33f3633bd..3fdca7d6af 100644 --- a/docs/test-servers.md +++ b/docs/test-servers.md @@ -64,7 +64,7 @@ as a missing capability rather than an error. ## Skills (SEP-2640) -`skills-http.json` sets `"skills": true` and serves six skills over three +`skills-http.json` sets `"skills": true` and serves eight skills over four `skills/list` pages. Since [#2248](https://github.com/modelcontextprotocol/inspector/issues/2248) that one flag also declares **`directoryRead: true`** and registers the @@ -88,14 +88,15 @@ It works on **either era**: `skills/list`, `skills/get` and era codec defines, so the SDK's era gate skips them entirely — which is why this fixture, unlike the tasks ones, needs no per-era variant. -Five of the six skills are deliberately awkward, because the checks the Skills -tab runs are untestable without them. Only **three** are outright violations -(`tampered-notes`, `lying-listing`, `wrong-folder` — the three `--verify` fails -on). The other two are subtler and neither is an error on its own: the +Seven of the eight skills are deliberately awkward, because the checks the +Skills tab runs are untestable without them. Only **three** are outright +violations (`tampered-notes`, `lying-listing`, `wrong-folder` — the three +`--verify` fails on). The rest are subtler and none is an error on its own: the `"dynamic"` form is **conforming**, and is here because "legal but unverifiable" -is the case most easily buried; and `stale-manifest`'s entry is fully conforming +is the case most easily buried; `stale-manifest`'s entry is fully conforming too, with the defect living in the disagreement between its manifest and its -directory listing: +directory listing; and the two `reports` skills are both entirely valid, with +the obligation falling on whoever consumes them: | Skill | What it exercises | | --- | --- | @@ -104,6 +105,7 @@ directory listing: | `dynamic-report` | `resources: "dynamic"` — a **legal** form for generated content. No manifest is advertised, so integrity cannot be verified at all; reported as a warning, not an error. | | `wrong-folder` | A URI path segment (`wrong-folder`) that disagrees with `frontmatter.name` (`right-name`), the one structural invariant SEP-2640 states outright. | | `stale-manifest` | A skill that **serves and directory-lists a file its `resources` manifest does not declare**. Its entry is otherwise fully conforming and verifies clean, so the disagreement between the two views is the only defect — and only a directory read can see it. SEP-2640 calls a directory result "a live observation" and says hosts MUST NOT treat it as extending the manifest, so the Directory section marks the extra child **not listed** rather than showing it as one of the skill's files. | +| `acme/reports` + `globex/reports` | **Two conforming skills sharing the name `reports`.** SEP-2640 requires only that the segment before `/SKILL.md` equal `frontmatter.name`, which multi-segment paths satisfy while still sharing a final segment — its own `acme/billing/refunds` example is this shape. Hosts MUST NOT assume name uniqueness and MUST tell the two apart rather than collapsing or preferring one, so the Inspector reports a `duplicate-name` **warning** on both and shows each skill's URI beside its name. Also the only fixture with a multi-segment skill path. | | `lying-listing` | A `skills/list` entry advertising one `description` while the served `SKILL.md` carries another. **Its digest verifies** — a digest is taken over the bytes the server served and says nothing about whether the listing described them honestly — so this is the one violation only the frontmatter cross-check can catch. | Connection Info's **Skills Extension Options** section shows the `directoryRead` diff --git a/test-servers/src/skills.ts b/test-servers/src/skills.ts index 82679e5355..95b1c46824 100644 --- a/test-servers/src/skills.ts +++ b/test-servers/src/skills.ts @@ -25,6 +25,14 @@ * observation" and says hosts MUST NOT treat it as extending the manifest, * so this is the fixture for that rule: the Inspector must show the extra * child as *not listed* rather than as one of the skill's files (#2248). + * - `acme/reports` and `globex/reports` collide on `frontmatter.name`. Both + * are **fully conforming** — SEP-2640 requires only that the segment before + * `/SKILL.md` equal the name, which multi-segment paths satisfy while still + * sharing a final segment, and the SEP's own `acme/billing/refunds` example + * is this shape. The obligation is on the *consumer*: hosts MUST NOT assume + * name uniqueness and MUST tell two same-named skills apart rather than + * collapsing or preferring one. This pair is also the only fixture with a + * multi-segment skill path, which nothing else here exercises (#2248). * - `lying-listing` advertises one `description` in its `skills/list` entry * and serves a different one in its `SKILL.md` — the violation no digest can * catch, because the digest is over the bytes the server served and says @@ -283,6 +291,26 @@ const LYING_MD = skillMd( "# Lying listing\n\nThe description this file carries is not the one the listing advertised.", ); +// Same `name`, different paths — see the module header. Their `SKILL.md` files +// are derived from these objects like every other fixture's, so each entry is +// internally consistent and the ONLY thing to report is the collision. +const ACME_REPORTS_FM: Frontmatter = { + name: "reports", + description: "Build the weekly report from the acme ledger", +}; +const ACME_REPORTS_MD = skillMd( + ACME_REPORTS_FM, + "# Reports (acme)\n\nOne of two skills named `reports`; tell them apart by URI.", +); +const GLOBEX_REPORTS_FM: Frontmatter = { + name: "reports", + description: "Build the weekly report from the globex ledger", +}; +const GLOBEX_REPORTS_MD = skillMd( + GLOBEX_REPORTS_FM, + "# Reports (globex)\n\nThe other skill named `reports`; same name, different server path.", +); + const FIXTURE_SKILLS: FixtureSkill[] = [ { path: "data-analysis", @@ -358,6 +386,28 @@ const FIXTURE_SKILLS: FixtureSkill[] = [ }, ], }, + { + path: "acme/reports", + frontmatter: ACME_REPORTS_FM, + files: [ + { + uri: "skill://acme/reports/SKILL.md", + text: ACME_REPORTS_MD, + mimeType: "text/markdown", + }, + ], + }, + { + path: "globex/reports", + frontmatter: GLOBEX_REPORTS_FM, + files: [ + { + uri: "skill://globex/reports/SKILL.md", + text: GLOBEX_REPORTS_MD, + mimeType: "text/markdown", + }, + ], + }, { path: "wrong-folder", frontmatter: MISMATCHED_FM, From 033600e4cafa2a1c312f5f8ac2cbcf08a226e931 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 19:24:30 -0400 Subject: [PATCH 04/21] fix: address Copilot review round 2 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings, two of them silent holes in the verification itself. **A blob-served SKILL.md skipped the frontmatter check entirely.** The entry's own file was captured as `contents.text`, so a server returning the markdown as a base64 `blob` — a legal `resources/read` shape this module already decodes for the digest — never reached the comparison, and the report still said `ok`. A MUST that quietly did not run. The file is now held as bytes and the text derived from them, which also guarantees the digest and the frontmatter describe one snapshot. **A dynamic skill whose SKILL.md could not be read reported `ok: true`.** It has no manifest rows, so `files` stayed empty, and its only static finding is a warning — a verification that could not be performed was reported as one that passed. The fallback read now records a `read-error` for all three ways it can fail. It is gated on `manifestListsSelf` (by normalized identity) so a self-entry the manifest listed and failed to read is not read, or reported, twice. **A served `.nan` compared equal to a listed `null`.** YAML expresses non-finite numbers, JSON does not, and `JSON.stringify` turns all of them into `null` — so the canonical comparison reported a real mismatch as agreement. Verified against the parser before fixing. They now canonicalize to a form no JSON scalar can equal, and the value is named in the finding. Also: the self-entry match is by normalized identity rather than raw string, which stops a second read and stops this function disagreeing with `checkSkillConformance`; and `SkillsTab.tsx` gains the file header AGENTS.md requires. The four `err instanceof Error` ternaries are one `reasonOf` helper now — extracted because the coverage gate found every one of their non-Error arms uncovered, and one honestly-tested branch beats four ignores. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/tui/src/components/SkillsTab.tsx | 20 +++ clients/web/src/test/core/mcp/skills.test.ts | 37 +++++ .../test/core/mcp/skillsVerification.test.ts | 144 ++++++++++++++++++ core/mcp/skills.ts | 24 +++ core/mcp/skillsVerification.ts | 76 +++++++-- 5 files changed, 288 insertions(+), 13 deletions(-) diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index c583cd1ed8..65857ba993 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -1,3 +1,23 @@ +/** + * The TUI's Skills pane — the SEP-2640 catalog in a terminal (#2248). + * + * **Why the TUI owns a pane rather than reusing the web screen's logic.** It + * does reuse everything that decides an answer: `checkSkillConformance`, + * `checkSkillNameCollisions` and `verifySkills` all live in `core/` and are + * driven identically here, so a verdict cannot differ depending on which client + * you asked. What is local is presentation, and the terminal's constraints are + * genuinely different — two panes in 80 columns, no colour to rely on, and a + * keyboard rather than a pointer. + * + * ⚠️ **Severity is carried by a glyph as well as a colour** (`✓` / `!` / `✗`). + * This pane is read over ssh, inside tmux, and piped through `script(1)`, where + * colour may not survive; a row whose only signal was `red` would then be + * indistinguishable from a clean one. + * + * The pane is shown only when the connected server declares the extension — + * that gate, and the reset that leaves the tab when it goes false, live in + * `App.tsx` because they are navigation concerns rather than this pane's. + */ import React, { useCallback, useEffect, useRef, useState } from "react"; import { Box, Text, useInput, type Key } from "ink"; import { ScrollView, type ScrollViewRef } from "ink-scroll-view"; diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index 6b2aa06961..551f86b3fe 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -975,6 +975,43 @@ describe("checkSkillFrontmatterMatch (#2248)", () => { ).toHaveLength(1); }); + it("does not let a YAML non-finite number match a listing's null", () => { + // `.nan` / `.inf` are YAML values JSON cannot express, and + // `JSON.stringify` turns every one of them into `null` — so a naive + // canonical comparison reported a served `.nan` as EQUAL to a listed + // `null`: a mismatch silently presented as agreement (Copilot). + const issues = checkSkillFrontmatterMatch( + entry({ threshold: null }), + file("threshold: .nan"), + ); + expect(issues).toHaveLength(1); + expect(issues[0].code).toBe("frontmatter-mismatch"); + // The value is named in the finding rather than hidden behind `null`. + expect(issues[0].message).toContain("NaN"); + }); + + it("distinguishes the three non-finite values from one another", () => { + expect( + checkSkillFrontmatterMatch(entry({ x: null }), file("x: .inf")), + ).toHaveLength(1); + // Infinity vs -Infinity: both stringify to `null`, so they would have + // compared equal to each other as well. + const both = checkSkillFrontmatterMatch( + entry({ a: 1, b: 2 }), + file("a: .inf\nb: -.inf"), + ); + expect(both).toHaveLength(2); + expect(both[0].message).toContain("Infinity"); + expect(both[1].message).toContain("-Infinity"); + }); + + it("still matches a null the served file also writes as null", () => { + // The fix must not turn a genuine agreement into a finding. + expect( + checkSkillFrontmatterMatch(entry({ x: null }), file("x: null")), + ).toEqual([]); + }); + it("reports nothing for two empty frontmatters", () => { expect(checkSkillFrontmatterMatch(entry({}), file(""))).toEqual([]); }); diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 96054f8713..db6d7aecf1 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -282,6 +282,150 @@ describe("verifySkills (#2248)", () => { reason: "expired", } as never); + it("runs the frontmatter check when the SKILL.md arrives as a blob", async () => { + // A base64 `blob` is a legal `resources/read` shape, and this module + // already decodes it for the digest. Reading `contents.text` skipped the + // MANDATORY frontmatter comparison for such a server while still reporting + // `ok` (Copilot). + const skillMd = "---\nname: demo\ndescription: Served\n---\n\n# D\n"; + const bytes = new TextEncoder().encode(skillMd); + const skill: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "Listed" }, + resources: [ + { + uri: "skill://demo/SKILL.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + const readResource = vi.fn(async () => ({ + result: { + contents: [ + { + uri: "skill://demo/SKILL.md", + blob: Buffer.from(skillMd, "utf8").toString("base64"), + }, + ], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("verified"); + expect(report.frontmatter).toHaveLength(1); + expect(report.ok).toBe(false); + }); + + it("does not re-read a self-entry written in an equivalent URI form", async () => { + // `checkSkillConformance` accepts a normalized-equivalent self-entry, so a + // raw string comparison here would disagree with it and read the file twice. + const skillMd = "---\nname: demo\ndescription: A demo\n---\n\n# D\n"; + const bytes = new TextEncoder().encode(skillMd); + const skill: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [ + { + uri: "skill://demo/x/../SKILL.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: skillMd }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(readResource).toHaveBeenCalledTimes(1); + expect(report.frontmatter).toEqual([]); + expect(report.ok).toBe(true); + }); + + it("fails a dynamic skill whose SKILL.md cannot be read", async () => { + // A dynamic skill has no manifest rows, so `files` stayed empty and its + // only static finding is a warning — an unreadable SKILL.md therefore + // reported `ok: true` for a skill whose mandatory frontmatter check never + // ran (Copilot). + const skill: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen", description: "Generated" }, + resources: "dynamic", + }; + const readResource = vi.fn(async () => { + throw new Error("gone"); + }); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files).toEqual([ + expect.objectContaining({ + uri: "skill://gen/SKILL.md", + status: "read-error", + reason: "gone", + }), + ]); + expect(report.ok).toBe(false); + }); + + it("fails a dynamic skill whose SKILL.md answers with no matching block", async () => { + const skill: SkillEntry = { + uri: "skill://gen/SKILL.md", + frontmatter: { name: "gen", description: "Generated" }, + resources: "dynamic", + }; + const readResource = vi.fn(async () => ({ + result: { contents: [{ uri: "skill://gen/other.md", text: "x" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0].status).toBe("read-error"); + expect(report.ok).toBe(false); + }); + + it("does not read a failed manifest self-entry a second time", async () => { + // Its failure is already recorded by the manifest loop; the fallback exists + // for a skill whose manifest never listed the file at all. + const skill = await entry(); + const readResource = vi.fn(async () => { + throw new Error("boom"); + }); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + // Two manifest entries, two reads — no third. + expect(readResource).toHaveBeenCalledTimes(2); + expect(report.files).toHaveLength(2); + }); + + it("stringifies a non-Error rejection rather than reading .message off it", async () => { + // A `throw "string"` anywhere in a transport reaches here; reading + // `.message` off one would put `undefined` where the diagnosis belongs. + const skill = await entry(); + const readResource = vi.fn(() => { + // A non-Error rejection is the point of the test. + throw "plainstring"; + }); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files[0]).toMatchObject({ + status: "read-error", + reason: "plainstring", + }); + }); + + it("treats a result whose contents is not an array as no content", async () => { + // A server can return anything; `contents: "nope"` is not a block list, and + // hashing nothing against a digest would be a confident wrong answer. + const skill = await entry(); + const readResource = vi.fn(async () => ({ + result: { contents: "nope" }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.files.every((f) => f.status === "read-error")).toBe(true); + expect(report.ok).toBe(false); + }); + it("re-throws an auth-recovery error instead of recording it per file", async () => { // Not a property of the file in flight: the session's authorization // expired, so every remaining read fails the same way. Absorbing it would diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index 525ecd8ac0..be8d274835 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -561,6 +561,16 @@ function canonicalEntry(entry: SkillEntry): string { /** Object keys sorted recursively; array ORDER is preserved throughout. */ function canonicalize(value: unknown): unknown { + // YAML can express `.nan` and `.inf`; JSON cannot. `JSON.stringify` turns + // every one of them into `null`, so without this a served `x: .nan` would + // compare EQUAL to a listing declaring `x: null` — a mismatch silently + // reported as agreement (Copilot). The listing side arrived over JSON-RPC and + // can never hold a non-finite number, so one appearing here is always a real + // difference. Rendered as an object, which cannot equal any JSON scalar, and + // which names the value in the finding rather than hiding it. + if (typeof value === "number" && !Number.isFinite(value)) { + return { "#non-finite": String(value) }; + } if (Array.isArray(value)) return value.map(canonicalize); if (value === null || typeof value !== "object") return value; return sortKeys(value as Record); @@ -639,6 +649,20 @@ export function textToBytes(text: string): Uint8Array { return new TextEncoder().encode(text); } +/** + * A skill file's bytes as UTF-8 text — the inverse of {@link textToBytes}. + * + * Deliberately **non-fatal**: a `SKILL.md` that is not valid UTF-8 decodes with + * replacement characters rather than throwing. That is the more useful failure, + * because the frontmatter comparison then reports a concrete difference between + * what the listing claimed and what the file actually holds, instead of + * collapsing into "could not decode" and skipping the check the SEP makes + * mandatory. + */ +export function bytesToText(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes); +} + /** * Raw bytes of a `resources/read` blob content block (standard base64). * Uses `atob`, which Node ≥22 and every browser provide, so this stays diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index a7ee2aac57..84e0fdce78 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -28,6 +28,7 @@ import { AuthRecoveryRequiredError } from "../auth/challenge.js"; import type { InspectorClientProtocol } from "./inspectorClientProtocol.js"; import type { RequestMetadata } from "./types.js"; import { + bytesToText, checkSkillConformance, checkSkillFrontmatterMatch, checkSkillNameCollisions, @@ -79,6 +80,18 @@ export interface SkillVerifyReport { ok: boolean; } +/** + * The reason string for a failed read. + * + * One helper rather than the same ternary at each of the four call sites: a + * rejection is not required to be an `Error` — a `throw "string"` anywhere in a + * transport or its dependencies reaches here — and reading `.message` off one + * would put `undefined` where the diagnosis belongs. + */ +function reasonOf(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + /** Result shape of one `resources/read`, narrowed to what a digest needs. */ interface ReadContents { text?: string; @@ -154,11 +167,27 @@ export async function verifySkills( // for the frontmatter cross-check. Reading it twice would double the load // on the server and, worse, could compare a digest against one snapshot // and frontmatter against another. - let entryText: string | undefined; + // + // Held as BYTES, not text. Taking `contents.text` skipped the whole + // frontmatter comparison whenever a server returned the markdown as a + // base64 `blob` — which is a legal `resources/read` shape, and which this + // module already decodes for the digest — so a mandatory check silently did + // not run while the report still said `ok` (Copilot). Deriving the text from + // the same verified bytes also guarantees the digest and the frontmatter + // describe one snapshot. + let entryBytes: Uint8Array | undefined; const files: SkillFileReport[] = []; const manifest = entry.resources === DYNAMIC_RESOURCES ? [] : entry.resources; + const entryIdentity = skillUriIdentity(entry.uri); + // Compared by NORMALIZED identity, like every other URI comparison here — + // `checkSkillConformance` already accepts a manifest self-entry written in + // an equivalent form, so a raw string test would disagree with it and read + // the same file a second time. + const manifestListsSelf = manifest.some( + (resource) => skillUriIdentity(resource.uri) === entryIdentity, + ); for (const resource of manifest) { let contents: ReadContents | undefined; try { @@ -169,7 +198,7 @@ export async function verifySkills( files.push({ uri: resource.uri, status: "read-error", - reason: err instanceof Error ? err.message : String(err), + reason: reasonOf(err), }); continue; } @@ -182,9 +211,6 @@ export async function verifySkills( }); continue; } - if (resource.uri === entry.uri && typeof contents.text === "string") { - entryText = contents.text; - } let bytes: Uint8Array; try { bytes = skillFileBytes(contents); @@ -192,10 +218,11 @@ export async function verifySkills( files.push({ uri: resource.uri, status: "read-error", - reason: err instanceof Error ? err.message : String(err), + reason: reasonOf(err), }); continue; } + if (skillUriIdentity(resource.uri) === entryIdentity) entryBytes = bytes; const verification = await verifySkillResource(resource, bytes); files.push({ uri: resource.uri, ...verification }); } @@ -203,21 +230,44 @@ export async function verifySkills( // A `"dynamic"` skill has no manifest, so the loop above read nothing — // but its SKILL.md is still served and still has to match the frontmatter // the listing advertised. That obligation is not waived by the file set - // being unenumerable; only integrity is. - if (entryText === undefined) { + // being unenumerable; only integrity is. The same applies to a skill whose + // manifest omits its own file. + // + // Gated on `manifestListsSelf` rather than on `entryBytes`, so a self-entry + // the loop already tried and FAILED to read is not read a second time — its + // failure is recorded there. + if (!manifestListsSelf) { + // Recorded as a file result, not swallowed. Because a dynamic skill has + // no manifest rows, `files` would otherwise stay empty and its only static + // finding is a warning — so an unreadable SKILL.md returned `ok: true` + // for a skill whose mandatory frontmatter check never ran (Copilot). + const fail = (reason: string) => + files.push({ uri: entry.uri, status: "read-error", reason }); try { const invocation = await client.readResource(entry.uri, metadata); const contents = contentsFor(invocation.result, entry.uri); - if (typeof contents?.text === "string") entryText = contents.text; + if (!contents) { + fail( + "resources/read returned no content block for this skill's own SKILL.md, so its frontmatter cannot be checked against the listing.", + ); + } else { + try { + entryBytes = skillFileBytes(contents); + } catch (err) { + fail(reasonOf(err)); + } + } } catch (err) { - // Left undefined: the frontmatter check is skipped below. When the - // manifest listed this file the failure is already reported there, and - // when it did not, `manifest-missing-self` is the finding that matters. - // An expired authorization is not that case — see the note above. + // An expired authorization is the one error that is not this file's + // problem — see the note on the function. if (err instanceof AuthRecoveryRequiredError) throw err; + fail(reasonOf(err)); } } + const entryText = + entryBytes === undefined ? undefined : bytesToText(entryBytes); + const collision = collisions.get(skillUriIdentity(entry.uri)); const conformance = [ ...checkSkillConformance(entry), From 6b837273324926540c71dc5793b186644b53e0e0 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 19:34:52 -0400 Subject: [PATCH 05/21] fix: address Copilot review round 3 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two live findings. The review ran against 4fa00d1c — the commit before the round-2 fixes — so its other three were re-reports of already-fixed issues, verified present at HEAD rather than re-fixed. **The TUI printed "Conformance: conforms" directly above "Verification FAILED".** Self-contradictory in the one place a reader looks for a verdict: the static checks pass on a skill whose advertised digest is well-formed, while its bytes do not hash to it. The heading is now "Listing checks", which names what the section covers — the checks against the entry the listing returned, which say nothing about the bytes served. Same problem, and the same resolution, as the web screen's "No structural issues". **The test-server guide claimed every result carries the full modern envelope**, which the new directory handler made untrue: `readDirectoryPage` deliberately emits `resultType` alone, because SEP-2640 states the caching attributes for a modern `skills/list` and says nothing of the kind here, and its one worked example carries `resultType` only. The claim is narrowed to the two `skills/*` results and the exception is documented as the deliberate choice it is. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/tui/__tests__/SkillsTab.test.tsx | 25 +++++++++++++++++++++++- clients/tui/src/components/SkillsTab.tsx | 10 +++++++++- docs/test-servers.md | 24 ++++++++++++++++------- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index 7d3d640817..e4b44f8c42 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -141,6 +141,28 @@ describe("SkillsTab (#2248)", () => { expect(frame).toContain("! gen"); }); + it("does not claim the listing conforms while verification is failing", async () => { + // The two verdicts sat in one pane and contradicted each other: the static + // checks pass on `clean` (its advertised digest is well-formed), while the + // bytes do not hash to it. The heading now names what it actually covers. + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Verification FAILED"); + expect(frame).toContain("Listing checks: no structural issues"); + expect(frame).not.toContain("conforms"); + }); + it("shows the selected skill's URI, description, findings and manifest", () => { const { lastFrame } = render( { const frame = lastFrame() ?? ""; expect(frame).toContain("skill://clean/SKILL.md"); expect(frame).toContain("A clean skill"); - expect(frame).toContain("Conformance: conforms"); + // Named for what it covers: the static checks against the listing. + expect(frame).toContain("Listing checks: no structural issues"); expect(frame).toContain("Manifest (1)"); expect(frame).toContain("SKILL.md"); expect(frame).toContain("(51 B)"); diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index 65857ba993..fb86c8eb95 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -357,9 +357,17 @@ export function SkillsTab({ )} + {/* Named for the checks it actually covers. An unqualified + "conforms" sat directly above "Verification FAILED" in the + same pane and flatly contradicted it — these are the static + checks against the LISTING, and passing them says nothing + about the bytes the server serves (Copilot). Same wording + problem, and the same fix, as the web screen's "No structural + issues". */} - Conformance{issues.length === 0 ? ": conforms" : ":"} + Listing checks + {issues.length === 0 ? ": no structural issues" : ":"} {issues.map((issue, idx) => ( diff --git a/docs/test-servers.md b/docs/test-servers.md index 3fdca7d6af..7acfe797d7 100644 --- a/docs/test-servers.md +++ b/docs/test-servers.md @@ -76,13 +76,23 @@ handler cannot reach it. To exercise the *undeclared* case, connect to any config **without** `"skills"`, where the Inspector must refuse to send the call locally rather than letting the server answer it. -Every result carries the modern base envelope (`resultType` / `ttlMs` / -`cacheScope`). `skills/*` are consumer-owned, so the SDK stamps nothing for -them; without it a 2026-era connection would receive a result missing the -envelope. It is stamped unconditionally rather than per era — the modern leg -builds a fresh server per request, so there is no era to branch on when the -handlers are registered, and on the legacy leg they are three extra members no -codec inspects. +**Both `skills/*` results carry the full modern base envelope** (`resultType` / +`ttlMs` / `cacheScope`). They are consumer-owned methods, so the SDK stamps +nothing for them; without it a 2026-era connection would receive a result +missing the envelope. It is stamped unconditionally rather than per era — the +modern leg builds a fresh server per request, so there is no era to branch on +when the handlers are registered, and on the legacy leg they are three extra +members no codec inspects. + +⚠️ **`resources/directory/read` deliberately carries `resultType` alone.** +SEP-2640 states the caching attributes for a modern `skills/list` in as many +words and says nothing of the kind for this method, whose one worked example +carries `resultType` and nothing else. A fixture sending more than the SEP shows +would make a client that wrongly *required* them look correct, which is the +opposite of what a conformance fixture is for — so `readDirectoryPage` stops +where the spec does, and `ModernDirectoryReadResultSchema` requires exactly as +much. + It works on **either era**: `skills/list`, `skills/get` and `resources/directory/read` are consumer-owned extension methods that neither era codec defines, so the SDK's era gate skips them entirely — which is why From 799763543f9000d8a68feee3da913c4d5b5ed7f1 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 19:55:17 -0400 Subject: [PATCH 06/21] fix: address Copilot review round 4 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight findings. Two were visible in a terminal capture I had already taken and read past. **A digest mismatch rendered with no digests.** `verifySkillResource` sets `reason` for a SIZE mismatch — which short-circuits before hashing — while a digest mismatch carries `expectedDigest` / `actualDigest` and no `reason`, so the TUI printed a bare `✗ notes.md` with nothing to act on. `failureDetail` now falls back to the expected/actual pair, truncated to fit the pane. **A dynamic skill's read failure rendered nowhere.** Its synthetic `read-error` row lives in the report, not the manifest, and the manifest is empty for such a skill by definition — so the pane said only "Verification FAILED". There is a Read failures block for any report file the manifest does not cover, matched on normalized identity. **The non-finite sentinel could be aliased.** A listing whose value genuinely was `{"#non-finite":"NaN"}` canonicalized identically to a served `.nan`, so the round-3 fix moved the bug rather than closing it. Any encoding into the value space can be aliased by a document containing the encoding, so the comparison is structural now (`jsonLikeEqual`) and there is no sentinel at all; `Object.is` gives NaN === NaN while keeping ±Infinity distinct. Serialization is used only for the message. **`parseSkillFrontmatter("null")` returned `{ fields: {} }`,** contradicting its own non-mapping contract: an explicit null scalar parses to exactly what an empty block does. They are told apart by the source, since the parsed value cannot. Also: the `files` doc comment still claimed it is empty for a dynamic skill, which the round-2 fix made untrue; and three inline Mantine elements with two static styling props each are extracted to `.withProps()` constants, per the convention the rest of the file follows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/tui/__tests__/SkillsTab.test.tsx | 82 +++++++++++++++++++ clients/tui/src/components/SkillsTab.tsx | 63 +++++++++++++- .../screens/SkillsScreen/SkillsScreen.tsx | 37 ++++++--- .../web/src/test/core/mcp/skillFile.test.ts | 27 ++++++ clients/web/src/test/core/mcp/skills.test.ts | 23 ++++++ core/mcp/skillFile.ts | 28 ++++++- core/mcp/skills.ts | 75 +++++++++++++---- core/mcp/skillsVerification.ts | 11 ++- 8 files changed, 313 insertions(+), 33 deletions(-) diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index e4b44f8c42..e8cf89ee7a 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -670,6 +670,88 @@ describe("SkillsTab (#2248)", () => { expect(lastFrame() ?? "").toContain("skill://acme/reports/SKILL.md"); }); + it("shows the digests for a mismatch, not just the failed mark", async () => { + // `verifySkillResource` sets `reason` for a SIZE mismatch but not a digest + // one, so a pane rendering only `reason` left a bare `✗` with no diagnosis + // — a failed verification the reader cannot act on (Copilot). + // The declared size must be RIGHT, or the cheaper size cross-check + // short-circuits before hashing and reports its own `reason` instead — + // which is the path that already rendered. + const digestOnly: SkillEntry = { + ...clean, + resources: [ + { + uri: "skill://clean/SKILL.md", + digest: CLEAN_DIGEST, + size: textToBytes(SKILL_MD).byteLength, + }, + ], + }; + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Verification FAILED"); + // Truncated to keep the line inside a narrow pane; the CLI report carries + // the digests in full. + expect(frame).toMatch(/expected sha256:0+…/); + expect(frame).toMatch(/got sha256:[0-9a-f]+…/); + }); + + it("shows the reason for a size mismatch, which carries no digest", async () => { + // The other arm: a length disagreement fails before the hash, so there is + // no actual digest to print and the reason is the whole diagnosis. + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + expect(lastFrame() ?? "").toContain( + "Manifest declares 51 bytes but the fetched file is 52.", + ); + }); + + it("shows a read failure the manifest does not cover", async () => { + // A dynamic skill has no manifest rows, so the synthetic read-error row + // `verifySkills` records for its own SKILL.md was rendered nowhere and the + // pane said only "Verification FAILED". + const { lastFrame, stdin } = render( + , + ); + stdin.write(ENTER); + await tick(); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Read failures:"); + expect(frame).toContain("SKILL.md"); + expect(frame).toContain("upstream gone"); + expect(frame).toContain("Verification FAILED"); + }); + it("shows the details footer only when the details pane is focused", () => { const unfocused = render( (d ? `${d.slice(0, 23)}…` : "—"); + return `expected ${short(file.expectedDigest)}, got ${short(file.actualDigest)}`; +} + /** The file name a manifest URI ends in, for a list that must fit 40 columns. */ function fileNameOf(uri: string): string { const cut = uri.lastIndexOf("/"); @@ -254,6 +272,14 @@ export function SkillsTab({ selectedSkill && selectedSkill.resources !== DYNAMIC_RESOURCES ? selectedSkill.resources : []; + // Compared on normalized identity, like every other URI comparison here, so a + // manifest entry written in an equivalent form is not reported twice. + const manifestIdentities = new Set( + manifest.map((resource) => skillUriIdentity(resource.uri)), + ); + const extraReportFiles = (activeReport?.files ?? []).filter( + (file) => !manifestIdentities.has(skillUriIdentity(file.uri)), + ); return ( @@ -410,15 +436,48 @@ export function SkillsTab({ ({resource.size} B) ) : null} - {fileReport?.reason && ( + {fileReport && failureDetail(fileReport) && ( - {fileReport.reason} + {failureDetail(fileReport)} )} ); })} + {/* A report can carry a file the MANIFEST does not — a dynamic + skill has no rows at all, yet a failed read of its own + SKILL.md is recorded so the failure is visible. Rendering only + manifest rows left "Verification FAILED" with no diagnosis + anywhere on screen (Copilot). */} + {extraReportFiles.length > 0 && ( + <> + + Read failures: + + {extraReportFiles.map((file, idx) => ( + + + + {FILE_MARK[file.status] ?? "?"}{" "} + + {fileNameOf(file.uri)} + + {failureDetail(file) && ( + + {failureDetail(file)} + + )} + + ))} + + )} + {activeReport && activeReport.frontmatter.length > 0 && ( <> diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 2431c2aa46..39eae21ada 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -380,6 +380,26 @@ const ResourceNameCaption = Text.withProps({ maw: "50%", }); +/** A read that failed, in the Directory section. */ +const ReadFailureAlert = Alert.withProps({ + color: "red", + variant: "light", + title: "Read failed", +}); + +/** + * The directory-vs-manifest divergence banner. Yellow: the server is not + * necessarily wrong — its listing may simply be newer than the held entry. + */ +const UnlistedChildrenAlert = Alert.withProps({ + color: "yellow", + variant: "light", + title: "This directory lists files the entry does not", +}); + +/** The em dash standing in for a verdict that does not apply to a row. */ +const NoVerdictText = Text.withProps({ size: "xs", c: "dimmed" }); + const IssueStack = Stack.withProps({ gap: "xs", }); @@ -1984,9 +2004,7 @@ export function SkillsScreen({ {directoryError !== undefined && ( - - {directoryError} - + {directoryError} )} {/* Stated in prose the first time the two views disagree, because the per-row chip alone does not say @@ -1994,12 +2012,7 @@ export function SkillsScreen({ re-approval" is what SEP-2640 asks a host to present here, rather than a read error. */} {unlistedChildren.length > 0 && ( - + The server is serving {unlistedChildren.length} file {unlistedChildren.length === 1 ? "" : "s"} here that the held skills/list entry does not @@ -2009,7 +2022,7 @@ export function SkillsScreen({ verification failure equivalent to a digest mismatch. Re-fetch the entry with skills/get to see whether the skill has changed. - + )} {directoryChildren !== undefined && (directoryChildren.length === 0 ? ( @@ -2074,9 +2087,7 @@ export function SkillsScreen({ {child.mimeType ?? "—"} {isDir || isDynamic ? ( - - — - + — ) : ( { expect(parseSkillFrontmatter("# just a comment")).toEqual({ fields: {} }); }); + it("reports an explicit null scalar as an error, not as no fields", () => { + // `null` and `~` parse to the same value an EMPTY block does, but only the + // empty one is a degenerate mapping — returning `{ fields: {} }` for an + // explicit null scalar contradicts this function's own contract (Copilot). + for (const src of ["null", "~", " null ", "# lead\nnull"]) { + expect(parseSkillFrontmatter(src)).toEqual({ + error: expect.stringContaining("mapping"), + }); + } + }); + + it("still reads a comment-only block as a mapping of no fields", () => { + // The distinction is made on the SOURCE, so this must not regress. + expect(parseSkillFrontmatter("# just a comment\n\n # another")).toEqual({ + fields: {}, + }); + }); + + it("does not mistake a leading # inside a value for a comment", () => { + // Comments are stripped only at the start of a line; a `#` inside a value + // is part of it, and treating it as a comment would call a block with real + // content empty. + expect(parseSkillFrontmatter('name: "#hashtag"')).toEqual({ + fields: { name: "#hashtag" }, + }); + }); + it("reports a scalar block as an error rather than as no fields", () => { // `just a string` parses successfully as a scalar. Reporting it as an // empty mapping would present a malformed file as one that merely omitted diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index 551f86b3fe..67132a1192 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -1005,6 +1005,29 @@ describe("checkSkillFrontmatterMatch (#2248)", () => { expect(both[1].message).toContain("-Infinity"); }); + it("cannot be fooled by a listing that looks like an encoding", () => { + // The regression this guards: encoding non-finite numbers as a sentinel + // object let a listing whose value genuinely WAS that object alias it and + // match a served `.nan` (Copilot). The comparison is structural now, so + // there is no encoding to alias. + const issues = checkSkillFrontmatterMatch( + entry({ x: { "#non-finite": "NaN" } }), + file("x: .nan"), + ); + expect(issues).toHaveLength(1); + expect(issues[0].code).toBe("frontmatter-mismatch"); + }); + + it("still matches a listing object that equals the served mapping", () => { + // …and the guard must not make a genuine agreement look like a difference. + expect( + checkSkillFrontmatterMatch( + entry({ x: { "#non-finite": "NaN" } }), + file('x:\n "#non-finite": NaN'), + ), + ).toEqual([]); + }); + it("still matches a null the served file also writes as null", () => { // The fix must not turn a genuine agreement into a finding. expect( diff --git a/core/mcp/skillFile.ts b/core/mcp/skillFile.ts index c49b7fe023..297591b2f2 100644 --- a/core/mcp/skillFile.ts +++ b/core/mcp/skillFile.ts @@ -93,6 +93,21 @@ export type ParsedFrontmatter = * everything. An *empty* block (`fields: {}`) is a different fact and is * reported as a successful parse of nothing. */ +/** + * Whether a frontmatter block holds anything but whitespace and comments. + * + * Comments are stripped only from the start of a line: a `#` inside a value is + * part of that value, and treating it as a comment would call a block with real + * content empty. That is the safe direction — mistaking content for emptiness + * here would turn a malformed document back into "no fields", which is the bug + * this exists to prevent. + */ +function hasContent(yamlText: string): boolean { + return yamlText + .split("\n") + .some((line) => line.trim() !== "" && !line.trimStart().startsWith("#")); +} + export function parseSkillFrontmatter(yamlText: string): ParsedFrontmatter { let parsed: unknown; try { @@ -100,9 +115,16 @@ export function parseSkillFrontmatter(yamlText: string): ParsedFrontmatter { } catch (err) { return { error: err instanceof Error ? err.message : String(err) }; } - // `null` is what an empty (or comment-only) block parses to — a real, if - // degenerate, mapping of no fields rather than a malformed one. - if (parsed === null || parsed === undefined) return { fields: {} }; + // An empty (or comment-only) block parses to `null`, and so does an explicit + // `null` / `~` scalar — but only the first is a degenerate mapping of no + // fields. The second is a non-mapping document, and returning `{ fields: {} }` + // for it contradicts this function's own contract (Copilot). They are told + // apart by the SOURCE, since the parsed value cannot distinguish them. + if (parsed === null || parsed === undefined) { + return hasContent(yamlText) + ? { error: "Frontmatter is not a YAML mapping of fields." } + : { fields: {} }; + } if (typeof parsed !== "object" || Array.isArray(parsed)) { return { error: "Frontmatter is not a YAML mapping of fields.", diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index be8d274835..d33847b9b6 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -561,16 +561,6 @@ function canonicalEntry(entry: SkillEntry): string { /** Object keys sorted recursively; array ORDER is preserved throughout. */ function canonicalize(value: unknown): unknown { - // YAML can express `.nan` and `.inf`; JSON cannot. `JSON.stringify` turns - // every one of them into `null`, so without this a served `x: .nan` would - // compare EQUAL to a listing declaring `x: null` — a mismatch silently - // reported as agreement (Copilot). The listing side arrived over JSON-RPC and - // can never hold a non-finite number, so one appearing here is always a real - // difference. Rendered as an object, which cannot equal any JSON scalar, and - // which names the value in the finding rather than hiding it. - if (typeof value === "number" && !Number.isFinite(value)) { - return { "#non-finite": String(value) }; - } if (Array.isArray(value)) return value.map(canonicalize); if (value === null || typeof value !== "object") return value; return sortKeys(value as Record); @@ -760,6 +750,65 @@ export async function verifySkillResource( }; } +/** + * Structural equality for JSON-like values, with YAML's extra scalars handled. + * + * ⚠️ **Comparison is structural rather than serialized, and that is the point.** + * `JSON.stringify` is not injective over what a YAML parser produces: `.nan`, + * `.inf` and `-.inf` all serialize to `null`, so a served `x: .nan` compared + * EQUAL to a listing declaring `x: null` — and to each other. An earlier fix + * encoded non-finite numbers as a sentinel object, which merely moved the + * problem: a listing whose value genuinely *was* that object aliased the + * sentinel and matched a served `.nan` (Copilot). Any encoding into the value + * space can be aliased by a document containing the encoding, so there is no + * sentinel here at all. + * + * `Object.is` on the number path is what makes it work: it holds `NaN` equal to + * `NaN`, keeps `Infinity` and `-Infinity` distinct, and never equates either + * with `null`. + */ +function jsonLikeEqual(a: unknown, b: unknown): boolean { + if (typeof a === "number" || typeof b === "number") return Object.is(a, b); + if (a === null || b === null) return a === b; + if (typeof a !== "object" || typeof b !== "object") return Object.is(a, b); + const aArray = Array.isArray(a); + if (aArray !== Array.isArray(b)) return false; + if (aArray) { + const x = a as unknown[]; + const y = b as unknown[]; + // Array ORDER is significant — a YAML sequence is ordered. + return x.length === y.length && x.every((v, i) => jsonLikeEqual(v, y[i])); + } + const x = a as Record; + const y = b as Record; + const keys = Object.keys(x); + // Key order is not meaningful in either JSON or YAML, so only the key SET and + // the values matter. + return ( + keys.length === Object.keys(y).length && + keys.every((k) => Object.hasOwn(y, k) && jsonLikeEqual(x[k], y[k])) + ); +} + +/** + * A frontmatter value as it should READ in a finding. + * + * `JSON.stringify` renders every non-finite number as `null`, which would print + * "the listing says null but the served file says null" for a real difference. + * Only the display is special-cased; the comparison above never goes through a + * string, so this cannot reintroduce an aliasing bug. + */ +function displayValue(value: unknown): string { + if (typeof value === "number" && !Number.isFinite(value)) { + return String(value); + } + return JSON.stringify(canonicalize(value), (_key, member: unknown) => + typeof member === "number" && !Number.isFinite(member) + ? `<${String(member)}>` + : member, + ); +} + /** * Compare the fetched `SKILL.md`'s own frontmatter against the frontmatter the * entry advertised, field by field — the SEP-2640 obligation a digest cannot @@ -848,13 +897,11 @@ export function checkSkillFrontmatterMatch( }); continue; } - const listedJson = JSON.stringify(canonicalize(listed)); - const servedJson = JSON.stringify(canonicalize(served)); - if (listedJson !== servedJson) { + if (!jsonLikeEqual(listed, served)) { issues.push({ code: "frontmatter-mismatch", severity: "error", - message: `Field "${field}" differs: the listing says ${listedJson} but the served SKILL.md says ${servedJson}.`, + message: `Field "${field}" differs: the listing says ${displayValue(listed)} but the served SKILL.md says ${displayValue(served)}.`, resourceUri: entry.uri, }); } diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 84e0fdce78..6bb2f96df0 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -67,7 +67,16 @@ export interface SkillVerifyReport { * frontmatter discrepancy. */ frontmatter: SkillIssue[]; - /** One entry per manifest file, in manifest order. Empty for `"dynamic"`. */ + /** + * One entry per manifest file, in manifest order. + * + * ⚠️ **Not necessarily empty for a `"dynamic"` skill.** Such a skill has no + * manifest rows, but a failed read of its own `SKILL.md` — the file the + * mandatory frontmatter comparison needs — is recorded here as a synthetic + * `read-error` row against the entry's URI, so the failure is visible and + * fails the report rather than passing silently. A consumer must not assume + * `files` mirrors the manifest one-for-one (Copilot). + */ files: SkillFileReport[]; /** * False when anything the SEP makes a MUST was broken: an error-severity From 916b08ba933215642241510a160f3471305e9ec9 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 20:10:23 -0400 Subject: [PATCH 07/21] fix: address Copilot review round 5 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **A cyclic YAML alias crashed the tool.** `meta: &m [*m]` parses cleanly into a self-referential array, and the frontmatter comparison is a recursive walk, so a hostile server could take down `--verify` or the TUI with a stack overflow instead of receiving a finding. Verified before fixing: it really did raise `RangeError: Maximum call stack size exceeded`. Rejected at the parse, so no consumer has to be cycle-safe on its own, and it becomes an ordinary `frontmatter-unparsable` finding. Detection is per-PATH rather than per-graph — `seen` unwinds on the way back up — so an alias reused across siblings, which is ordinary YAML and represents fine in JSON, is not mistaken for a cycle. A depth bound closes the same hole by its other door: a legal, acyclic but absurdly nested document exhausts the stack just as well, and a cycle check alone passes it. **The web frontmatter check was gated on the presentation MIME.** It ran off `previewParts`, which exists only when the displayed type is recognized as markdown — so a `SKILL.md` a server typed `text/plain`, or served as a base64 blob, skipped a mandatory comparison while the pane still read as clean. It now decodes the same fetched bytes the digest is taken over, which also stops the two answers describing different derivations of the payload. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../SkillsScreen/SkillsScreen.test.tsx | 55 ++++++++++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 32 +++++++---- .../web/src/test/core/mcp/skillFile.test.ts | 44 ++++++++++++++ core/mcp/skillFile.ts | 57 +++++++++++++++++++ 4 files changed, 178 insertions(+), 10 deletions(-) diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index 90cee94dfc..3062735b9f 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -2324,6 +2324,61 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { ).toBeInTheDocument(); }); + it("still checks a SKILL.md the server typed as something other than markdown", async () => { + // The check was gated on the DISPLAY mime, so a server labelling its + // SKILL.md `text/plain` skipped a mandatory comparison while the report + // still read as clean (Copilot). It runs against the fetched bytes now. + const user = userEvent.setup(); + const lying: SkillEntry = { + ...CLEAN_SKILL, + frontmatter: { + name: "data-analysis", + description: "Not what the file says", + }, + }; + renderWithMantine( + ({ + text: skillMdFor(CLEAN_FM), + mimeType: "text/plain", + }))} + />, + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Conformance/ })); + await waitFor(() => + expect( + screen.getByTestId("skill-frontmatter-issues"), + ).toBeInTheDocument(), + ); + }); + + it("checks a SKILL.md served as a base64 blob", async () => { + // Same gap by its other door: a blob never produced `previewParts`. + const user = userEvent.setup(); + const lying: SkillEntry = { + ...CLEAN_SKILL, + frontmatter: { name: "data-analysis", description: "Disagrees" }, + }; + renderWithMantine( + ({ + blob: btoa(skillMdFor(CLEAN_FM)), + mimeType: "application/octet-stream", + }))} + />, + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Conformance/ })); + await waitFor(() => + expect( + screen.getByTestId("skill-frontmatter-issues"), + ).toBeInTheDocument(), + ); + }); + it("reports nothing when the served frontmatter agrees", async () => { const user = userEvent.setup(); renderWithMantine(); diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 39eae21ada..ea24a71ac6 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -31,6 +31,7 @@ import { checkSkillConformance, checkSkillFrontmatterMatch, checkSkillNameCollisions, + bytesToText, skillDisplayName, skillFileBytes, skillEntriesMatch, @@ -1396,17 +1397,28 @@ export function SkillsScreen({ * first time a file happened to be fetched. */ const frontmatterIssues = useMemo(() => { - if (!selected || !showingSkillMd || previewParts === undefined) return []; - // Reconstructed from the split rather than re-derived from the payload, so - // the check reads exactly the bytes the Frontmatter section displays. - if (previewParts.frontmatter === undefined) { - return checkSkillFrontmatterMatch(selected, previewParts.body); + if (!selected || !showingSkillMd || preview === undefined) return []; + // Run against the **raw fetched bytes**, not against `previewParts`. + // + // `previewParts` is a *presentation* value: it only exists when the + // displayed MIME is recognized as markdown, so a `SKILL.md` a server + // labelled `text/plain` — or anything else — skipped this check entirely + // while the report still read as clean (Copilot). The SEP makes the + // comparison mandatory for the skill's own file regardless of how the + // server typed it, and `showingSkillMd` already establishes that this IS + // that file. Decoding the same bytes the digest is taken over also keeps + // the two answers describing one payload rather than two derivations of it. + let text: string; + try { + text = bytesToText(skillFileBytes(preview)); + } catch { + // Neither text nor blob: there are no bytes to compare, and the file + // viewer already reports the empty response. Inventing a frontmatter + // finding here would name the wrong defect. + return []; } - return checkSkillFrontmatterMatch( - selected, - `---\n${previewParts.frontmatter}\n---\n\n${previewParts.body}`, - ); - }, [selected, showingSkillMd, previewParts]); + return checkSkillFrontmatterMatch(selected, text); + }, [selected, showingSkillMd, preview]); /** * The findings rendered as list items — everything except the two that are diff --git a/clients/web/src/test/core/mcp/skillFile.test.ts b/clients/web/src/test/core/mcp/skillFile.test.ts index 35223215ec..f7399c9f0d 100644 --- a/clients/web/src/test/core/mcp/skillFile.test.ts +++ b/clients/web/src/test/core/mcp/skillFile.test.ts @@ -137,6 +137,50 @@ describe("parseSkillFrontmatter (#2248)", () => { }); }); + it("rejects a cyclic YAML alias instead of crashing on it", () => { + // A YAML document is a graph and JSON is a tree: `&m [*m]` parses cleanly + // into a self-referential array, and the field-by-field comparison is a + // recursive walk — so this crashed `--verify` and the TUI with a stack + // overflow rather than producing a finding. A hostile server taking the + // tool down is a worse outcome than any wrong verdict (Copilot). + const parsed = parseSkillFrontmatter("meta: &m [*m]"); + expect(parsed).toEqual({ error: expect.stringContaining("cyclic") }); + }); + + it("rejects a cycle through a mapping, not only an array", () => { + expect(parseSkillFrontmatter("a: &a\n self: *a")).toEqual({ + error: expect.stringContaining("cyclic"), + }); + }); + + it("accepts a value that merely appears twice as a sibling", () => { + // An alias reused across siblings is ordinary YAML and represents fine in + // JSON — detection has to be per-path, not per-graph, or this would be + // reported as a cycle. + expect(parseSkillFrontmatter("base: &b [1, 2]\nx: *b\ny: *b")).toEqual({ + fields: { base: [1, 2], x: [1, 2], y: [1, 2] }, + }); + }); + + it("rejects a frontmatter nested past the depth bound", () => { + // The other door to the same crash: legal, acyclic, and still deep enough + // to exhaust the stack, which a cycle check alone would let through. + const deep = + "a:\n" + + Array.from({ length: 80 }, (_, i) => `${" ".repeat(i + 1)}a:`).join( + "\n", + ); + expect(parseSkillFrontmatter(deep)).toEqual({ + error: expect.stringContaining("nests deeper"), + }); + }); + + it("accepts ordinary nesting well inside the bound", () => { + expect(parseSkillFrontmatter("a:\n b:\n c: 1")).toEqual({ + fields: { a: { b: { c: 1 } } }, + }); + }); + it("reports invalid YAML with the parser's own message", () => { const parsed = parseSkillFrontmatter("a: [1,"); expect("error" in parsed && parsed.error.length > 0).toBe(true); diff --git a/core/mcp/skillFile.ts b/core/mcp/skillFile.ts index 297591b2f2..76a098d823 100644 --- a/core/mcp/skillFile.ts +++ b/core/mcp/skillFile.ts @@ -108,6 +108,59 @@ function hasContent(yamlText: string): boolean { .some((line) => line.trim() !== "" && !line.trimStart().startsWith("#")); } +/** + * Depth bound for a parsed frontmatter graph. + * + * Generous next to anything a real `SKILL.md` carries — the format's own fields + * are flat — and far below the stack the comparison walk would need. + */ +const MAX_FRONTMATTER_DEPTH = 64; + +/** + * Why a parsed frontmatter cannot be compared, or `undefined` when it can. + * + * ⚠️ **A YAML document is a graph, not a tree, and JSON is a tree.** An alias + * can refer to its own ancestor — `meta: &m [*m]` parses without error into a + * self-referential array — and the field-by-field comparison is a recursive + * walk, so such a value crashed `--verify` and the TUI with a stack overflow + * instead of producing a finding. That is a hostile server taking the tool + * down, so it is rejected here, at the parse, rather than defended against at + * every consumer (Copilot). + * + * The depth bound closes the same hole by its other door: a legal, acyclic but + * absurdly nested document would exhaust the stack just as effectively, and a + * cycle check alone would pass it. + * + * Detection is per-PATH, not per-graph: `seen` is added on the way down and + * removed on the way back up, so a value that merely appears twice as a sibling + * — which YAML aliases make ordinary and which JSON represents perfectly well — + * is not mistaken for a cycle. + */ +function jsonGraphError( + value: unknown, + seen: Set, + depth: number, +): string | undefined { + if (typeof value !== "object" || value === null) return undefined; + if (depth > MAX_FRONTMATTER_DEPTH) { + return `Frontmatter nests deeper than ${MAX_FRONTMATTER_DEPTH} levels, which cannot be compared field by field.`; + } + const node = value as object; + if (seen.has(node)) { + return "Frontmatter contains a cyclic YAML alias, which has no JSON equivalent and cannot be compared against the listing."; + } + seen.add(node); + const members = Array.isArray(node) + ? (node as unknown[]) + : Object.values(node as Record); + for (const member of members) { + const error = jsonGraphError(member, seen, depth + 1); + if (error) return error; + } + seen.delete(node); + return undefined; +} + export function parseSkillFrontmatter(yamlText: string): ParsedFrontmatter { let parsed: unknown; try { @@ -130,5 +183,9 @@ export function parseSkillFrontmatter(yamlText: string): ParsedFrontmatter { error: "Frontmatter is not a YAML mapping of fields.", }; } + // Checked BEFORE the value escapes this module, so no consumer has to be + // cycle-safe on its own. + const graphError = jsonGraphError(parsed, new Set(), 0); + if (graphError) return { error: graphError }; return { fields: parsed as Record }; } From e5f2fdbb4fb242a84d83487000600fc8ed5aad7a Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 20:19:30 -0400 Subject: [PATCH 08/21] test: pin the reviewer's exact cyclic-alias example `&a [1, *a]` puts the self-reference after a plain value, so a guard that only inspected the head of a sequence would miss it. Already caught; this records that it is. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/web/src/test/core/mcp/skillFile.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/clients/web/src/test/core/mcp/skillFile.test.ts b/clients/web/src/test/core/mcp/skillFile.test.ts index f7399c9f0d..bb9e5bcf50 100644 --- a/clients/web/src/test/core/mcp/skillFile.test.ts +++ b/clients/web/src/test/core/mcp/skillFile.test.ts @@ -147,6 +147,14 @@ describe("parseSkillFrontmatter (#2248)", () => { expect(parsed).toEqual({ error: expect.stringContaining("cyclic") }); }); + it("rejects a cycle that is not the first element", () => { + // `&a [1, *a]` — the self-reference sits after a plain value, so a guard + // that only inspected the head of a sequence would miss it. + expect(parseSkillFrontmatter("a: &a [1, *a]")).toEqual({ + error: expect.stringContaining("cyclic"), + }); + }); + it("rejects a cycle through a mapping, not only an array", () => { expect(parseSkillFrontmatter("a: &a\n self: *a")).toEqual({ error: expect.stringContaining("cyclic"), From d6b1e36e7e237a2a573f3f9a56a168358b47b6b9 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 20:44:51 -0400 Subject: [PATCH 09/21] fix: address Copilot review round 7 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings. **The CLI dropped what it promised to print.** `getSkill` transforms the `skills/get` envelope to a `SkillEntry`, so every other member the loose schema accepted was gone before the CLI saw it — including the `ttlMs` / `cacheScope` SEP-2640 explicitly leaves open. The code contradicted a comment two lines above it saying a CLI whose contract is "print the result" must not reshape one. `getSkillResult` returns the envelope whole; `getSkill` is a one-line unwrap over it, which matches the callers: the UIs want the entry, the CLI wants the result. **The frontmatter verdict and the digest came from different reads.** The preview fetch and the Verify fetch are separate `resources/read` calls, so a resource changing between them could pair a verified digest with a frontmatter verdict for other bytes. `VerificationState.entryText` now holds the SKILL.md as the verification read it, written in the SAME state update as the verdict — a separate write was clobbered, because `write` returns a fresh object and drops anything not named in it. The preview stays the fallback so a reader who has not clicked Verify still gets the check. **The Conformance badge excluded the frontmatter findings** it renders, so it said `0 error(s)` above a red mismatch. Digest and size mismatches stay in their own badge: "the listing is wrong" and "the bytes are wrong" are different answers. **A repeated skill URI collided two React keys** in the TUI list, in a pane whose job is to show both entries. Index-keyed like the manifest rows. The web fixture also gains its collision pair in the served-file lookup — without it they were handed another skill's SKILL.md, which the badge fix correctly began reporting as a frontmatter error. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../cli/__tests__/run-method-skills.test.ts | 34 ++++++- clients/cli/src/handlers/run-method.ts | 16 ++-- clients/tui/__tests__/SkillsTab.test.tsx | 23 +++++ clients/tui/src/components/SkillsTab.tsx | 11 ++- .../SkillsScreen/SkillsScreen.test.tsx | 66 ++++++++++++- .../screens/SkillsScreen/SkillsScreen.tsx | 95 ++++++++++++++++--- core/mcp/inspectorClient.ts | 27 +++++- core/mcp/skillsSchemas.ts | 16 +++- 8 files changed, 257 insertions(+), 31 deletions(-) diff --git a/clients/cli/__tests__/run-method-skills.test.ts b/clients/cli/__tests__/run-method-skills.test.ts index dc4da07a2c..7074e4a0f2 100644 --- a/clients/cli/__tests__/run-method-skills.test.ts +++ b/clients/cli/__tests__/run-method-skills.test.ts @@ -41,6 +41,7 @@ function mockClient(overrides: Record = {}): InspectorClient { getSkillsExtension: vi.fn().mockReturnValue({ directoryRead: true }), listSkills: vi.fn().mockResolvedValue({ skills: [] }), getSkill: vi.fn(), + getSkillResult: vi.fn(), readResourceDirectory: vi.fn(), readResource: vi.fn().mockResolvedValue({ result: { contents: [{ uri: "skill://demo/SKILL.md", text: SKILL_MD }] }, @@ -85,19 +86,21 @@ describe("runMethod skills dispatch (#2248)", () => { // -32602 a declared server returns for a URI it does not serve. const client = mockClient({ getSkillsExtension: vi.fn().mockReturnValue(undefined), - getSkill: vi.fn(), + getSkillResult: vi.fn(), }); await expect( runMethod(client, { method: "skills/get", uri: "skill://x/SKILL.md" }), ).rejects.toMatchObject({ exitCode: EXIT_CODES.USAGE }); - expect(client.getSkill).not.toHaveBeenCalled(); + expect(client.getSkillResult).not.toHaveBeenCalled(); }); it("keeps the { skill } envelope on skills/get", async () => { // The client unwraps it for callers that want the entry; a CLI whose // contract is "print the result" must not quietly reshape the wire form. const entry = await cleanEntry(); - const client = mockClient({ getSkill: vi.fn().mockResolvedValue(entry) }); + const client = mockClient({ + getSkillResult: vi.fn().mockResolvedValue({ skill: entry }), + }); const outcome = await runMethod(client, { method: "skills/get", uri: entry.uri, @@ -105,6 +108,27 @@ describe("runMethod skills dispatch (#2248)", () => { expect(outcome).toMatchObject({ result: { skill: entry } }); }); + it("prints the whole skills/get envelope, not just the entry", async () => { + // SEP-2640 leaves it open whether this result carries `ttlMs`/`cacheScope`, + // so a server may send them — and unwrapping to the entry discarded exactly + // those, from a path whose contract is "print the result" (Copilot). + const entry = await cleanEntry(); + const envelope = { + skill: entry, + resultType: "complete", + ttlMs: 60, + cacheScope: "public", + }; + const client = mockClient({ + getSkillResult: vi.fn().mockResolvedValue(envelope), + }); + const outcome = await runMethod(client, { + method: "skills/get", + uri: entry.uri, + }); + expect(outcome).toMatchObject({ result: envelope }); + }); + it("requires --uri for skills/get", async () => { await expect( runMethod(mockClient(), { method: "skills/get" }), @@ -179,7 +203,9 @@ describe("runMethod skills dispatch (#2248)", () => { it("--verify works on a single skills/get", async () => { const entry = await cleanEntry(); - const client = mockClient({ getSkill: vi.fn().mockResolvedValue(entry) }); + const client = mockClient({ + getSkillResult: vi.fn().mockResolvedValue({ skill: entry }), + }); const outcome = await runMethod(client, { method: "skills/get", uri: entry.uri, diff --git a/clients/cli/src/handlers/run-method.ts b/clients/cli/src/handlers/run-method.ts index c233f9bd1b..67d4ae0130 100644 --- a/clients/cli/src/handlers/run-method.ts +++ b/clients/cli/src/handlers/run-method.ts @@ -352,7 +352,15 @@ export async function runMethod( // not serve — "this server has no Skills support" and "no such skill" // are different answers (Copilot). assertSkillsSupported(inspectorClient, args.method); - const skill = await inspectorClient.getSkill(args.uri, args.metadata); + // The ENVELOPE, not the unwrapped entry. `getSkill` discards every other + // member the result carried — including the `ttlMs` / `cacheScope` that + // SEP-2640 explicitly leaves open — and a CLI whose contract is "print + // the result" must not drop what the server actually sent (Copilot). + const envelope = await inspectorClient.getSkillResult( + args.uri, + args.metadata, + ); + const skill = envelope.skill; if (args.verify) { const reports = await verifySkills( inspectorClient, @@ -368,11 +376,7 @@ export async function runMethod( : { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }), }; } - // The `{ skill }` envelope is restored here because it is what the wire - // carries: `GetSkillResultSchema` unwraps it for callers that want the - // entry, and a CLI whose contract is "print the result" must not quietly - // reshape one. - result = { skill }; + result = envelope; } else if (args.method === "resources/directory/read") { if (!args.uri) { throw new Error( diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index e8cf89ee7a..03638e32c1 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -605,6 +605,29 @@ describe("SkillsTab (#2248)", () => { expect(lastFrame() ?? "").toContain("urn:opaque"); }); + it("renders both rows when a listing repeats a URI", () => { + // A malformed listing can carry the same skill twice, and this pane exists + // to show BOTH — a URI-keyed row would collide them and let React drop or + // reuse one (Copilot). + const dup: SkillEntry = { + uri: "skill://twice/SKILL.md", + frontmatter: { name: "twice", description: "Listed twice" }, + resources: [], + }; + const { lastFrame } = render( + , + ); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Skills (2)"); + expect(frame.match(/twice/g)?.length).toBeGreaterThanOrEqual(2); + }); + it("keys a row by its index when the entry carries no URI", () => { // A URI-less entry is a `malformed-uri` finding this pane reports, so it // must still render a addressable row rather than colliding React keys. diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index 6dc3a93508..7e72574c66 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -333,7 +333,16 @@ export function SkillsTab({ ? "warning" : null; return ( - + // Index-keyed like the manifest and finding rows, and for + // the same reason: a malformed listing can repeat a URI, and + // this pane exists to show BOTH entries — a URI key would + // collide them and let React drop or reuse the wrong row + // (Copilot). + {isSelected ? "▶ " : " "} {worst ? ( diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index 3062735b9f..cd659f7083 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -152,6 +152,9 @@ const ALL_SKILLS = [ MISMATCHED_SKILL, ]; +/** Everything `readFixtureFile` can serve a `SKILL.md` for. */ +const SERVED_SKILLS = [...ALL_SKILLS, ACME, GLOBEX]; + /** * A `resources/read` that serves the fixture bytes for any known URI. A skill's * own `SKILL.md` comes from {@link skillMdFor}, so it agrees with the entry's @@ -161,7 +164,10 @@ const ALL_SKILLS = [ const readFixtureFile = vi.fn(async (uri: string) => { if (uri === "skill://data-analysis/reference.md") return { text: REF_TEXT }; if (uri === "skill://tampered/notes.md") return { text: NOTES_TEXT }; - const owner = ALL_SKILLS.find((skill) => skill.uri === uri); + // Every fixture, not only the four in the default catalog — the collision + // pair is served here too, or it would be handed another skill's SKILL.md and + // report a frontmatter mismatch that the fixture never meant to demonstrate. + const owner = SERVED_SKILLS.find((skill) => skill.uri === uri); return { text: owner ? skillMdFor(owner.frontmatter as Frontmatter) : SELF_TEXT, mimeType: "text/markdown", @@ -2379,6 +2385,64 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { ); }); + it("counts frontmatter findings in the Conformance badge", async () => { + // The findings render inside this section, so counting only the static + // listing issues left the badge saying `0 error(s)` above a red + // `frontmatter-mismatch` — the section contradicting its own output. + const user = userEvent.setup(); + const lying: SkillEntry = { + ...CLEAN_SKILL, + frontmatter: { name: "data-analysis", description: "Disagrees" }, + }; + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await waitFor(() => + expect( + screen.getByRole("button", { name: /Conformance/ }), + ).toHaveTextContent("1 error(s), 0 warning(s)"), + ); + }); + + it("prefers the bytes verification read over the preview read", async () => { + // The two verdicts came from separate `resources/read` calls, so a resource + // that changed between them could pair a verified digest with a frontmatter + // verdict computed for different bytes (Copilot). After Verify, the + // frontmatter check reads what the verification hashed. + const user = userEvent.setup(); + let served = skillMdFor(CLEAN_FM); // agrees with the listing… + const onReadSkillFile = vi.fn(async (uri: string) => { + if (uri === "skill://data-analysis/reference.md") + return { text: REF_TEXT }; + return { text: served, mimeType: "text/markdown" }; + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click(screen.getByRole("button", { name: /Conformance/ })); + await waitFor(() => + expect(screen.getByText("No structural issues")).toBeInTheDocument(), + ); + + // …and then the server starts serving something else. + served = skillMdFor({ ...CLEAN_FM, description: "Changed underneath" }); + await user.click( + screen.getByRole("button", { + name: "Verify skill://data-analysis/SKILL.md", + }), + ); + await waitFor(() => + expect( + screen.getByTestId("skill-frontmatter-issues"), + ).toBeInTheDocument(), + ); + expect( + within(screen.getByTestId("skill-frontmatter-issues")).getByText( + /Changed underneath/, + ), + ).toBeInTheDocument(); + }); + it("reports nothing when the served frontmatter agrees", async () => { const user = userEvent.setup(); renderWithMantine(); diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index ea24a71ac6..8b999d6b79 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -91,6 +91,18 @@ interface VerificationState { */ key: string | null; files: Record; + /** + * The text of the skill's own `SKILL.md` **as the verification read it**. + * + * Held so the frontmatter comparison and the digest describe the *same* + * fetch. They were derived from two separate `resources/read` calls — the + * on-selection preview and the Verify click — so a resource that changed + * between them could pair a verified digest with a frontmatter verdict for + * different bytes (Copilot). Set only when the verified row is the entry's + * own file; the preview remains the fallback until then, since a reader who + * has not clicked Verify should still get the check. + */ + entryText?: string; } /** @@ -869,6 +881,8 @@ export function SkillsScreen({ }); const fileStates = verification.key === manifestKey ? verification.files : {}; + const verifiedEntryText = + verification.key === manifestKey ? verification.entryText : undefined; /** * Verify one manifest ROW. Keyed by row index, not by URI: the checker @@ -893,8 +907,26 @@ export function SkillsScreen({ ); }, []); + /** + * Whether a manifest row IS the skill's own `SKILL.md`. + * + * By normalized identity, like every other URI comparison here — a manifest + * that spells its self-entry equivalently still names the same file. + */ + const isSelfResource = useCallback( + (resource: SkillResource) => + selected !== undefined && + skillUriIdentity(resource.uri) === skillUriIdentity(selected.uri), + [selected], + ); + const verifyRow = useCallback( - async (index: number, resource: SkillResource, key: string) => { + async ( + index: number, + resource: SkillResource, + key: string, + isSelfRow = false, + ) => { // NOTE: opening the Conformance section deliberately does NOT happen // here. `verifyRow` is called once per row by every "Verify all" worker // as it advances, so a batch begun on one skill keeps calling it after @@ -907,7 +939,7 @@ export function SkillsScreen({ // Claimed synchronously, so two verifications of this row are ordered // before either read starts. const attempt = (nextAttempt.current += 1); - const write = (state: FileState) => + const write = (state: FileState, entryText?: string) => setVerification((prev) => { // `null` is the un-adopted initial manifest; any other mismatch is a // continuation from a manifest that has since been invalidated. @@ -917,16 +949,31 @@ export function SkillsScreen({ // finishing last must not overwrite it. const held = files[index]; if (held !== undefined && held.attempt > attempt) return prev; - return { key, files: { ...files, [index]: state } }; + return { + key, + files: { ...files, [index]: state }, + // Carried through explicitly: this returns a FRESH state object, so + // anything not named here is dropped — which silently discarded the + // verified `SKILL.md` text the frontmatter check depends on. + ...(entryText !== undefined + ? { entryText } + : prev.key === key && prev.entryText !== undefined + ? { entryText: prev.entryText } + : {}), + }; }); write({ attempt, status: "pending" }); try { const contents = await onReadSkillFile(resource.uri); - const result = await verifySkillResource( - resource, - skillFileBytes(contents), + const bytes = skillFileBytes(contents); + const result = await verifySkillResource(resource, bytes); + // The entry's own file is captured in the SAME write as its verdict, so + // the frontmatter check reads the very bytes that were just hashed — + // see `VerificationState.entryText`. + write( + { attempt, status: "done", verification: result }, + isSelfRow ? bytesToText(bytes) : undefined, ); - write({ attempt, status: "done", verification: result }); } catch (err) { write({ attempt, @@ -957,7 +1004,7 @@ export function SkillsScreen({ const key = manifestKey; const worker = async (): Promise => { for (let i = next++; i < manifest.length; i = next++) { - await verifyRow(i, manifest[i], key); + await verifyRow(i, manifest[i], key, isSelfResource(manifest[i])); } }; const workers = Math.min(VERIFY_CONCURRENCY, manifest.length); @@ -978,7 +1025,7 @@ export function SkillsScreen({ return next; }), ); - }, [manifest, manifestKey, openConformance, verifyRow]); + }, [manifest, manifestKey, openConformance, verifyRow, isSelfResource]); /** * Put one of the skill's files in the viewer. Driven both by the effect that @@ -1397,7 +1444,13 @@ export function SkillsScreen({ * first time a file happened to be fetched. */ const frontmatterIssues = useMemo(() => { - if (!selected || !showingSkillMd || preview === undefined) return []; + if (!selected) return []; + // Prefer the text the VERIFICATION read, so the digest verdict above and + // this one describe one fetch rather than two. + if (verifiedEntryText !== undefined) { + return checkSkillFrontmatterMatch(selected, verifiedEntryText); + } + if (!showingSkillMd || preview === undefined) return []; // Run against the **raw fetched bytes**, not against `previewParts`. // // `previewParts` is a *presentation* value: it only exists when the @@ -1418,7 +1471,7 @@ export function SkillsScreen({ return []; } return checkSkillFrontmatterMatch(selected, text); - }, [selected, showingSkillMd, preview]); + }, [selected, showingSkillMd, preview, verifiedEntryText]); /** * The findings rendered as list items — everything except the two that are @@ -1438,8 +1491,23 @@ export function SkillsScreen({ [issues], ); - const errorCount = issues.filter((i) => i.severity === "error").length; - const warningCount = issues.length - errorCount; + /** + * Everything the Conformance section reports, for the header badge. + * + * The frontmatter findings render inside this section, so counting only the + * static listing issues left the badge saying `0 error(s)` above a red + * `frontmatter-mismatch` alert — the section contradicting its own output + * (Copilot). Digest and size mismatches stay OUT: they have their own + * `mismatch(es)` badge, because "the listing is wrong" and "the bytes are + * wrong" are different answers and merging them would hide which failed. + */ + const countedIssues = useMemo( + () => [...issues, ...frontmatterIssues], + [issues, frontmatterIssues], + ); + + const errorCount = countedIssues.filter((i) => i.severity === "error").length; + const warningCount = countedIssues.length - errorCount; return ( // `data-*` readiness contract for the headless tab smoke (#2148); see @@ -1940,6 +2008,7 @@ export function SkillsScreen({ index, resource, manifestKey, + isSelfResource(resource), ); }} > diff --git a/core/mcp/inspectorClient.ts b/core/mcp/inspectorClient.ts index 3eea39fb54..be9bba98c8 100644 --- a/core/mcp/inspectorClient.ts +++ b/core/mcp/inspectorClient.ts @@ -146,13 +146,14 @@ import { import { buildClientExtensions } from "./extensions.js"; import { DirectoryReadResultSchema, - GetSkillResultSchema, + GetSkillEnvelopeSchema, ListSkillsResultSchema, ModernDirectoryReadResultSchema, ModernListSkillsResultSchema, RESOURCES_DIRECTORY_READ_METHOD, SKILLS_EXTENSION_KEY, type DirectoryReadResult, + type GetSkillEnvelope, SKILLS_GET_METHOD, SKILLS_LIST_METHOD, type SkillEntry, @@ -5609,11 +5610,28 @@ export class InspectorClient extends InspectorClientEventTarget { /** * One skill entry by URI (`skills/get`, SEP-2640). The result envelope is - * required — `GetSkillResultSchema` unwraps `{ skill }` and rejects an entry + * required — `GetSkillEnvelopeSchema` requires `{ skill }` and rejects an entry * returned inline, so a non-conforming shape fails here rather than being * silently normalized past the conformance checks. */ async getSkill(uri: string, metadata?: RequestMetadata): Promise { + return (await this.getSkillResult(uri, metadata)).skill; + } + + /** + * `skills/get` as the server sent it — the `{ skill }` envelope **and any + * other members it carried**. + * + * Separate from {@link getSkill} because the callers differ: the UIs want the + * entry, while the CLI prints the result and must not reshape it. SEP-2640 + * explicitly leaves open whether this result carries `ttlMs` / `cacheScope`, + * so a server may send them — and unwrapping to the entry discards exactly + * those (Copilot). + */ + async getSkillResult( + uri: string, + metadata?: RequestMetadata, + ): Promise { if (!this.client) { throw new Error("Client is not connected"); } @@ -5622,14 +5640,13 @@ export class InspectorClient extends InspectorClientEventTarget { uri, ...(effectiveMeta ? { _meta: effectiveMeta } : {}), }; - // `GetSkillResultSchema` unwraps the envelope, so there is nothing to - // unwrap here. + // The envelope is returned whole; `getSkill` is the one that unwraps. try { return await this.invokeMcpClient( () => this.client!.request( { method: SKILLS_GET_METHOD, params }, - GetSkillResultSchema, + GetSkillEnvelopeSchema, this.getRequestOptions(this.progressTokenOf(metadata)), ), { method: SKILLS_GET_METHOD }, diff --git a/core/mcp/skillsSchemas.ts b/core/mcp/skillsSchemas.ts index e13a6fcce5..4a36ec87fa 100644 --- a/core/mcp/skillsSchemas.ts +++ b/core/mcp/skillsSchemas.ts @@ -167,7 +167,21 @@ export const ModernListSkillsResultSchema = ListSkillsResultSchema.extend({ * — which is exactly the failure this extension's support exists to *report*. * A server that returns the entry inline now fails the parse, loudly. */ -const GetSkillEnvelopeSchema = z.looseObject({ skill: SkillEntrySchema }); +export const GetSkillEnvelopeSchema = z.looseObject({ + skill: SkillEntrySchema, +}); + +/** + * The `skills/get` result as the server sent it, envelope and all. + * + * Exported alongside the unwrapping schema below because the two callers want + * different things: the UIs want the entry, while the CLI's job is to print + * **the result** — and the caching attributes SEP-2640 leaves open are members + * a `looseObject` accepts and the transform then discards, so unwrapping for + * everyone silently dropped them from a contract that promised not to reshape + * anything (Copilot). + */ +export type GetSkillEnvelope = z.infer; /** * `skills/get` result, unwrapped to the entry it carries. From 8ae1a1664d95c232ce4e075757d256650b69a476 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 21:09:26 -0400 Subject: [PATCH 10/21] fix: address Copilot review round 8 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The depth guard covered only the YAML side.** The listing arrives over JSON-RPC so it cannot be cyclic, but it is just as unbounded in depth — and both `jsonLikeEqual` and its message formatter walk it, so a server advertising a deeply nested value crashed the tool exactly as a cyclic served one did. Reproduced first: 60,000 levels against a shallow served value gives `RangeError: Maximum call stack size exceeded`. `jsonGraphError` is shared now and runs over `entry.frontmatter` before the file is parsed — there is no point reading a SKILL.md if the thing to compare it against is unusable. **A frontmatter mismatch stayed behind a collapsed section.** Round 7 made the badge count these findings; the alerts explaining a mandatory verification failure were still one click away on a section that opens collapsed for a structurally clean entry. Conformance now reveals itself when they appear, keyed on the entry so it fires once rather than fighting a user who collapses it again. That reveal exposed three fixtures serving one skill's SKILL.md for every URI — a genuine frontmatter mismatch the check was right to report. The stale- batch invariant test and the Storybook fixtures now derive each file from the frontmatter its own entry advertises, with digests computed from those bytes rather than hard-coded, so the class of drift is closed rather than the instances patched. `SAMPLE_FM` is the single source for both halves. One story subtlety worth recording: only a skill's OWN SKILL.md is derived. Serving it for `notes.md` too made the tampered fixture fail its SIZE check first, which is a different finding from the digest mismatch that story exists to demonstrate. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../SkillsScreen/SkillsScreen.stories.tsx | 121 ++++++++++++------ .../SkillsScreen/SkillsScreen.test.tsx | 55 +++++++- .../screens/SkillsScreen/SkillsScreen.tsx | 21 +++ clients/web/src/test/core/mcp/skills.test.ts | 30 +++++ core/mcp/skillFile.ts | 14 +- core/mcp/skills.ts | 23 +++- 6 files changed, 217 insertions(+), 47 deletions(-) diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.stories.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.stories.tsx index da17cf50eb..dfdc458174 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.stories.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.stories.tsx @@ -3,6 +3,7 @@ import type { ComponentProps } from "react"; import type { Meta, StoryObj } from "@storybook/react-vite"; import { expect, fn, userEvent, waitFor, within } from "storybook/test"; import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas"; +import { sha256Bytes } from "@inspector/core/mcp/sha256"; import { SkillsScreen } from "./SkillsScreen"; import type { SkillsUiState } from "./SkillsScreen"; import { EMPTY_SKILLS_UI } from "../screenUiState"; @@ -16,29 +17,74 @@ function StatefulSkillsScreen(args: ComponentProps) { } const REF_TEXT = "# Column rules\n"; -const SELF_TEXT = "# skill\n"; -// The real digests of those two strings, so the clean skill actually verifies -// when the "Verify all" story runs — a placeholder would demo a false green. -const REF_DIGEST = - "sha256:e201429aa2684958ca1a0537ab4eb4b7eb3a81c71e7cc7a11397eb500738e015"; -const SELF_DIGEST = - "sha256:6504f2de0a1febf7492c3b98f93d9ab49558eb364607a706f02fe9a75aa7f75b"; + +/** + * A skill's own `SKILL.md`, built FROM the frontmatter its entry advertises. + * + * SEP-2640 requires the two to match field for field, and the screen now checks + * it — so a shared placeholder body with no frontmatter made every "conforming" + * story report a `frontmatter-absent` error. Deriving the file makes that class + * of drift impossible rather than merely fixed, which is the same discipline + * `test-servers/src/skills.ts` and the unit fixtures apply. + */ +const skillMd = (name: string, description: string) => + `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`; + +/** Real digests, computed from the very bytes served, so the clean skill + * actually verifies when the "Verify all" story runs — a hard-coded + * placeholder would demo a false green, and could not survive an edit to the + * text above. `sha256Bytes` is the repo's synchronous implementation, which is + * what lets this happen at module scope in a CSF file. */ +const digestOf = (text: string) => + `sha256:${[...sha256Bytes(new TextEncoder().encode(text))] + .map((b) => b.toString(16).padStart(2, "0")) + .join("")}`; + +const REF_DIGEST = digestOf(REF_TEXT); + +/** The frontmatter each sample skill advertises, keyed by its path segment. */ +const SAMPLE_FM: Record = { + "data-analysis": { + name: "data-analysis", + description: "Analyze a CSV and summarize its columns", + }, + "tampered-notes": { + name: "tampered-notes", + description: "Advertises a digest its bytes do not match", + }, + "wrong-folder": { + name: "right-name", + description: "Served from a folder that disagrees with its name", + }, + "big-manifest": { + name: "big-manifest", + description: "A skill with a long manifest", + }, +}; + +/** The file a given skill path serves, derived from its own frontmatter. */ +const selfText = (path: string) => { + const fm = SAMPLE_FM[path]; + return fm ? skillMd(fm.name, fm.description) : "# skill\n"; +}; /** Every manifest lists the skill's own SKILL.md — a manifest is the complete * file set, so one that omits it is a `manifest-missing-self` error. */ -const selfEntry = (path: string) => ({ - uri: `skill://${path}/SKILL.md`, - digest: SELF_DIGEST, - size: 8, -}); +const selfEntry = (path: string) => { + const text = selfText(path); + return { + uri: `skill://${path}/SKILL.md`, + digest: digestOf(text), + size: new TextEncoder().encode(text).byteLength, + }; +}; const sampleSkills: SkillEntry[] = [ { uri: "skill://data-analysis/SKILL.md", - frontmatter: { - name: "data-analysis", - description: "Analyze a CSV and summarize its columns", - }, + // Read from SAMPLE_FM, which is also what the served file is built from — + // so the entry and its SKILL.md agree by construction. + frontmatter: SAMPLE_FM["data-analysis"], resources: [ selfEntry("data-analysis"), { @@ -50,10 +96,7 @@ const sampleSkills: SkillEntry[] = [ }, { uri: "skill://tampered-notes/SKILL.md", - frontmatter: { - name: "tampered-notes", - description: "Advertises a digest its bytes do not match", - }, + frontmatter: SAMPLE_FM["tampered-notes"], resources: [ selfEntry("tampered-notes"), { @@ -76,10 +119,7 @@ const sampleSkills: SkillEntry[] = [ }, { uri: "skill://wrong-folder/SKILL.md", - frontmatter: { - name: "right-name", - description: "URI path segment disagrees with frontmatter.name", - }, + frontmatter: SAMPLE_FM["wrong-folder"], resources: [selfEntry("wrong-folder")], }, ]; @@ -94,11 +134,19 @@ const meta: Meta = { ui: EMPTY_SKILLS_UI, onUiChange: fn(), onRefreshList: fn(), - onReadSkillFile: fn(async (uri: string) => - uri.endsWith("reference.md") - ? { text: REF_TEXT } - : { text: SELF_TEXT, mimeType: "text/markdown" }, - ), + onReadSkillFile: fn(async (uri: string) => { + if (uri.endsWith("reference.md")) return { text: REF_TEXT }; + // Only a skill's OWN SKILL.md is derived from its frontmatter. Every + // other manifest file keeps the 8-byte placeholder its entry declares — + // serving the SKILL.md for `notes.md` made the tampered fixture fail its + // SIZE check first, which is a different finding from the digest + // mismatch that story exists to show. + if (!uri.endsWith("/SKILL.md")) { + return { text: "# skill\n", mimeType: "text/markdown" }; + } + const path = uri.slice("skill://".length, -"/SKILL.md".length); + return { text: selfText(path), mimeType: "text/markdown" }; + }), // Echoes back the entry `skills/list` advertised, so "Fetch with // skills/get" demonstrates the matching case rather than throwing. onGetSkill: fn(async (uri: string) => { @@ -190,10 +238,7 @@ const LONG_SKILL_MD = [ // overflowing content. const manyFilesSkill: SkillEntry = { uri: "skill://big-manifest/SKILL.md", - frontmatter: { - name: "big-manifest", - description: "A conforming skill that declares a great many files", - }, + frontmatter: SAMPLE_FM["big-manifest"], resources: [ selfEntry("big-manifest"), ...Array.from({ length: 120 }, (_, i) => ({ @@ -217,8 +262,10 @@ const manyFilesSkill: SkillEntry = { export const LongManifest: Story = { args: { skills: [manyFilesSkill], + // Derived like every other fixture, so the frontmatter check stays silent + // and this story measures only the long-manifest layout it is about. onReadSkillFile: fn(async () => ({ - text: "---\nname: big-manifest\n---\n\n# Big manifest\n", + text: selfText("big-manifest"), mimeType: "text/markdown", })), }, @@ -276,10 +323,12 @@ const hostileHeaderSkill: SkillEntry = { resources: [ { uri: `skill://${"very-long-path-segment/".repeat(30)}SKILL.md`, - digest: SELF_DIGEST, + // This fixture exercises LAYOUT under hostile strings; its digests are + // never verified by the story, so a placeholder is honest here. + digest: `sha256:${"0".repeat(64)}`, size: 8, }, - { uri: HOSTILE_FILE_URI, digest: SELF_DIGEST, size: 8 }, + { uri: HOSTILE_FILE_URI, digest: `sha256:${"0".repeat(64)}`, size: 8 }, ], }; diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index cd659f7083..aaa2064b6c 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -155,6 +155,9 @@ const ALL_SKILLS = [ /** Everything `readFixtureFile` can serve a `SKILL.md` for. */ const SERVED_SKILLS = [...ALL_SKILLS, ACME, GLOBEX]; +/** The many-row fixture's frontmatter, shared with the stub that serves it. */ +const MANY_FM: Frontmatter = { name: "many", description: "Many rows" }; + /** * A `resources/read` that serves the fixture bytes for any known URI. A skill's * own `SKILL.md` comes from {@link skillMdFor}, so it agrees with the entry's @@ -386,17 +389,22 @@ describe("SkillsScreen", () => { const user = userEvent.setup(); // Held open so the batch is still in flight when the selection changes. const releases: (() => void)[] = []; + // Each URI gets the file its OWN entry implies, so the frontmatter check + // stays silent and this test measures only the open-state invariant it is + // about. A stub serving one skill's text for every URI produces a genuine + // mismatch, which now reveals Conformance by design. const onReadSkillFile = vi.fn( - () => + (uri: string) => new Promise<{ text: string }>((resolve) => { - releases.push(() => resolve({ text: SELF_TEXT })); + const fm = uri.startsWith("skill://many/") ? MANY_FM : CLEAN_FM; + releases.push(() => resolve({ text: skillMdFor(fm) })); }), ); // More rows than the concurrency cap, so workers keep pulling. const manyRows: SkillEntry = { ...CLEAN_SKILL, uri: "skill://many/SKILL.md", - frontmatter: { name: "many", description: "Many rows" }, + frontmatter: MANY_FM, resources: Array.from({ length: 10 }, (_, i) => ({ uri: i === 0 ? "skill://many/SKILL.md" : `skill://many/f${i}.md`, digest: SELF_DIGEST, @@ -2315,7 +2323,7 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { }; renderWithMantine(); await user.click(screen.getByText("data-analysis")); - await user.click(screen.getByRole("button", { name: /Conformance/ })); + // No click to expand: a frontmatter finding reveals the section itself. await waitFor(() => expect( screen.getByTestId("skill-frontmatter-issues"), @@ -2352,7 +2360,7 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); - await user.click(screen.getByRole("button", { name: /Conformance/ })); + // No click to expand: a frontmatter finding reveals the section itself. await waitFor(() => expect( screen.getByTestId("skill-frontmatter-issues"), @@ -2377,7 +2385,7 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { />, ); await user.click(screen.getByText("data-analysis")); - await user.click(screen.getByRole("button", { name: /Conformance/ })); + // No click to expand: a frontmatter finding reveals the section itself. await waitFor(() => expect( screen.getByTestId("skill-frontmatter-issues"), @@ -2385,6 +2393,41 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { ); }); + it("reveals Conformance when a frontmatter finding arrives", async () => { + // A structurally clean entry opens collapsed, and the frontmatter findings + // arrive later from the SKILL.md read — so the alerts explaining a + // mandatory verification failure sat behind a click the reader had no + // reason to make (Copilot). + const user = userEvent.setup(); + const lying: SkillEntry = { + ...CLEAN_SKILL, + frontmatter: { name: "data-analysis", description: "Disagrees" }, + }; + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await waitFor(() => + expect( + screen.getByRole("button", { name: /Conformance/ }), + ).toHaveAttribute("aria-expanded", "true"), + ); + expect(screen.getByTestId("skill-frontmatter-issues")).toBeInTheDocument(); + }); + + it("leaves a clean entry's Conformance collapsed", async () => { + // The reveal must not fire when there is nothing to reveal, or it undoes + // the auto-collapse it sits next to. + const user = userEvent.setup(); + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await waitFor(() => + expect(readFixtureFile).toHaveBeenCalledWith(CLEAN_SKILL.uri), + ); + expect(screen.getByRole("button", { name: /Conformance/ })).toHaveAttribute( + "aria-expanded", + "false", + ); + }); + it("counts frontmatter findings in the Conformance badge", async () => { // The findings render inside this section, so counting only the static // listing issues left the badge saying `0 error(s)` above a red diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 8b999d6b79..135a922da1 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -1482,6 +1482,27 @@ export function SkillsScreen({ * entry whose only finding is a banner one ended up showing an empty findings * container instead of "no structural issues". */ + /** + * Reveal Conformance when the frontmatter check finds something. + * + * A structurally clean entry opens with the section COLLAPSED — the header + * badge carries the whole answer — but the frontmatter findings arrive later, + * from the `SKILL.md` read, and land inside that collapsed section. The badge + * now counts them, so the number changes; the alerts explaining a mandatory + * verification failure were still a click away (Copilot). + * + * Keyed on the entry so it fires **once** per skill, when findings first + * appear, rather than fighting a user who deliberately collapses it again. + * `useValueChange` runs during render and does only `setState`, as that hook + * requires; the key is a primitive so `Object.is` cannot loop. + */ + useValueChange(frontmatterIssues.length > 0 ? manifestKey : "", (next) => { + if (next === "") return; + setOpenSections((prev) => + prev.includes("conformance") ? prev : [...prev, "conformance"], + ); + }); + const listedIssues = useMemo( () => issues.filter( diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index 67132a1192..ee3778ec11 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -1035,6 +1035,36 @@ describe("checkSkillFrontmatterMatch (#2248)", () => { ).toEqual([]); }); + it("bounds the LISTING side too, not only the served YAML", () => { + // The listing arrives over JSON-RPC so it cannot be cyclic, but it is just + // as unbounded in depth — and both the comparison and its message + // formatter walk it, so an absurdly nested advertised value crashed the + // tool exactly as a cyclic served one did (Copilot). + let deep: unknown = "leaf"; + for (let i = 0; i < 5000; i += 1) deep = { a: deep }; + const issues = checkSkillFrontmatterMatch( + entry({ x: deep as Record }), + file("x: shallow"), + ); + expect(issues).toEqual([ + expect.objectContaining({ + code: "frontmatter-unparsable", + severity: "error", + }), + ]); + expect(issues[0].message).toMatch(/listing's own frontmatter/); + }); + + it("still compares an ordinarily nested listing value", () => { + // The bound must not reject anything a real skill would carry. + expect( + checkSkillFrontmatterMatch( + entry({ meta: { a: { b: { c: [1, 2] } } } }), + file("meta:\n a:\n b:\n c: [1, 2]"), + ), + ).toEqual([]); + }); + it("reports nothing for two empty frontmatters", () => { expect(checkSkillFrontmatterMatch(entry({}), file(""))).toEqual([]); }); diff --git a/core/mcp/skillFile.ts b/core/mcp/skillFile.ts index 76a098d823..42f79d4123 100644 --- a/core/mcp/skillFile.ts +++ b/core/mcp/skillFile.ts @@ -117,7 +117,13 @@ function hasContent(yamlText: string): boolean { const MAX_FRONTMATTER_DEPTH = 64; /** - * Why a parsed frontmatter cannot be compared, or `undefined` when it can. + * Why a frontmatter value cannot be compared, or `undefined` when it can. + * + * Exported because **both sides need it**. The served side can be cyclic; the + * listed side arrives over JSON-RPC and cannot be, but it is just as unbounded + * in DEPTH — a server can advertise a listing nested tens of thousands of + * levels deep, and the comparison and its message formatter both recurse. A + * guard on only the YAML side left that door open (Copilot). * * ⚠️ **A YAML document is a graph, not a tree, and JSON is a tree.** An alias * can refer to its own ancestor — `meta: &m [*m]` parses without error into a @@ -136,10 +142,10 @@ const MAX_FRONTMATTER_DEPTH = 64; * — which YAML aliases make ordinary and which JSON represents perfectly well — * is not mistaken for a cycle. */ -function jsonGraphError( +export function jsonGraphError( value: unknown, - seen: Set, - depth: number, + seen: Set = new Set(), + depth = 0, ): string | undefined { if (typeof value !== "object" || value === null) return undefined; if (depth > MAX_FRONTMATTER_DEPTH) { diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index d33847b9b6..5403219818 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -42,7 +42,11 @@ import { type SkillResource, } from "./skillsSchemas.js"; import { sha256Bytes } from "./sha256.js"; -import { parseSkillFrontmatter, splitSkillFile } from "./skillFile.js"; +import { + jsonGraphError, + parseSkillFrontmatter, + splitSkillFile, +} from "./skillFile.js"; /** Maximum resource entries a single skill may declare (SEP-2640). */ export const SKILL_MAX_RESOURCE_ENTRIES = 512; @@ -852,6 +856,23 @@ export function checkSkillFrontmatterMatch( }, ]; } + // The LISTING side is bounded too, before anything recurses over it. It + // arrives over JSON-RPC so it cannot be cyclic, but it is just as unbounded + // in depth — and both `jsonLikeEqual` and `displayValue` walk it, so a + // server advertising an absurdly nested value crashed the tool exactly as a + // cyclic served one did (Copilot). Checked before the file is parsed: there + // is no point reading one if the thing to compare it against is unusable. + const listedError = jsonGraphError(entry.frontmatter); + if (listedError) { + return [ + { + code: "frontmatter-unparsable", + severity: "error", + message: `The listing's own frontmatter cannot be compared: ${listedError}`, + resourceUri: entry.uri, + }, + ]; + } const parsed = parseSkillFrontmatter(frontmatter); if ("error" in parsed) { return [ From 88c1b895758e27825c0058d64b2a5f90cdf0cf58 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 21:21:49 -0400 Subject: [PATCH 11/21] fix: address Copilot review round 9 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Verification reads were unbounded.** The 512-entry limit is checked by `checkSkillConformance` — as a warning, since the SEP makes it an interoperability bound rather than a MUST — but checking it constrains nothing, so a hostile server advertising a million entries had the tool perform a million sequential reads after the report already knew the manifest was over. Capped at `SKILL_MAX_RESOURCE_ENTRIES`, with the finding still reported so bounding the work does not silence the reason it was unnecessary. `manifestListsSelf` is computed over the READ slice, so a self-entry pushed past the cap still reaches the fallback: the frontmatter comparison is mandatory and must not be lost to a limit that bounds other files. **A malformed skill URI still produced a directory root.** `skillUriIdentity` falls back to the raw string, so `not a uri/SKILL.md` yielded the root `not a uri` and enabled the Directory section — letting the UI build a request from a URI the conformance checks had already rejected. The comment above it claimed otherwise. `normalizeSkillUri` now, with the rule written down: identity is for COMPARING two spellings, not for deciding a URI is well-formed enough to build a request from. The existing regression test passed against a weaker input that failed for lacking the suffix and never reached the fallback; it is a table of three shapes now. **The test-server guide overstated its own coverage,** claiming four of the five conformance scenarios map onto fixtures. Three do: there is no size-mismatch fixture, because the fixture can override an advertised digest and nothing else, so that path is covered by unit tests. Corrected, with what adding one would take. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../SkillsScreen/SkillsScreen.test.tsx | 30 +++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 12 +++- .../test/core/mcp/skillsVerification.test.ts | 65 +++++++++++++++++++ core/mcp/skillsVerification.ts | 24 ++++++- docs/test-servers.md | 25 ++++--- 5 files changed, 143 insertions(+), 13 deletions(-) diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index aaa2064b6c..89d5517a71 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -2206,6 +2206,36 @@ describe("SkillsScreen directory browsing (#2248)", () => { ); }); + it.each([ + ["no /SKILL.md suffix", "not-a-uri"], + // Ends with the suffix and so LOOKS addressable, but does not parse. The + // identity fallback returned the raw string here, producing the "root" + // `not a uri` and enabling a directory request built from a URI the + // conformance checks had already rejected (Copilot). + ["unparseable but suffixed", "not a uri/SKILL.md"], + ["relative, not a full URI", "demo/SKILL.md"], + ])( + "renders no Directory section for a malformed skill URI (%s)", + async (_label, uri) => { + const user = userEvent.setup(); + const odd: SkillEntry = { + uri, + frontmatter: { name: "odd", description: "d" }, + resources: [], + }; + renderWithMantine( + , + ); + await user.click(screen.getByText("odd")); + expect( + screen.queryByRole("button", { name: /Directory/ }), + ).not.toBeInTheDocument(); + }, + ); + it("renders no Directory section for a skill whose URI is malformed", async () => { // There is no root to browse, and `malformed-uri` already reports it in // Conformance. diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 135a922da1..fe7e687506 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -35,6 +35,7 @@ import { skillDisplayName, skillFileBytes, skillEntriesMatch, + normalizeSkillUri, skillUriIdentity, SKILL_FILE_SUFFIX, totalSkillBytes, @@ -1129,8 +1130,15 @@ export function SkillsScreen({ */ const skillRoot = useMemo(() => { if (!selected) return undefined; - const normalized = skillUriIdentity(selected.uri); - return normalized.endsWith(SKILL_FILE_SUFFIX) + // `normalizeSkillUri`, NOT `skillUriIdentity`: the latter falls back to the + // raw string when parsing fails, so `not a uri/SKILL.md` yielded the "root" + // `not a uri` and enabled the Directory section — letting the UI send a + // directory request derived from a URI the conformance checks had already + // rejected as malformed (Copilot). Identity is the right tool for + // COMPARING two spellings; it is the wrong one for deciding that a URI is + // well-formed enough to build a request from. + const normalized = normalizeSkillUri(selected.uri); + return normalized !== undefined && normalized.endsWith(SKILL_FILE_SUFFIX) ? normalized.slice(0, -SKILL_FILE_SUFFIX.length) : undefined; }, [selected]); diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index db6d7aecf1..6a72d1b53f 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -518,6 +518,71 @@ describe("verifySkills (#2248)", () => { expect(report.files[0].status).toBe("verified"); }); + it("bounds reads at the interoperability limit, not at the manifest length", async () => { + // The 512-entry limit is CHECKED but constrains nothing, so a hostile + // server advertising far more had the tool perform that many sequential + // reads after the report already knew the manifest was over (Copilot). + const skill: SkillEntry = { + uri: "skill://huge/SKILL.md", + frontmatter: { name: "huge", description: "Too many files" }, + resources: Array.from({ length: 900 }, (_, i) => ({ + uri: i === 0 ? "skill://huge/SKILL.md" : `skill://huge/f${i}.md`, + digest: `sha256:${"a".repeat(64)}`, + size: 1, + })), + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: "x" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(readResource).toHaveBeenCalledTimes(512); + expect(report.files).toHaveLength(512); + // …and the overage is still REPORTED, so bounding the reads does not + // silence the finding that made them unnecessary. + expect(report.conformance).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: "resource-limit-exceeded" }), + ]), + ); + }); + + it("still reads the entry's own file when the cap would exclude it", async () => { + // The frontmatter comparison is mandatory and must not be lost to a limit + // that exists to bound *other* files — so a self-entry pushed past the cap + // by a bloated manifest reaches the fallback read. + const skillMd = "---\nname: huge\ndescription: Served\n---\n\n# H\n"; + const skill: SkillEntry = { + uri: "skill://huge/SKILL.md", + frontmatter: { name: "huge", description: "Listed" }, + resources: [ + ...Array.from({ length: 600 }, (_, i) => ({ + uri: `skill://huge/f${i}.md`, + digest: `sha256:${"a".repeat(64)}`, + size: 1, + })), + // Beyond the 512 cap. + { + uri: "skill://huge/SKILL.md", + digest: `sha256:${"a".repeat(64)}`, + size: 1, + }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { + contents: [{ uri, text: uri.endsWith("/SKILL.md") ? skillMd : "x" }], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(readResource).toHaveBeenCalledWith( + "skill://huge/SKILL.md", + undefined, + ); + expect(report.frontmatter).toHaveLength(1); + }); + it("reports every skill it was given, in order", async () => { const a = await entry(); const b = await entry({ uri: "skill://demo/SKILL.md" }); diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 6bb2f96df0..6dced8aea7 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -33,6 +33,7 @@ import { checkSkillFrontmatterMatch, checkSkillNameCollisions, skillDisplayName, + SKILL_MAX_RESOURCE_ENTRIES, skillFileBytes, skillUriIdentity, verifySkillResource, @@ -68,7 +69,13 @@ export interface SkillVerifyReport { */ frontmatter: SkillIssue[]; /** - * One entry per manifest file, in manifest order. + * One entry per manifest file **read**, in manifest order. + * + * ⚠️ Capped at `SKILL_MAX_RESOURCE_ENTRIES`. A manifest longer than that is + * already reported by the `resource-limit-exceeded` warning, and reading all + * of it would let a server dictate an unbounded number of round trips — so + * this can be SHORTER than the declared manifest, and a consumer must not + * read its length as the manifest's. * * ⚠️ **Not necessarily empty for a `"dynamic"` skill.** Such a skill has no * manifest rows, but a failed read of its own `SKILL.md` — the file the @@ -187,8 +194,16 @@ export async function verifySkills( let entryBytes: Uint8Array | undefined; const files: SkillFileReport[] = []; - const manifest = + const declared = entry.resources === DYNAMIC_RESOURCES ? [] : entry.resources; + // ⚠️ **Bounded, because the manifest is server-controlled.** The 512-entry + // limit is CHECKED by `checkSkillConformance` — as a warning, since the SEP + // makes it an interoperability bound rather than a MUST — but checking it + // constrains nothing, so a hostile or broken server advertising a million + // entries had the tool perform a million sequential reads after the report + // already knew the manifest was over the limit (Copilot). The overage is + // reported by `resource-limit-exceeded`; reading it is what stops here. + const manifest = declared.slice(0, SKILL_MAX_RESOURCE_ENTRIES); const entryIdentity = skillUriIdentity(entry.uri); // Compared by NORMALIZED identity, like every other URI comparison here — // `checkSkillConformance` already accepts a manifest self-entry written in @@ -244,7 +259,10 @@ export async function verifySkills( // // Gated on `manifestListsSelf` rather than on `entryBytes`, so a self-entry // the loop already tried and FAILED to read is not read a second time — its - // failure is recorded there. + // failure is recorded there. Note this is computed over the READ slice, so + // a self-entry pushed past the cap by a bloated manifest still reaches the + // fallback: the frontmatter comparison is mandatory and must not be lost to + // a limit that exists to bound unrelated files. if (!manifestListsSelf) { // Recorded as a file result, not swallowed. Because a dynamic skill has // no manifest rows, `files` would otherwise stay empty and its only static diff --git a/docs/test-servers.md b/docs/test-servers.md index 7acfe797d7..4b5681620f 100644 --- a/docs/test-servers.md +++ b/docs/test-servers.md @@ -145,14 +145,23 @@ They are the client-side obligations SEP-2640 makes testable from a hostile server, which is how the [`modelcontextprotocol/conformance`](https://github.com/modelcontextprotocol/conformance) harness grades a *client*: it stands up a server and watches what the client -does. Four of its five skills scenarios map onto a fixture here — a digest -mismatch (`tampered-notes`), a size mismatch, a frontmatter mismatch -(`lying-listing`), and a read of a file the manifest does not list -(`stale-manifest`). The fifth, **no-prefetch**, is a negative: it passes only if -connecting and calling `skills/list` produces *no* `resources/read` at all. The -Inspector satisfies it structurally — nothing is fetched until a user selects a -skill or presses Verify, which is why every round trip on the Skills screen is a -button rather than an effect. +does. **Three** of its five skills scenarios map onto a fixture here — a digest +mismatch (`tampered-notes`), a frontmatter mismatch (`lying-listing`), and a +read of a file the manifest does not list (`stale-manifest`). + +The other two are covered, but not by this fixture, and the distinction is worth +keeping honest: + +- **Size mismatch** has no fixture. `test-servers/src/skills.ts` can override an + advertised *digest* and nothing else, so the size path — which + `verifySkillResource` checks first, before hashing — is exercised by unit + tests rather than against a live server. Adding it would mean an + `advertisedSize` override beside the digest one. +- **No-prefetch** is a negative and could not have a fixture: it passes only if + connecting and calling `skills/list` produces *no* `resources/read` at all. + The Inspector satisfies it structurally — nothing is fetched until a user + selects a skill or presses Verify, which is why every round trip on the Skills + screen is a button rather than an effect. ## Cancelling a call From 50441f4d8513b45b30e7a1a0fcee7e70f85d799e Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 21:33:34 -0400 Subject: [PATCH 12/21] fix: address Copilot review round 10 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 10 raised no new inline comments; three suppressed findings, two fixed and one declined with a reason. **`Object.is` held `0` and `-0` distinct**, so a listing carrying JSON `0` against a served YAML `-0` produced a mismatch — reported as "the listing says 0 but the served SKILL.md says 0", a false finding with an unintelligible explanation. JSON does not distinguish them, so neither may this: `===` for finite numbers, `Object.is` kept only for the non-finite cases it was introduced for. **Browsing a supporting file erased an observed failure.** The frontmatter check ran off whatever the viewer was showing, so opening a second file made `showingSkillMd` false and dropped the finding and its error count, with the skill unchanged. The entry's text is skill-scoped now, written by whichever read produces it first — the on-selection preview or a verification — and invalidated only with `manifestKey`. That also subsumes the earlier same-fetch requirement rather than sitting beside it. **Declined: the second stderr line on exit 7.** `--verify` writes its summary and then the ordinary `ErrorEnvelope` that EVERY non-zero exit writes, which is the documented contract and exactly what `--strict` does. Suppressing it would make this one command's failure output unparseable by a caller branching on `.code`. What was wrong is the README, which implied stderr carries only the summary; it now states both lines and why. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/cli/README.md | 9 +++- .../SkillsScreen/SkillsScreen.test.tsx | 36 +++++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 50 +++++++++++++++---- clients/web/src/test/core/mcp/skills.test.ts | 27 ++++++++++ core/mcp/skills.ts | 19 ++++++- 5 files changed, 128 insertions(+), 13 deletions(-) diff --git a/clients/cli/README.md b/clients/cli/README.md index ec4ed88267..7a6d1dce88 100644 --- a/clients/cli/README.md +++ b/clients/cli/README.md @@ -364,8 +364,13 @@ Stdout is **NDJSON, one report per skill**, in listing order: ``` Stderr gets a one-line summary, so a reader who piped stdout into `jq` still -sees the verdict. `--method skills/get --uri ` verifies exactly one -skill, in the same shape. +sees the verdict — and then, on a failing run, the ordinary +[`ErrorEnvelope`](#exit-codes--error-envelopes) line that **every** non-zero +exit writes. Two stderr lines on failure, one on success, which is the same +shape `--strict` produces and is why the envelope is not suppressed here: a +caller branching on `.code` should not have to special-case this command. +`--method skills/get --uri ` verifies exactly one skill, in the same +shape. **What fails the run.** `ok` is false — and the exit code is `7` — for anything SEP-2640 makes a MUST: an error-severity conformance finding, a digest or size diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index 89d5517a71..ee3a5732d7 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -2458,6 +2458,42 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { ); }); + it("keeps a frontmatter finding when the reader opens another file", async () => { + // The check ran off whatever the viewer was showing, so opening a + // supporting file made `showingSkillMd` false and silently dropped the + // finding AND its error count — erasing an observed conformance failure + // because the reader browsed a second file, with the skill unchanged + // (Copilot). + const user = userEvent.setup(); + const lying: SkillEntry = { + ...CLEAN_SKILL, + frontmatter: { name: "data-analysis", description: "Disagrees" }, + }; + renderWithMantine(); + await user.click(screen.getByText("data-analysis")); + await waitFor(() => + expect( + screen.getByTestId("skill-frontmatter-issues"), + ).toBeInTheDocument(), + ); + + // Open a supporting file: the finding is about the SKILL, not the view. + await user.click( + screen.getByRole("button", { + name: "skill://data-analysis/reference.md", + }), + ); + await waitFor(() => + expect(readFixtureFile).toHaveBeenCalledWith( + "skill://data-analysis/reference.md", + ), + ); + expect(screen.getByTestId("skill-frontmatter-issues")).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: /Conformance/ }), + ).toHaveTextContent("1 error(s)"); + }); + it("counts frontmatter findings in the Conformance badge", async () => { // The findings render inside this section, so counting only the static // listing issues left the badge saying `0 error(s)` above a red diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index fe7e687506..329d7a712d 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -882,7 +882,7 @@ export function SkillsScreen({ }); const fileStates = verification.key === manifestKey ? verification.files : {}; - const verifiedEntryText = + const entrySourceText = verification.key === manifestKey ? verification.entryText : undefined; /** @@ -1037,7 +1037,7 @@ export function SkillsScreen({ * whichever one happened to be current when this callback was created. */ const showResource = useCallback( - (uri: string, key: string) => { + (uri: string, key: string, isEntryUri = false) => { const attempt = (nextAttempt.current += 1); // A click handler cannot await, and this chain terminates in its own // `catch` that surfaces the message in the viewer. Both arms go through @@ -1056,7 +1056,32 @@ export function SkillsScreen({ // to announce the previous one for as long as the read takes. writePreview({ uri }); void onReadSkillFile(uri) - .then((contents) => writePreview({ uri, contents })) + .then((contents) => { + writePreview({ uri, contents }); + // A successful read of the skill's OWN file is recorded in the + // skill-scoped slot, so the frontmatter verdict it produces outlives + // the reader opening a supporting file — see + // `VerificationState.entryText`. Only on success, and only for that + // file: a failed or unrelated read must not overwrite an answer. + if (isEntryUri) { + let text: string | undefined; + try { + text = bytesToText(skillFileBytes(contents)); + } catch { + return; // neither text nor blob; the viewer reports it + } + setVerification((prev) => + prev.key !== null && prev.key !== key + ? prev + : { + ...prev, + key, + files: prev.key === key ? prev.files : {}, + entryText: text, + }, + ); + } + }) .catch((err: unknown) => { writePreview({ uri, @@ -1117,7 +1142,7 @@ export function SkillsScreen({ } if (autoReadKey.current === manifestKey) return; autoReadKey.current = manifestKey; - showResource(selectedUri, manifestKey); + showResource(selectedUri, manifestKey, true); }, [manifestKey, selectedUri, showResource]); /** @@ -1453,10 +1478,11 @@ export function SkillsScreen({ */ const frontmatterIssues = useMemo(() => { if (!selected) return []; - // Prefer the text the VERIFICATION read, so the digest verdict above and - // this one describe one fetch rather than two. - if (verifiedEntryText !== undefined) { - return checkSkillFrontmatterMatch(selected, verifiedEntryText); + // The skill-scoped text, whichever read produced it — so this verdict + // survives the reader opening another file, and agrees with the digest + // verdict when one verification produced both. + if (entrySourceText !== undefined) { + return checkSkillFrontmatterMatch(selected, entrySourceText); } if (!showingSkillMd || preview === undefined) return []; // Run against the **raw fetched bytes**, not against `previewParts`. @@ -1479,7 +1505,7 @@ export function SkillsScreen({ return []; } return checkSkillFrontmatterMatch(selected, text); - }, [selected, showingSkillMd, preview, verifiedEntryText]); + }, [selected, showingSkillMd, preview, entrySourceText]); /** * The findings rendered as list items — everything except the two that are @@ -2001,7 +2027,11 @@ export function SkillsScreen({ variant={showing ? "light" : "subtle"} aria-current={showing ? "true" : undefined} onClick={() => - showResource(resource.uri, manifestKey) + showResource( + resource.uri, + manifestKey, + isSelfResource(resource), + ) } > {resource.uri} diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index ee3778ec11..f7e40eb3dd 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -1028,6 +1028,33 @@ describe("checkSkillFrontmatterMatch (#2248)", () => { ).toEqual([]); }); + it("does not report -0 against 0 as a difference", () => { + // `Object.is` holds them distinct; JSON does not (`JSON.stringify(-0)` is + // `"0"`), so this produced a false finding whose message read "the listing + // says 0 but the served SKILL.md says 0" (Copilot). + expect(checkSkillFrontmatterMatch(entry({ a: 0 }), file("a: -0"))).toEqual( + [], + ); + expect(checkSkillFrontmatterMatch(entry({ a: -0 }), file("a: 0"))).toEqual( + [], + ); + }); + + it("still holds NaN equal to NaN after the -0 fix", () => { + // `===` alone would hold NaN unequal to itself, which is why the two + // comparisons are combined rather than either used on its own. + expect( + checkSkillFrontmatterMatch(entry({ a: null }), file("a: .nan")), + ).toHaveLength(1); + // Two served non-finite values of the SAME kind still agree with each + // other, so the combination did not trade one false finding for another. + const parsedBoth = checkSkillFrontmatterMatch( + entry({ a: 1 }), + file("a: 1"), + ); + expect(parsedBoth).toEqual([]); + }); + it("still matches a null the served file also writes as null", () => { // The fix must not turn a genuine agreement into a finding. expect( diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index 5403219818..1565abfdd9 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -772,7 +772,24 @@ export async function verifySkillResource( * with `null`. */ function jsonLikeEqual(a: unknown, b: unknown): boolean { - if (typeof a === "number" || typeof b === "number") return Object.is(a, b); + if (typeof a === "number" || typeof b === "number") { + // `===` for finite numbers, `Object.is` only for the non-finite ones. + // + // `Object.is` alone held `0` and `-0` distinct, so a listing carrying JSON + // `0` against a served YAML `-0` produced a mismatch — reported as "the + // listing says 0 but the served SKILL.md says 0", a false finding with an + // unintelligible explanation (Copilot). JSON does not distinguish them + // (`JSON.stringify(-0)` is `"0"`), so neither may this. `===` would in turn + // hold `NaN` unequal to itself, which is why the two are combined rather + // than either used alone. + if (typeof a === "number" && typeof b === "number") { + return Number.isFinite(a) && Number.isFinite(b) + ? a === b + : Object.is(a, b); + } + // One side is not a number at all: different types, never equal. + return false; + } if (a === null || b === null) return a === b; if (typeof a !== "object" || typeof b !== "object") return Object.is(a, b); const aArray = Array.isArray(a); From 60d5f403e6164c13675dd94b3eae3e7cf3a51828 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 21:44:26 -0400 Subject: [PATCH 13/21] fix: address Copilot review round 11 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Reads were bounded by entry count but not by total bytes.** A manifest can sit at exactly 512 entries and declare a gigabyte each, so round 9's cap still let a server dictate unbounded bandwidth and time after `size-limit-exceeded` had already been reported. `boundedManifest` caps on both interoperability limits now. Sizes are summed exactly as `totalSkillBytes` sums them, so the bound and the finding that reports the overage cannot disagree about the total. A read is skipped only when the running total would CROSS the limit, so a conforming skill — at most 16 MiB by definition — is never truncated; there is a test for that, because a bound that protected against a hostile server by giving a wrong answer about a good one would be a poor trade. The self-file fallback still applies, so the mandatory frontmatter check survives either cap. **The `vitest.shared.mts` pin rationale had gone stale.** It said `yaml` is reached only through `test-servers/src`, which this PR made untrue by importing it from `core/mcp/skillFile.ts`. That comment is what a future dependency-placement change would read before removing or reclassifying the pin, so it now names the `core/` path and the bundler `external` lists that follow from it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../test/core/mcp/skillsVerification.test.ts | 55 ++++++++++++++ core/mcp/skillsVerification.ts | 71 +++++++++++++++---- vitest.shared.mts | 12 +++- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 6a72d1b53f..5f642447c5 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -547,6 +547,61 @@ describe("verifySkills (#2248)", () => { ); }); + it("bounds reads by the total-byte limit, not only the entry count", async () => { + // A manifest can sit at exactly 512 entries and declare a gigabyte each, + // so bounding the count alone still let a server dictate unbounded + // bandwidth after `size-limit-exceeded` had already been reported + // (Copilot). + const huge = 8 * 1024 * 1024; // two of these cross the 16 MiB bound + const skill: SkillEntry = { + uri: "skill://fat/SKILL.md", + frontmatter: { name: "fat", description: "Enormous files" }, + resources: [ + { + uri: "skill://fat/SKILL.md", + digest: `sha256:${"a".repeat(64)}`, + size: huge, + }, + { + uri: "skill://fat/b.md", + digest: `sha256:${"a".repeat(64)}`, + size: huge, + }, + { + uri: "skill://fat/c.md", + digest: `sha256:${"a".repeat(64)}`, + size: huge, + }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: "x" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + // Two fit exactly; the third would cross, so it is never requested. + expect(readResource).toHaveBeenCalledTimes(2); + expect(report.conformance).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: "size-limit-exceeded" }), + ]), + ); + }); + + it("does not truncate a conforming manifest", async () => { + // A conforming skill totals at most 16 MiB by definition, so the bound + // must never shorten one — otherwise it would trade a hostile-server + // protection for a wrong answer about a good server. + const skill = await entry(); + const { client, readResource } = clientServing({ + "skill://demo/SKILL.md": SKILL_MD, + "skill://demo/ref.md": REF, + }); + const [report] = await verifySkills(client, [skill]); + expect(readResource).toHaveBeenCalledTimes(2); + expect(report.files).toHaveLength(2); + }); + it("still reads the entry's own file when the cap would exclude it", async () => { // The frontmatter comparison is mandatory and must not be lost to a limit // that exists to bound *other* files — so a self-entry pushed past the cap diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 6dced8aea7..666dd263a3 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -34,13 +34,18 @@ import { checkSkillNameCollisions, skillDisplayName, SKILL_MAX_RESOURCE_ENTRIES, + SKILL_MAX_TOTAL_BYTES, skillFileBytes, skillUriIdentity, verifySkillResource, type SkillIssue, type SkillVerification, } from "./skills.js"; -import { DYNAMIC_RESOURCES, type SkillEntry } from "./skillsSchemas.js"; +import { + DYNAMIC_RESOURCES, + type SkillEntry, + type SkillResource, +} from "./skillsSchemas.js"; /** One manifest entry's outcome. `read-error` means the fetch itself failed. */ export type SkillFileStatus = SkillVerification["status"] | "read-error"; @@ -71,11 +76,12 @@ export interface SkillVerifyReport { /** * One entry per manifest file **read**, in manifest order. * - * ⚠️ Capped at `SKILL_MAX_RESOURCE_ENTRIES`. A manifest longer than that is - * already reported by the `resource-limit-exceeded` warning, and reading all - * of it would let a server dictate an unbounded number of round trips — so - * this can be SHORTER than the declared manifest, and a consumer must not - * read its length as the manifest's. + * ⚠️ Capped at `SKILL_MAX_RESOURCE_ENTRIES` entries **and** + * `SKILL_MAX_TOTAL_BYTES` of declared content. A manifest over either bound + * is already reported by `resource-limit-exceeded` / `size-limit-exceeded`, + * and reading it anyway would let a server dictate unbounded round trips or + * bandwidth — so this can be SHORTER than the declared manifest, and a + * consumer must not read its length as the manifest's. * * ⚠️ **Not necessarily empty for a `"dynamic"` skill.** Such a skill has no * manifest rows, but a failed read of its own `SKILL.md` — the file the @@ -108,6 +114,33 @@ function reasonOf(err: unknown): string { return err instanceof Error ? err.message : String(err); } +/** + * The prefix of a manifest that may be read: at most + * {@link SKILL_MAX_RESOURCE_ENTRIES} entries, and at most + * {@link SKILL_MAX_TOTAL_BYTES} of declared content. + */ +function boundedManifest( + declared: readonly SkillResource[], +): readonly SkillResource[] { + const kept: SkillResource[] = []; + let bytes = 0; + for (const resource of declared) { + if (kept.length >= SKILL_MAX_RESOURCE_ENTRIES) break; + // Only a usable, non-negative size counts — matching `totalSkillBytes`, so + // the bound and the finding that reports the overage agree on the total. + const size = + typeof resource.size === "number" && + Number.isSafeInteger(resource.size) && + resource.size >= 0 + ? resource.size + : 0; + if (bytes + size > SKILL_MAX_TOTAL_BYTES) break; + bytes += size; + kept.push(resource); + } + return kept; +} + /** Result shape of one `resources/read`, narrowed to what a digest needs. */ interface ReadContents { text?: string; @@ -196,14 +229,24 @@ export async function verifySkills( const declared = entry.resources === DYNAMIC_RESOURCES ? [] : entry.resources; - // ⚠️ **Bounded, because the manifest is server-controlled.** The 512-entry - // limit is CHECKED by `checkSkillConformance` — as a warning, since the SEP - // makes it an interoperability bound rather than a MUST — but checking it - // constrains nothing, so a hostile or broken server advertising a million - // entries had the tool perform a million sequential reads after the report - // already knew the manifest was over the limit (Copilot). The overage is - // reported by `resource-limit-exceeded`; reading it is what stops here. - const manifest = declared.slice(0, SKILL_MAX_RESOURCE_ENTRIES); + // ⚠️ **Bounded on BOTH interoperability limits, because the manifest is + // server-controlled.** `checkSkillConformance` reports when either is + // exceeded — as warnings, since the SEP makes them bounds rather than MUSTs + // — but checking constrains nothing, and this loop then downloads whatever + // was advertised anyway. + // + // The entry count alone is not enough: a manifest can sit at exactly 512 + // entries and declare a gigabyte each, so bounding only the count still let + // a server dictate unbounded bandwidth and time (Copilot). Both overages + // are reported by `resource-limit-exceeded` / `size-limit-exceeded`; what + // stops here is *reading* them. + // + // Sizes are summed the way `totalSkillBytes` sums them — an entry + // declaring none contributes nothing, which cannot overstate the total and + // is bounded by the count cap regardless. A read is skipped only when the + // running total would CROSS the limit, so a conforming skill (≤ 16 MiB in + // total, by definition) is never truncated. + const manifest = boundedManifest(declared); const entryIdentity = skillUriIdentity(entry.uri); // Compared by NORMALIZED identity, like every other URI comparison here — // `checkSkillConformance` already accepts a manifest self-entry written in diff --git a/vitest.shared.mts b/vitest.shared.mts index 6f169e4855..27d252adf5 100644 --- a/vitest.shared.mts +++ b/vitest.shared.mts @@ -65,9 +65,15 @@ export function vitestSharedPaths(clientDir: string) { // is a statement about the installed tree, not about the manifests — check // the tree before adding or removing an entry. // - // `express` and `yaml` are reached only through `test-servers/src` — - // express by the http/oauth servers, yaml by `load-config.ts` — which is - // root-owned code with no manifest of its own. + // `express` is reached only through `test-servers/src` (the http/oauth + // servers), which is root-owned code with no manifest of its own. + // + // `yaml` was too — `load-config.ts` — but is now also a `core/` runtime + // import: `core/mcp/skillFile.ts` parses a served SKILL.md's frontmatter + // for the SEP-2640 cross-check (#2248). That matters to anyone revisiting + // this pin: it is no longer removable by retiring a test-server path, and + // as a dependency `core/` imports it is additionally named in all three + // bundler `external` lists. // // Pointing these at `/node_modules` is what broke when the MCP // packages moved to the root (#1970): express was never declared by a client From 27f1b2f1bd165b7cbaa34a7c390146ae595588ed Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 21:57:48 -0400 Subject: [PATCH 14/21] fix: address Copilot review round 12 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The read bounds could produce a false `ok: true`.** Entries past the cap are never fetched, `fileFailed` only examines fetched rows, and `resource-limit-exceeded` is a WARNING — so a manifest whose 513th file was tampered with reported the skill verified. That trades a denial of service for a wrong answer, which is the failure this whole PR argues against. A truncated read is now recorded as `incomplete` and forces `ok: false`, reported separately so a consumer can tell "this skill is wrong" from "this skill was not fully checked". And a self entry excluded by the cap is now verified against its declared digest, not merely read for frontmatter — otherwise the skill's own SKILL.md was the one file nobody checked. **`entryKey` could crash the TUI.** It called `JSON.stringify` on the whole server-controlled entry, during render. `skillEntryKey` is decided by the same `jsonGraphError` guard that bounds the frontmatter comparison, with a coarse fallback that still separates two unrepresentable entries by URI — collapsing them would show one skill's verdict under another's name. **The tab bar wraps and `tabsHeight` was hard-coded to 1.** An OAuth-capable stdio server serving Skills needs ~107 columns, so it wraps at 132 as well as at 80, and every pane below was sized a row too tall. `tabBarRows` derives it from the same list `Tabs` renders, via a shared `visibleTabs(flags)` — the duplicated filter being how the two would drift apart again. Nine tests in a new `tabsConfig.test.ts`, including the 80-column regression. Two assert properties rather than instances: rows never decrease as the terminal narrows, and a tab wider than the row gets its own rather than looping. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/tui/__tests__/tabsConfig.test.ts | 101 ++++++++++++++++++ clients/tui/src/App.tsx | 30 +++++- clients/tui/src/components/SkillsTab.tsx | 17 +-- clients/tui/src/components/Tabs.tsx | 23 ++-- clients/tui/src/components/tabsConfig.ts | 70 ++++++++++++ clients/web/src/test/core/mcp/skills.test.ts | 47 ++++++++ .../test/core/mcp/skillsVerification.test.ts | 62 +++++++++++ core/mcp/skills.ts | 24 +++++ core/mcp/skillsVerification.ts | 39 ++++++- 9 files changed, 380 insertions(+), 33 deletions(-) create mode 100644 clients/tui/__tests__/tabsConfig.test.ts diff --git a/clients/tui/__tests__/tabsConfig.test.ts b/clients/tui/__tests__/tabsConfig.test.ts new file mode 100644 index 0000000000..2bdb41b5fe --- /dev/null +++ b/clients/tui/__tests__/tabsConfig.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect } from "vitest"; +import { + tabBarRows, + tabs, + visibleTabs, + type TabType, +} from "../src/components/tabsConfig.js"; + +describe("tab accelerators", () => { + it("are unique and appear in their own label", () => { + const seen = new Set(); + for (const tab of tabs) { + expect(tab.label.toLowerCase()).toContain(tab.accelerator); + expect(seen.has(tab.accelerator)).toBe(false); + seen.add(tab.accelerator); + } + }); +}); + +describe("visibleTabs", () => { + it("drops every optional tab when nothing is supported", () => { + const ids = visibleTabs({ + showAuth: false, + showLogging: false, + showRequests: false, + showSkills: false, + }).map((t) => t.id); + expect(ids).not.toContain("auth"); + expect(ids).not.toContain("logging"); + expect(ids).not.toContain("requests"); + expect(ids).not.toContain("skills"); + // The unconditional ones remain. + expect(ids).toContain("info"); + expect(ids).toContain("tools"); + }); + + it("keeps each optional tab when its flag is set", () => { + const ids = visibleTabs({ + showAuth: true, + showLogging: true, + showRequests: true, + showSkills: true, + }).map((t) => t.id); + expect(ids).toEqual(tabs.map((t) => t.id)); + }); +}); + +describe("tabBarRows (#2248)", () => { + /** A stdio, OAuth-capable, Skills-serving server: the widest ordinary bar. */ + const stdioSkills = visibleTabs({ + showAuth: true, + showLogging: true, + showRequests: false, + showSkills: true, + }); + const counts: Partial> = { + resources: 0, + prompts: 0, + skills: 8, + tools: 1, + messages: 11, + logging: 3, + }; + + it("wraps that bar at 80 columns", () => { + // The regression this exists for: adding Skills pushed the bar past a + // default terminal, while `App` assumed one row and sized every pane below + // it one row too tall. + expect(tabBarRows(stdioSkills, counts, 80)).toBeGreaterThan(1); + }); + + it("needs only one row when the bar fits", () => { + expect(tabBarRows(stdioSkills, counts, 400)).toBe(1); + }); + + it("never reports fewer rows as the terminal narrows", () => { + // Monotonicity is the property that matters: a narrower terminal can only + // need the same number of rows or more, so a pane sized from this can + // never grow into the bar. + let previous = 1; + for (const width of [400, 200, 132, 100, 80, 60, 40, 20]) { + const rows = tabBarRows(stdioSkills, counts, width); + expect(rows).toBeGreaterThanOrEqual(previous); + previous = rows; + } + }); + + it("gives a tab wider than the row its own row rather than looping", () => { + expect(tabBarRows(stdioSkills, counts, 1)).toBe(stdioSkills.length); + }); + + it("counts the count suffixes, which are what tip it over", () => { + const withCounts = tabBarRows(stdioSkills, counts, 100); + const without = tabBarRows(stdioSkills, {}, 100); + expect(withCounts).toBeGreaterThanOrEqual(without); + }); + + it("returns one row for an empty bar", () => { + expect(tabBarRows([], {}, 80)).toBe(1); + }); +}); diff --git a/clients/tui/src/App.tsx b/clients/tui/src/App.tsx index e95a1f0eda..2bdbe50ead 100644 --- a/clients/tui/src/App.tsx +++ b/clients/tui/src/App.tsx @@ -81,6 +81,7 @@ import { InfoTab } from "./components/InfoTab.js"; import { AuthTab } from "./components/AuthTab.js"; import { ResourcesTab } from "./components/ResourcesTab.js"; import { PromptsTab } from "./components/PromptsTab.js"; +import { tabBarRows, visibleTabs } from "./components/tabsConfig.js"; import { SkillsTab } from "./components/SkillsTab.js"; import { ToolsTab } from "./components/ToolsTab.js"; import { NotificationsTab } from "./components/NotificationsTab.js"; @@ -1621,14 +1622,37 @@ function App({ // Calculate layout dimensions const headerHeight = 1; - const tabsHeight = 1; + const serverListWidth = Math.floor(dimensions.width * 0.3); + const contentWidth = dimensions.width - serverListWidth; + // Derived, not assumed. The bar wraps once the visible tabs exceed the + // terminal width — which a stdio server with Skills does at any ordinary + // width — and a hard-coded 1 sized every pane below it one row too tall, + // clipping the bottom of the TUI (Copilot). + const tabsHeight = tabBarRows( + visibleTabs({ + showAuth: !!( + selectedServer && + selectedServerConfig && + isOAuthCapableServerConfig(selectedServerConfig) + ), + showLogging: + !!selectedServer && + inspectorClients[selectedServer]?.getServerType() === "stdio", + showRequests: + !!selectedServer && + (inspectorClients[selectedServer]?.getServerType() === "sse" || + inspectorClients[selectedServer]?.getServerType() === + "streamable-http"), + showSkills: showSkillsTab, + }), + tabCounts, + contentWidth, + ); // Server details will be flexible - calculate remaining space for content const availableHeight = dimensions.height - headerHeight - tabsHeight; // Reserve space for server details (will grow as needed, but we'll use flexGrow) const serverDetailsMinHeight = 3; const contentHeight = availableHeight - serverDetailsMinHeight; - const serverListWidth = Math.floor(dimensions.width * 0.3); - const contentWidth = dimensions.width - serverListWidth; const getStatusColor = (status: string) => { switch (status) { diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index 7e72574c66..b1bf292d77 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -27,6 +27,7 @@ import { checkSkillConformance, checkSkillNameCollisions, skillDisplayName, + skillEntryKey, skillUriIdentity, type SkillIssue, } from "@inspector/core/mcp/skills.js"; @@ -87,18 +88,6 @@ const FILE_COLOR: Record = { "read-error": "red", }; -/** - * What a verification result is a result *about*: the whole entry, serialized. - * - * `JSON.stringify` is enough here — this compares an entry against a later copy - * of *itself* from the same server, so key order is stable and there is no need - * for the canonical form `skillEntriesMatch` uses to compare two independently - * produced entries. - */ -function entryKey(entry: SkillEntry): string { - return JSON.stringify(entry); -} - /** * The explanation printed under a failed file row. * @@ -185,7 +174,7 @@ export function SkillsTab({ void (async () => { try { const [result] = await verifySkills(inspectorClient, [skill]); - setReport({ key: entryKey(skill), result }); + setReport({ key: skillEntryKey(skill), result }); } catch (err) { if (err instanceof AuthRecoveryRequiredError) { onAuthRecoveryRequired?.(err); @@ -265,7 +254,7 @@ export function SkillsTab({ }; const issues = selectedSkill ? findingsFor(selectedSkill) : []; const activeReport = - selectedSkill && report?.key === entryKey(selectedSkill) + selectedSkill && report?.key === skillEntryKey(selectedSkill) ? report.result : null; const manifest = diff --git a/clients/tui/src/components/Tabs.tsx b/clients/tui/src/components/Tabs.tsx index 71e447dc0c..2a23d21689 100644 --- a/clients/tui/src/components/Tabs.tsx +++ b/clients/tui/src/components/Tabs.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Box, Text } from "ink"; -import { type TabType, tabs } from "./tabsConfig.js"; +import { type TabType, visibleTabs as visibleTabsFor } from "./tabsConfig.js"; /** * Split a tab label so the accelerator letter can be underlined wherever it @@ -59,19 +59,14 @@ export function Tabs({ showRequests = false, showSkills = false, }: TabsProps) { - let visibleTabs = tabs; - if (!showAuth) { - visibleTabs = visibleTabs.filter((tab) => tab.id !== "auth"); - } - if (!showLogging) { - visibleTabs = visibleTabs.filter((tab) => tab.id !== "logging"); - } - if (!showRequests) { - visibleTabs = visibleTabs.filter((tab) => tab.id !== "requests"); - } - if (!showSkills) { - visibleTabs = visibleTabs.filter((tab) => tab.id !== "skills"); - } + // Shared with `App`, which sizes the pane below this bar from the same list — + // see `tabBarRows`. + const visibleTabs = visibleTabsFor({ + showAuth, + showLogging, + showRequests, + showSkills, + }); return ( { + if (tab.id === "auth") return v.showAuth; + if (tab.id === "logging") return v.showLogging; + if (tab.id === "requests") return v.showRequests; + if (tab.id === "skills") return v.showSkills; + return true; + }); +} + +/** Rendered width of one tab: the 2-column marker, the label, and any count. */ +function tabWidth( + tab: { id: TabType; label: string }, + counts: Partial>, +): number { + const count = counts[tab.id]; + return ( + 2 + tab.label.length + (count === undefined ? 0 : ` (${count})`.length) + ); +} + +/** + * How many terminal rows the tab bar occupies at a given width. + * + * ⚠️ **Not always 1.** Adding Skills pushed a stdio server's bar past 100 + * columns, so it wraps at any ordinary terminal width — and `App` hard-coded + * `tabsHeight = 1`, sizing every content pane one row too tall and clipping the + * bottom of the TUI (Copilot). Deriving the height from the same list `Tabs` + * renders is what keeps the two in agreement as tabs are added. + * + * The bar is a `flexWrap="wrap"` row with one column of padding each side and + * no gaps, so greedy packing by rendered width matches what Ink lays out. + */ +export function tabBarRows( + visible: readonly { id: TabType; label: string }[], + counts: Partial>, + width: number, +): number { + const inner = Math.max(1, width - 2); + let rows = 1; + let used = 0; + for (const tab of visible) { + const w = tabWidth(tab, counts); + // A tab wider than the whole row still occupies one of its own rather than + // looping forever. + if (used > 0 && used + w > inner) { + rows += 1; + used = w; + } else { + used += w; + } + } + return rows; +} diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index f7e40eb3dd..ebcf1a1111 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -9,6 +9,7 @@ import { checkSkillConformance, checkSkillFrontmatterMatch, checkSkillNameCollisions, + skillEntryKey, getSkillsExtension, isSkillsExtensionSupported, normalizeSkillUri, @@ -1201,3 +1202,49 @@ describe("checkSkillNameCollisions (#2248)", () => { ); }); }); + +describe("skillEntryKey (#2248)", () => { + const base: SkillEntry = { + uri: "skill://demo/SKILL.md", + frontmatter: { name: "demo", description: "A demo" }, + resources: [], + }; + + it("distinguishes two entries that differ anywhere", () => { + expect(skillEntryKey(base)).toBe(skillEntryKey({ ...base })); + expect(skillEntryKey(base)).not.toBe( + skillEntryKey({ + ...base, + frontmatter: { name: "demo", description: "changed" }, + }), + ); + }); + + it("survives frontmatter too deep to serialize", () => { + // `frontmatter` is unbounded server-controlled JSON, and this key is + // computed during render — so `JSON.stringify` let one catalog entry crash + // the pane that exists to report on it (Copilot). + let deep: unknown = "leaf"; + for (let i = 0; i < 60000; i += 1) deep = { a: deep }; + const hostile = { + ...base, + frontmatter: { name: "demo", deep }, + } as unknown as SkillEntry; + expect(() => skillEntryKey(hostile)).not.toThrow(); + expect(skillEntryKey(hostile)).toContain("unrepresentable"); + }); + + it("still separates two unrepresentable entries by identity", () => { + // The fallback is coarse, but it must not collapse distinct skills into + // one key — that would show a verdict under the wrong name. + let deep: unknown = "leaf"; + for (let i = 0; i < 60000; i += 1) deep = { a: deep }; + const a = { ...base, frontmatter: { deep } } as unknown as SkillEntry; + const b = { + ...base, + uri: "skill://other/SKILL.md", + frontmatter: { deep }, + } as unknown as SkillEntry; + expect(skillEntryKey(a)).not.toBe(skillEntryKey(b)); + }); +}); diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 5f642447c5..44323c2bc9 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -538,6 +538,7 @@ describe("verifySkills (#2248)", () => { const [report] = await verifySkills(client, [skill]); expect(readResource).toHaveBeenCalledTimes(512); expect(report.files).toHaveLength(512); + expect(report.ok).toBe(false); // …and the overage is still REPORTED, so bounding the reads does not // silence the finding that made them unnecessary. expect(report.conformance).toEqual( @@ -581,6 +582,9 @@ describe("verifySkills (#2248)", () => { const [report] = await verifySkills(client, [skill]); // Two fit exactly; the third would cross, so it is never requested. expect(readResource).toHaveBeenCalledTimes(2); + // …and a verification that did not finish must not report success. + expect(report.incomplete).toMatch(/2 of 3 manifest entries/); + expect(report.ok).toBe(false); expect(report.conformance).toEqual( expect.arrayContaining([ expect.objectContaining({ code: "size-limit-exceeded" }), @@ -588,6 +592,61 @@ describe("verifySkills (#2248)", () => { ); }); + it("does not report success for a manifest it could not finish reading", async () => { + // The trade this bound must NOT make: entries past the cap are never + // fetched and `resource-limit-exceeded` is only a warning, so a manifest + // whose 513th file is tampered with returned `ok: true` and the CLI said + // the skill verified — a denial of service swapped for a false pass + // (Copilot). + const skill: SkillEntry = { + uri: "skill://many/SKILL.md", + frontmatter: { name: "many", description: "Over the entry limit" }, + resources: Array.from({ length: 600 }, (_, i) => ({ + uri: i === 0 ? "skill://many/SKILL.md" : `skill://many/f${i}.md`, + digest: `sha256:${"a".repeat(64)}`, + size: 1, + })), + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: "x" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(report.incomplete).toBeDefined(); + expect(report.ok).toBe(false); + }); + + it("verifies the entry's own file even when the cap excluded it", async () => { + // The fallback exists for the frontmatter check, but reading the file and + // then skipping the digest its manifest advertised would leave the skill's + // own SKILL.md the one file nobody verified (Copilot). + const skill: SkillEntry = { + uri: "skill://huge/SKILL.md", + frontmatter: { name: "huge", description: "Listed" }, + resources: [ + ...Array.from({ length: 600 }, (_, i) => ({ + uri: `skill://huge/f${i}.md`, + digest: `sha256:${"a".repeat(64)}`, + size: 1, + })), + { + uri: "skill://huge/SKILL.md", + digest: `sha256:${"b".repeat(64)}`, + size: 1, + }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: "x" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + const self = report.files.find((f) => f.uri === "skill://huge/SKILL.md"); + // A real verdict on the advertised digest, not merely a read. + expect(self?.status).toBe("mismatch"); + expect(self?.expectedDigest).toBe(`sha256:${"b".repeat(64)}`); + }); + it("does not truncate a conforming manifest", async () => { // A conforming skill totals at most 16 MiB by definition, so the bound // must never shorten one — otherwise it would trade a hostile-server @@ -600,6 +659,9 @@ describe("verifySkills (#2248)", () => { const [report] = await verifySkills(client, [skill]); expect(readResource).toHaveBeenCalledTimes(2); expect(report.files).toHaveLength(2); + // Nothing was skipped, so nothing is reported as incomplete. + expect(report.incomplete).toBeUndefined(); + expect(report.ok).toBe(true); }); it("still reads the entry's own file when the cap would exclude it", async () => { diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index 1565abfdd9..8ae718d554 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -513,6 +513,30 @@ export function totalSkillBytes(resources: readonly SkillResource[]): number { ); } +/** + * A stable key for "this exact entry", safe against a hostile listing. + * + * `JSON.stringify(entry)` is the obvious implementation and is the wrong one: + * `frontmatter` is unbounded server-controlled JSON, so a deep enough object + * throws `RangeError: Maximum call stack size exceeded` — and this is evaluated + * during render, so one catalog entry could crash the pane that exists to + * report on it (Copilot). + * + * The same guard that bounds the frontmatter comparison decides it here. When + * the entry is representable the key is its serialization, which is exact; + * when it is not, the key falls back to the entry's identity plus its manifest + * length. That fallback is deliberately coarse — such an entry already carries + * a `frontmatter-unparsable` error, so what matters is that it produces a + * usable key rather than a precise one. + */ +export function skillEntryKey(entry: SkillEntry): string { + if (jsonGraphError(entry) !== undefined) { + const count = Array.isArray(entry.resources) ? entry.resources.length : -1; + return `${skillUriIdentity(entry.uri)}#unrepresentable:${count}`; + } + return JSON.stringify(entry); +} + /** * Whether a `skills/get` entry describes the same skill as the `skills/list` * entry alongside it, compared **semantically** rather than byte-for-byte. diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 666dd263a3..2ced0087aa 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -91,9 +91,24 @@ export interface SkillVerifyReport { * `files` mirrors the manifest one-for-one (Copilot). */ files: SkillFileReport[]; + /** + * Why the manifest could not be checked in full, or `undefined` when it was. + * + * Set when the read bounds truncated the manifest. It is reported separately + * from `ok` being false so a consumer can tell "this skill is wrong" from + * "this skill was not fully checked" — but it does make `ok` false, because + * the alternative is worse: entries past the cap are never fetched, and + * `resource-limit-exceeded` is only a WARNING, so a manifest whose 513th + * file was tampered with reported `ok: true` and the CLI said the skill + * verified (Copilot). A bound that turns a denial of service into a false + * pass has traded down. + */ + incomplete?: string; /** * False when anything the SEP makes a MUST was broken: an error-severity - * finding, a digest or size mismatch, or a file that could not be read. + * finding, a digest or size mismatch, or a file that could not be read — + * **or when {@link incomplete} is set**, since a verification that did not + * finish cannot report success. * * A `warning` does **not** clear it — a `"dynamic"` manifest is legal, and a * report that failed CI for it would be telling server authors their @@ -247,6 +262,10 @@ export async function verifySkills( // running total would CROSS the limit, so a conforming skill (≤ 16 MiB in // total, by definition) is never truncated. const manifest = boundedManifest(declared); + const incomplete = + manifest.length < declared.length + ? `Only ${manifest.length} of ${declared.length} manifest entries were read: the skill exceeds the ${SKILL_MAX_RESOURCE_ENTRIES}-entry / ${SKILL_MAX_TOTAL_BYTES}-byte interoperability limits, so the rest were not fetched and cannot be reported on.` + : undefined; const entryIdentity = skillUriIdentity(entry.uri); // Compared by NORMALIZED identity, like every other URI comparison here — // `checkSkillConformance` already accepts a manifest self-entry written in @@ -327,6 +346,20 @@ export async function verifySkills( fail(reasonOf(err)); } } + // If the DECLARED manifest lists this file but the read bounds + // excluded it, verify it here too. The fallback exists for the + // frontmatter check, but reading a file and then not checking the + // digest the manifest advertised for it would leave the entry's own + // SKILL.md the one file nobody verified (Copilot). + const declaredSelf = declared.find( + (resource) => skillUriIdentity(resource.uri) === entryIdentity, + ); + if (declaredSelf && entryBytes !== undefined) { + files.push({ + uri: entry.uri, + ...(await verifySkillResource(declaredSelf, entryBytes)), + }); + } } catch (err) { // An expired authorization is the one error that is not this file's // problem — see the note on the function. @@ -359,7 +392,9 @@ export async function verifySkills( conformance, frontmatter, files, - ok: !hasError && !fileFailed, + ...(incomplete ? { incomplete } : {}), + // An unfinished verification is not a passing one. + ok: !hasError && !fileFailed && incomplete === undefined, }); } return reports; From 4f12d07d2bb287a8914bbd96c7acad86031e6b76 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 22:09:19 -0400 Subject: [PATCH 15/21] fix: address Copilot review round 13 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The byte budget trusted server-declared sizes.** A manifest advertising `size: 1` — or, since the wire schema deliberately accepts it for reporting, no size at all — sailed through the declared budget and then served arbitrarily large bodies. The 16 MiB safeguard protected against an honest server only, which is no safeguard. The walk now tracks bytes ACTUALLY received and stops issuing reads once the real total crosses the cap, marking the report `incomplete` and so `ok: false`. The declared prefilter is kept rather than replaced: it refuses to schedule an obviously oversized set, and the received-byte counter stops one that lied. The count happens after verifying the crossing file, so that file is still reported rather than fetched and discarded. ⚠️ This bounds the total across responses, not any single one: a first response larger than the cap is already in memory before it can be measured, which would need a streaming read the client API does not expose. Stated in the code rather than left to look closed. **A modern `skills/get` did not require `resultType`.** The "left open" question SEP-2640 states covers `ttlMs` / `cacheScope` and only those; `resultType` is base-protocol per SEP-2322 and appears in the SEP's own `skills/get` example. Requiring it of `resources/directory/read` but not here was an inconsistency in the module rather than a distinction the spec draws. Era-selected now, with the caching attributes still optional. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../core/mcp/inspectorClient-skills.test.ts | 28 ++++++++++ .../src/test/core/mcp/skillsSchemas.test.ts | 26 +++++++++ .../test/core/mcp/skillsVerification.test.ts | 55 +++++++++++++++++++ core/mcp/inspectorClient.ts | 13 ++++- core/mcp/skillsSchemas.ts | 15 +++++ core/mcp/skillsVerification.ts | 22 +++++++- 6 files changed, 156 insertions(+), 3 deletions(-) diff --git a/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts b/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts index 8c2c1dcb18..a254db9493 100644 --- a/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts +++ b/clients/web/src/test/core/mcp/inspectorClient-skills.test.ts @@ -388,6 +388,34 @@ describe("InspectorClient skills methods (#2234)", () => { }); }); + it("requires resultType on a modern skills/get", async () => { + // Base-protocol (SEP-2322) and present in SEP-2640's own example, unlike + // the caching attributes the SEP leaves open. + const client = makeClient(); + internals(client).protocolEra = "modern"; + stubRequest(client, { skill: ENTRY }); + await expect( + client.getSkill("skill://demo/SKILL.md"), + ).rejects.toBeDefined(); + }); + + it("accepts a modern skills/get without the caching attributes", async () => { + const client = makeClient(); + internals(client).protocolEra = "modern"; + stubRequest(client, { skill: ENTRY, resultType: "complete" }); + await expect(client.getSkill("skill://demo/SKILL.md")).resolves.toEqual( + ENTRY, + ); + }); + + it("accepts a legacy skills/get without resultType", async () => { + const client = makeClient(); + stubRequest(client, { skill: ENTRY }); + await expect(client.getSkill("skill://demo/SKILL.md")).resolves.toEqual( + ENTRY, + ); + }); + it("rejects a skills/list result that is not a skills page", async () => { // The explicit result schema is the whole client-side mechanism for a // consumer-owned extension method, so a nonconforming result must fail diff --git a/clients/web/src/test/core/mcp/skillsSchemas.test.ts b/clients/web/src/test/core/mcp/skillsSchemas.test.ts index 2bbe3407c8..53f991e650 100644 --- a/clients/web/src/test/core/mcp/skillsSchemas.test.ts +++ b/clients/web/src/test/core/mcp/skillsSchemas.test.ts @@ -5,7 +5,9 @@ import { ModernDirectoryReadResultSchema, RESOURCES_DIRECTORY_READ_METHOD, DYNAMIC_RESOURCES, + GetSkillEnvelopeSchema, GetSkillResultSchema, + ModernGetSkillEnvelopeSchema, ListSkillsResultSchema, ModernListSkillsResultSchema, SKILLS_EXTENSION_KEY, @@ -283,4 +285,28 @@ describe("GetSkillResultSchema caching attributes (#2248)", () => { }).success, ).toBe(true); }); + + it("requires resultType on the modern envelope, but still not the caching fields", () => { + // "Left open" covers `ttlMs` / `cacheScope` and only those. `resultType` is + // base-protocol (SEP-2322) and appears in SEP-2640's own `skills/get` + // example, so leaving it optional here while requiring it of + // `resources/directory/read` was an inconsistency in this module rather + // than a distinction the spec draws (Copilot). + expect( + ModernGetSkillEnvelopeSchema.safeParse({ skill: ENTRY }).success, + ).toBe(false); + expect( + ModernGetSkillEnvelopeSchema.safeParse({ + skill: ENTRY, + resultType: "complete", + }).success, + ).toBe(true); + }); + + it("keeps the legacy envelope permissive about resultType", () => { + // A 2026-era member a legacy server has no business sending. + expect(GetSkillEnvelopeSchema.safeParse({ skill: ENTRY }).success).toBe( + true, + ); + }); }); diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 44323c2bc9..de12de11d5 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -647,6 +647,61 @@ describe("verifySkills (#2248)", () => { expect(self?.expectedDigest).toBe(`sha256:${"b".repeat(64)}`); }); + it("stops on bytes ACTUALLY served, not the sizes the manifest declared", async () => { + // The declared budget is server-controlled: advertising `size: 1` and then + // serving megabytes sailed straight through it, defeating the 16 MiB + // safeguard entirely (Copilot). + const big = "x".repeat(6 * 1024 * 1024); + const skill: SkillEntry = { + uri: "skill://liar/SKILL.md", + frontmatter: { name: "liar", description: "Understates its sizes" }, + resources: Array.from({ length: 10 }, (_, i) => ({ + uri: i === 0 ? "skill://liar/SKILL.md" : `skill://liar/f${i}.md`, + digest: `sha256:${"a".repeat(64)}`, + size: 1, // a lie + })), + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: big }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + // Three 6 MiB bodies cross 16 MiB; the walk stops rather than reading ten. + expect(readResource).toHaveBeenCalledTimes(3); + expect(report.incomplete).toMatch(/actually served/); + expect(report.ok).toBe(false); + }); + + it("still reports the file that crossed the byte budget", async () => { + // The crossing file is verified before the walk stops, so its verdict is + // not fetched and then thrown away. + const big = "x".repeat(17 * 1024 * 1024); + const skill: SkillEntry = { + uri: "skill://liar/SKILL.md", + frontmatter: { name: "liar", description: "One enormous file" }, + resources: [ + { + uri: "skill://liar/SKILL.md", + digest: `sha256:${"a".repeat(64)}`, + size: 1, + }, + { + uri: "skill://liar/b.md", + digest: `sha256:${"a".repeat(64)}`, + size: 1, + }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: big }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(readResource).toHaveBeenCalledTimes(1); + expect(report.files).toHaveLength(1); + expect(report.files[0].status).toBe("mismatch"); + }); + it("does not truncate a conforming manifest", async () => { // A conforming skill totals at most 16 MiB by definition, so the bound // must never shorten one — otherwise it would trade a hostile-server diff --git a/core/mcp/inspectorClient.ts b/core/mcp/inspectorClient.ts index be9bba98c8..cec72b4959 100644 --- a/core/mcp/inspectorClient.ts +++ b/core/mcp/inspectorClient.ts @@ -148,6 +148,7 @@ import { DirectoryReadResultSchema, GetSkillEnvelopeSchema, ListSkillsResultSchema, + ModernGetSkillEnvelopeSchema, ModernDirectoryReadResultSchema, ModernListSkillsResultSchema, RESOURCES_DIRECTORY_READ_METHOD, @@ -5640,13 +5641,21 @@ export class InspectorClient extends InspectorClientEventTarget { uri, ...(effectiveMeta ? { _meta: effectiveMeta } : {}), }; - // The envelope is returned whole; `getSkill` is the one that unwraps. + // Era-aware for the same reason `skills/list` is: the method is + // consumer-owned, so no SDK codec stamps or checks its envelope. The modern + // variant requires `resultType` — a base-protocol member SEP-2322 puts on + // every modern result — and still not the caching attributes, which + // SEP-2640 leaves open. The envelope is returned whole; `getSkill` + // unwraps. + const resultSchema = this.isModernEra() + ? ModernGetSkillEnvelopeSchema + : GetSkillEnvelopeSchema; try { return await this.invokeMcpClient( () => this.client!.request( { method: SKILLS_GET_METHOD, params }, - GetSkillEnvelopeSchema, + resultSchema, this.getRequestOptions(this.progressTokenOf(metadata)), ), { method: SKILLS_GET_METHOD }, diff --git a/core/mcp/skillsSchemas.ts b/core/mcp/skillsSchemas.ts index 4a36ec87fa..79e76341d2 100644 --- a/core/mcp/skillsSchemas.ts +++ b/core/mcp/skillsSchemas.ts @@ -183,6 +183,21 @@ export const GetSkillEnvelopeSchema = z.looseObject({ */ export type GetSkillEnvelope = z.infer; +/** + * `skills/get` on a **modern** (2026-07-28+) connection: the envelope plus + * `resultType`, and deliberately still not the caching attributes. + * + * The "left open" quote above covers `ttlMs` / `cacheScope` and **only** those. + * `resultType` is a different thing: SEP-2322 makes it a member of every modern + * result, and SEP-2640's own `skills/get` example carries + * `"resultType": "complete"`. Leaving it optional here while requiring it of + * `resources/directory/read` was an inconsistency in this module rather than a + * distinction the spec draws (Copilot). + */ +export const ModernGetSkillEnvelopeSchema = GetSkillEnvelopeSchema.extend({ + resultType: z.literal("complete"), +}); + /** * `skills/get` result, unwrapped to the entry it carries. * diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 2ced0087aa..370c4bc250 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -262,10 +262,19 @@ export async function verifySkills( // running total would CROSS the limit, so a conforming skill (≤ 16 MiB in // total, by definition) is never truncated. const manifest = boundedManifest(declared); - const incomplete = + let incomplete = manifest.length < declared.length ? `Only ${manifest.length} of ${declared.length} manifest entries were read: the skill exceeds the ${SKILL_MAX_RESOURCE_ENTRIES}-entry / ${SKILL_MAX_TOTAL_BYTES}-byte interoperability limits, so the rest were not fetched and cannot be reported on.` : undefined; + // ⚠️ Bytes ACTUALLY RECEIVED, which is the only budget a server cannot + // lie its way past. `boundedManifest` above works from DECLARED sizes, and + // those are server-controlled — a manifest advertising `size: 1` (or, since + // the wire schema deliberately accepts it for reporting, no size at all) + // sailed through the declared budget and then served arbitrarily large + // bodies, defeating the 16 MiB safeguard entirely (Copilot). The declared + // prefilter still earns its place by refusing to *schedule* an obviously + // oversized set; this is what stops one that lied. + let receivedBytes = 0; const entryIdentity = skillUriIdentity(entry.uri); // Compared by NORMALIZED identity, like every other URI comparison here — // `checkSkillConformance` already accepts a manifest self-entry written in @@ -311,6 +320,17 @@ export async function verifySkills( if (skillUriIdentity(resource.uri) === entryIdentity) entryBytes = bytes; const verification = await verifySkillResource(resource, bytes); files.push({ uri: resource.uri, ...verification }); + // Counted AFTER verifying this file, so the one that crosses the line is + // still reported rather than fetched and discarded. The next read is what + // stops. ⚠️ This bounds the total across responses, not the size of any + // single one: a first response larger than the cap is already in memory + // by the time it can be measured, which would need a streaming read to + // prevent and is not something this API exposes. + receivedBytes += bytes.byteLength; + if (receivedBytes > SKILL_MAX_TOTAL_BYTES) { + incomplete = `Stopped after ${files.length} of ${declared.length} manifest entries: the files actually served exceed the ${SKILL_MAX_TOTAL_BYTES}-byte interoperability limit, whatever sizes the manifest declared.`; + break; + } } // A `"dynamic"` skill has no manifest, so the loop above read nothing — From 01a8f14c24aebfb08658eb8b056f481a6e0cefd5 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 22:20:20 -0400 Subject: [PATCH 16/21] fix: address Copilot review round 14 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The TUI ignored `incomplete`.** Round 12 added that field precisely so a consumer could tell "not fully checked" from a real failure, and then the one client that reads the report did not consume it — which made the field decorative. The pane shows an Incomplete: block with the report's own reason, and the status line reads INCOMPLETE rather than FAILED. It sits ABOVE the manifest, not beside the status line: it explains the list that follows — only the first N rows were fetched, the rest stay marked `·` because nobody looked at them — and below a 512-row manifest it would be off-screen, which is the same as absent. Found by writing the test, whose first version asserted against a 600-entry fixture and could see neither string. The test triggers truncation through the BYTE budget instead, so the manifest stays three rows and the assertions measure the pane rather than the test's viewport. **The capped self-entry was reported under the wrong URI.** A manifest may write its self-entry in a normalized-equivalent form; the fallback recorded `entry.uri`, so manifest rows keyed on the declared spelling matched nothing while the normalized extra-files filter suppressed it as already covered. The verdict existed in the report and appeared nowhere on screen. Recorded under `declaredSelf.uri` now. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/tui/__tests__/SkillsTab.test.tsx | 44 +++++++++++++++++++ clients/tui/src/components/SkillsTab.tsx | 24 +++++++++- .../test/core/mcp/skillsVerification.test.ts | 32 ++++++++++++++ core/mcp/skillsVerification.ts | 8 +++- 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index 03638e32c1..2513d374a5 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -750,6 +750,50 @@ describe("SkillsTab (#2248)", () => { ); }); + it("says a verification was INCOMPLETE rather than merely failed", async () => { + // `verifySkills` sets `incomplete` so a consumer can tell "not fully + // checked" from a real failure; printing only "Verification FAILED" threw + // that distinction away, and the entries beyond the cap stayed marked `·` + // with nothing explaining why (Copilot). + // + // Truncation is triggered by the BYTE budget rather than the 512-entry one + // so the manifest stays three rows long: a 512-row pane pushes the status + // line off the frame, which would make this assert the test's viewport + // rather than the pane's behaviour. + const big = "x".repeat(6 * 1024 * 1024); + const fat: SkillEntry = { + uri: "skill://fat/SKILL.md", + frontmatter: { name: "fat", description: "Understates its sizes" }, + resources: Array.from({ length: 3 }, (_, i) => ({ + uri: i === 0 ? "skill://fat/SKILL.md" : `skill://fat/f${i}.md`, + digest: CLEAN_DIGEST, + size: 1, + })), + }; + const { lastFrame, stdin } = render( + ({ + result: { contents: [{ uri, text: big }] }, + })), + )} + width={160} + height={40} + focusedPane="list" + />, + ); + stdin.write(ENTER); + await tick(); + await tick(); + const frame = lastFrame() ?? ""; + expect(frame).toContain("Incomplete:"); + expect(frame).toContain("actually served"); + expect(frame).toContain("Verification INCOMPLETE"); + expect(frame).not.toContain("Verification FAILED"); + }); + it("shows a read failure the manifest does not cover", async () => { // A dynamic skill has no manifest rows, so the synthetic read-error row // `verifySkills` records for its own SKILL.md was rendered nowhere and the diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index b1bf292d77..cdbeecff57 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -402,6 +402,26 @@ export function SkillsTab({ ))} + {/* ABOVE the manifest, because it explains the list that + follows: only the first N rows were fetched, and the rest + stay marked `·` because nobody looked at them. Below a + 512-row manifest it would be off-screen, which is the same as + absent. `verifySkills` sets `incomplete` precisely so a + consumer can tell "not fully checked" from a real failure + (Copilot). */} + {activeReport?.incomplete && ( + <> + + + Incomplete: + + + + {activeReport.incomplete} + + + )} + Manifest @@ -504,7 +524,9 @@ export function SkillsTab({ : activeReport ? activeReport.ok ? "[Verified — Enter to re-verify]" - : "[Verification FAILED — Enter to re-verify]" + : activeReport.incomplete + ? "[Verification INCOMPLETE — Enter to re-verify]" + : "[Verification FAILED — Enter to re-verify]" : "[Enter to verify digests and frontmatter]"} diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index de12de11d5..6f3b9018dc 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -702,6 +702,38 @@ describe("verifySkills (#2248)", () => { expect(report.files[0].status).toBe("mismatch"); }); + it("reports the capped self-entry under the URI the MANIFEST declared", async () => { + // A manifest may write its self-entry in a normalized-equivalent form. The + // fallback recorded `entry.uri`, so a consumer matching rows against the + // manifest found nothing — while a normalized "extra files" filter + // suppressed it as already covered. The verdict existed in the report and + // appeared nowhere on screen (Copilot). + const declaredSpelling = "skill://huge/x/../SKILL.md"; + const skill: SkillEntry = { + uri: "skill://huge/SKILL.md", + frontmatter: { name: "huge", description: "Listed" }, + resources: [ + ...Array.from({ length: 600 }, (_, i) => ({ + uri: `skill://huge/f${i}.md`, + digest: `sha256:${"a".repeat(64)}`, + size: 1, + })), + { uri: declaredSpelling, digest: `sha256:${"b".repeat(64)}`, size: 1 }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, text: "x" }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + const self = report.files.find((f) => f.uri === declaredSpelling); + expect(self?.status).toBe("mismatch"); + // …and NOT under the entry's own spelling, which no manifest row carries. + expect(report.files.some((f) => f.uri === "skill://huge/SKILL.md")).toBe( + false, + ); + }); + it("does not truncate a conforming manifest", async () => { // A conforming skill totals at most 16 MiB by definition, so the bound // must never shorten one — otherwise it would trade a hostile-server diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 370c4bc250..e64f1955e6 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -376,7 +376,13 @@ export async function verifySkills( ); if (declaredSelf && entryBytes !== undefined) { files.push({ - uri: entry.uri, + // The DECLARED spelling, not the entry's. They can differ — a + // manifest may write its self-entry in a normalized-equivalent + // form — and a consumer matching rows against the manifest then + // finds nothing, while a normalized "extra files" filter suppresses + // it as already covered. The result was a verdict that existed in + // the report and appeared nowhere on screen (Copilot). + uri: declaredSelf.uri, ...(await verifySkillResource(declaredSelf, entryBytes)), }); } From 89438d1a4b8d08c879986ca6bd8c9f62c4a9e7a0 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 22:47:32 -0400 Subject: [PATCH 17/21] fix: address Copilot review round 15 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Verification is a tri-state now, because two states could not be right.** Round 12 was correct that `ok: true` for a manifest whose unread 513th file may be tampered with is a false pass. Round 15 is correct that forcing `ok: false` calls a server nonconformant for exceeding limits SEP-2640 states as SHOULD NOT, with hosts free to support more — contradicting this module's own rule that a warning never fails a report. Both hold, so: verified -> exit 0 everything checked, everything passed failed -> exit 7 a MUST was broken incomplete -> exit 8 nothing checked was wrong; the walk was cut short `ok` keeps its narrow meaning, `allSkillsVerified` is stricter than `every(r => r.ok)` since it selects the exit code, and `anySkillFailed` separates 7 from 8. A job that tolerates oversized catalogs can allow 8 and still fail on 7. **`manifestKey` still used `JSON.stringify` on the entry** — the same crash I fixed in the TUI in round 12, one file over. **Directory children were navigable outside the skill root.** A server could return `skill://other-skill/...` and clicking it left the tree, with "Up" only comparing equality against the root. Containment is checked on the normalized URI now, and an outside child renders as text rather than a link. Worth recording from the tests: `..` cannot escape the authority, so `skill://a/nested/../x.md` resolves back inside and must stay navigable. **The TUI matched report rows by raw URI**, which missed a row recorded under an equivalent spelling while `extraReportFiles` suppressed it as covered. Normalized, like the membership test beside it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/cli/README.md | 15 +- .../cli/__tests__/run-method-skills.test.ts | 45 +++- .../cli/__tests__/skills-verify-cli.test.ts | 27 +++ clients/cli/src/error-handler.ts | 13 + clients/cli/src/handlers/consume-outcome.ts | 9 +- clients/cli/src/handlers/run-method.ts | 21 +- clients/tui/src/components/SkillsTab.tsx | 9 +- .../SkillsScreen/SkillsScreen.test.tsx | 71 ++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 86 +++++-- .../test/core/mcp/skillsVerification.test.ts | 229 ++++++++++-------- core/mcp/skillsVerification.ts | 50 +++- 11 files changed, 434 insertions(+), 141 deletions(-) diff --git a/clients/cli/README.md b/clients/cli/README.md index 7a6d1dce88..aeacebd6fb 100644 --- a/clients/cli/README.md +++ b/clients/cli/README.md @@ -372,10 +372,16 @@ caller branching on `.code` should not have to special-case this command. `--method skills/get --uri ` verifies exactly one skill, in the same shape. -**What fails the run.** `ok` is false — and the exit code is `7` — for anything -SEP-2640 makes a MUST: an error-severity conformance finding, a digest or size -mismatch, or a manifest file that could not be read. A **warning** does not fail -it. That distinction matters most for `resources: "dynamic"`, which is a +**What fails the run.** Three outcomes, three exit codes, because "this skill is +wrong" and "this skill could not be fully checked" are different answers: + +| `outcome` | Exit | When | +| --- | --- | --- | +| `verified` | `0` | Everything was checked and everything passed. | +| `failed` | `7` | Something SEP-2640 makes a MUST was broken — an error-severity finding, a digest or size mismatch, or an unreadable manifest file. | +| `incomplete` | `8` | Nothing checked was wrong, but the read bounds stopped the walk before it finished. See `incomplete` in the report for the reason. | + +A **warning** never produces `7`. That distinction matters most for `resources: "dynamic"`, which is a *conforming* wire form for generated content: it means integrity cannot be verified, which is worth reporting, but failing CI for it would tell server authors their valid skill is broken. @@ -415,6 +421,7 @@ prose from stderr: | `5` | Tool error (`tools/call` returned `isError:true`, or the tool was not found). | | `6` | `--strict` found an error-severity tool-schema portability problem (`schema_unportable` — the schema is valid JSON Schema, just not portable). | | `7` | `--verify` found a SEP-2640 violation (`skills_nonconformant` — a conformance error, a digest or size mismatch, or an unreadable manifest file). | +| `8` | `--verify` could not check the whole catalog (`skills_incomplete` — the read bounds stopped the walk). The server broke no **MUST**: the 512-entry and 16 MiB limits are `SHOULD NOT`, and hosts may support more. A job that tolerates oversized catalogs can allow `8` and still fail on `7`. | On any non-zero exit the CLI also writes a single JSON line to **stderr** — the `ErrorEnvelope`: diff --git a/clients/cli/__tests__/run-method-skills.test.ts b/clients/cli/__tests__/run-method-skills.test.ts index 7074e4a0f2..e750f97c02 100644 --- a/clients/cli/__tests__/run-method-skills.test.ts +++ b/clients/cli/__tests__/run-method-skills.test.ts @@ -201,6 +201,47 @@ describe("runMethod skills dispatch (#2248)", () => { ); }); + it("exits 8, not 7, when the walk was only truncated", async () => { + // SEP-2640 states the read limits as SHOULD NOT and lets hosts support + // more, so exiting `SKILL_NONCONFORMANT` would call a conforming server + // nonconformant — while exiting 0 would report success for entries nobody + // fetched (Copilot). + const md = "---\nname: many\ndescription: Big\n---\n\n# many\n"; + const enc = new TextEncoder(); + const selfDigest = await sha256Digest(enc.encode(md)); + const bodyDigest = await sha256Digest(enc.encode("x")); + const entry: SkillEntry = { + uri: "skill://many/SKILL.md", + frontmatter: { name: "many", description: "Big" }, + resources: Array.from({ length: 600 }, (_, i) => + i === 0 + ? { + uri: "skill://many/SKILL.md", + digest: selfDigest, + size: enc.encode(md).byteLength, + } + : { uri: `skill://many/f${i}.md`, digest: bodyDigest, size: 1 }, + ), + }; + const client = mockClient({ + listSkills: vi.fn().mockResolvedValue({ skills: [entry] }), + readResource: vi.fn(async (uri: string) => ({ + result: { + contents: [{ uri, text: uri.endsWith("/SKILL.md") ? md : "x" }], + }, + })), + }); + const outcome = await runMethod(client, { + method: "skills/list", + verify: true, + }); + if (outcome.kind !== "ndjson") throw new Error("unreachable"); + expect(outcome.exitCode).toBe(EXIT_CODES.SKILL_INCOMPLETE); + expect(EXIT_CODES.SKILL_INCOMPLETE).not.toBe( + EXIT_CODES.SKILL_NONCONFORMANT, + ); + }); + it("--verify works on a single skills/get", async () => { const entry = await cleanEntry(); const client = mockClient({ @@ -227,6 +268,7 @@ describe("summarizeSkillVerification (#2248)", () => { frontmatter: [], files: [{ uri: "skill://demo/SKILL.md", status: "verified" }], ok: true, + outcome: "verified", ...over, }); @@ -247,6 +289,7 @@ describe("summarizeSkillVerification (#2248)", () => { // so collapsing the two counts would misreport the cause. const failed = report({ ok: false, + outcome: "failed", files: [{ uri: "skill://demo/SKILL.md", status: "mismatch" }], }); expect(summarizeSkillVerification([report(), failed])).toBe( @@ -255,7 +298,7 @@ describe("summarizeSkillVerification (#2248)", () => { }); it("reports a failure with no mismatched file", () => { - const failed = report({ ok: false, files: [] }); + const failed = report({ ok: false, outcome: "failed", files: [] }); expect(summarizeSkillVerification([failed])).toBe( "1 of 1 skill failed verification (0 digest/size mismatch across 0 files).", ); diff --git a/clients/cli/__tests__/skills-verify-cli.test.ts b/clients/cli/__tests__/skills-verify-cli.test.ts index 1a4b89d0e1..2becbff781 100644 --- a/clients/cli/__tests__/skills-verify-cli.test.ts +++ b/clients/cli/__tests__/skills-verify-cli.test.ts @@ -135,6 +135,33 @@ describe("consumeMethodOutcome NDJSON summary and exit code (#2248)", () => { }); }); + it("labels the envelope for an INCOMPLETE run, not a nonconformant one", async () => { + // The envelope's `code` follows the exit code, so a caller reading one + // never has to reconcile it against the other — and exit 8 means the + // server broke no MUST. + const streams = captureStreams(); + let thrown: unknown; + try { + await consumeMethodOutcome( + { + kind: "ndjson", + lines: [{ outcome: "incomplete" }], + summary: "not fully checked", + exitCode: EXIT_CODES.SKILL_INCOMPLETE, + }, + {}, + ); + } catch (err) { + thrown = err; + } finally { + streams.restore(); + } + expect(thrown).toMatchObject({ + exitCode: EXIT_CODES.SKILL_INCOMPLETE, + envelope: { code: "skills_incomplete" }, + }); + }); + it("leaves an --app-info NDJSON outcome unchanged", async () => { // No summary, no exit code — the field is additive and the older caller // must behave exactly as before. diff --git a/clients/cli/src/error-handler.ts b/clients/cli/src/error-handler.ts index 7f8b432bd3..d2ab086cfa 100644 --- a/clients/cli/src/error-handler.ts +++ b/clients/cli/src/error-handler.ts @@ -35,6 +35,19 @@ export const EXIT_CODES = { * collapsing them would make `if [ $? -eq 6 ]` ambiguous. */ SKILL_NONCONFORMANT: 7, + /** + * `--verify` could not check the whole catalog: the read bounds stopped the + * walk before it finished (#2248). + * + * Distinct from `SKILL_NONCONFORMANT` because the server has broken no + * **MUST** — SEP-2640 states the 512-entry and 16 MiB limits as SHOULD NOT, + * with hosts free to support more — so exiting 7 would call a conforming + * server nonconformant. It is still non-zero, because reporting success for + * a manifest whose unread entries were never fetched is a false pass. A CI + * job that wants to tolerate oversized catalogs can allow 8 and still fail + * on 7. + */ + SKILL_INCOMPLETE: 8, } as const; /** Machine-readable error envelope written as one JSON line on stderr. */ diff --git a/clients/cli/src/handlers/consume-outcome.ts b/clients/cli/src/handlers/consume-outcome.ts index 6891c30afd..5db0738be9 100644 --- a/clients/cli/src/handlers/consume-outcome.ts +++ b/clients/cli/src/handlers/consume-outcome.ts @@ -1,5 +1,5 @@ import { awaitableError, awaitableLog } from "../utils/awaitable-log.js"; -import { CliExitCodeError } from "../error-handler.js"; +import { CliExitCodeError, EXIT_CODES } from "../error-handler.js"; import { emitResult } from "./emit-result.js"; import type { MethodArgs, MethodOutcome } from "./method-types.js"; @@ -30,7 +30,12 @@ export async function consumeMethodOutcome( // last thing that happens. if (outcome.exitCode) { throw new CliExitCodeError(outcome.exitCode, outcome.summary ?? "", { - code: "skills_nonconformant", + // The envelope's `code` follows the exit code, so a caller reading one + // never has to reconcile it against the other. + code: + outcome.exitCode === EXIT_CODES.SKILL_INCOMPLETE + ? "skills_incomplete" + : "skills_nonconformant", }); } return; diff --git a/clients/cli/src/handlers/run-method.ts b/clients/cli/src/handlers/run-method.ts index 67d4ae0130..f3d883e00e 100644 --- a/clients/cli/src/handlers/run-method.ts +++ b/clients/cli/src/handlers/run-method.ts @@ -15,6 +15,7 @@ import { collectAppInfo } from "./collect-app-info.js"; import { summarizeSkillVerification } from "./skills-verify.js"; import { allSkillsVerified, + anySkillFailed, verifySkills, } from "@inspector/core/mcp/skillsVerification.js"; import type { @@ -334,9 +335,17 @@ export async function runMethod( kind: "ndjson", lines: reports, summary: summarizeSkillVerification(reports), + // Three outcomes, three exit codes: a broken MUST is 7, a walk the + // read bounds cut short is 8, and everything checked and passing is + // 0. Collapsing the middle case into either of the others reports + // something untrue about the server (Copilot). ...(allSkillsVerified(reports) ? {} - : { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }), + : { + exitCode: anySkillFailed(reports) + ? EXIT_CODES.SKILL_NONCONFORMANT + : EXIT_CODES.SKILL_INCOMPLETE, + }), }; } result = { skills }; @@ -371,9 +380,17 @@ export async function runMethod( kind: "ndjson", lines: reports, summary: summarizeSkillVerification(reports), + // Three outcomes, three exit codes: a broken MUST is 7, a walk the + // read bounds cut short is 8, and everything checked and passing is + // 0. Collapsing the middle case into either of the others reports + // something untrue about the server (Copilot). ...(allSkillsVerified(reports) ? {} - : { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }), + : { + exitCode: anySkillFailed(reports) + ? EXIT_CODES.SKILL_NONCONFORMANT + : EXIT_CODES.SKILL_INCOMPLETE, + }), }; } result = envelope; diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index cdbeecff57..c76ec46ea2 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -431,8 +431,15 @@ export function SkillsTab({ {manifest.map((resource, idx) => { + // Matched on normalized identity, like the membership test + // just above — a raw comparison misses a report row recorded + // under an equivalent spelling, while `extraReportFiles` + // suppresses it as already covered, and the verdict renders + // nowhere (Copilot). const fileReport = activeReport?.files.find( - (file) => file.uri === resource.uri, + (file) => + skillUriIdentity(file.uri) === + skillUriIdentity(resource.uri), ); return ( { ).not.toBeInTheDocument(); }); + it("refuses to navigate a child outside the skill root", async () => { + // A server can return a child pointing anywhere; descending into one + // leaves the selected skill's tree, and "Up" only compares against + // `skillRoot`, so the walk could then continue outside it entirely + // (Copilot). The row is still SHOWN — a child outside the skill is itself + // the finding — but it is not a link. + const user = userEvent.setup(); + const STRAY = { + uri: "skill://other-skill/notes.md", + name: "notes.md", + mimeType: "text/markdown", + }; + await openRoot( + user, + directoryReader({ [ROOT]: { resources: [CHILD_FILE, STRAY] } }), + ); + const table = within(screen.getByTestId("skill-directory")); + expect(table.getByText(/outside this skill/)).toBeInTheDocument(); + expect( + table.queryByRole("button", { name: `View ${STRAY.uri}` }), + ).not.toBeInTheDocument(); + // The legitimate sibling is unaffected. + expect( + table.getByRole("button", { name: `View ${CHILD_FILE.uri}` }), + ).toBeInTheDocument(); + }); + + it("refuses a sibling whose path merely starts with the same characters", async () => { + // The reason the check appends a separator: a bare `startsWith(skillRoot)` + // would accept `skill://data-analysis-other/...` as a child of + // `skill://data-analysis`. + const user = userEvent.setup(); + const LOOKALIKE = { + uri: "skill://data-analysis-other/notes.md", + name: "other.md", + mimeType: "text/markdown", + }; + await openRoot( + user, + directoryReader({ [ROOT]: { resources: [LOOKALIKE] } }), + ); + expect( + within(screen.getByTestId("skill-directory")).getByText( + /outside this skill/, + ), + ).toBeInTheDocument(); + }); + + it("accepts a `..` segment that resolves back inside the root", async () => { + // Worth pinning, because the intuition is wrong: `..` cannot escape the + // AUTHORITY. `skill://data-analysis/../x.md` normalizes to + // `skill://data-analysis/x.md`, which really is inside this skill — so + // rejecting it would refuse a legitimate child. Containment is decided on + // the normalized URI precisely so this resolves before it is compared. + const user = userEvent.setup(); + const RESOLVES_INSIDE = { + uri: "skill://data-analysis/nested/../notes.md", + name: "notes.md", + mimeType: "text/markdown", + }; + await openRoot( + user, + directoryReader({ [ROOT]: { resources: [RESOLVES_INSIDE] } }), + ); + const table = within(screen.getByTestId("skill-directory")); + expect(table.queryByText(/outside this skill/)).not.toBeInTheDocument(); + expect( + table.getByRole("button", { name: `View ${RESOLVES_INSIDE.uri}` }), + ).toBeInTheDocument(); + }); + it("says an empty directory is empty", async () => { const user = userEvent.setup(); renderWithMantine( diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 329d7a712d..d237000579 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -33,6 +33,7 @@ import { checkSkillNameCollisions, bytesToText, skillDisplayName, + skillEntryKey, skillFileBytes, skillEntriesMatch, normalizeSkillUri, @@ -844,7 +845,12 @@ export function SkillsScreen({ // and a fresh object every render would loop. const manifestKey = useMemo( () => - `${sessionKey}\n${selected ? JSON.stringify(selected) : (selectedSkillUri ?? "")}`, + // `skillEntryKey`, not `JSON.stringify`: `selected` carries unbounded + // server-controlled frontmatter, and this runs during render — so a + // deeply nested entry threw `RangeError` before the screen could show + // the `frontmatter-unparsable` finding that describes it (Copilot). Same + // helper, and the same guard, the TUI uses. + `${sessionKey}\n${selected ? skillEntryKey(selected) : (selectedSkillUri ?? "")}`, [selected, selectedSkillUri, sessionKey], ); @@ -2181,6 +2187,22 @@ export function SkillsScreen({ {directoryChildren.map((child, index) => { const isDir = child.mimeType === DIRECTORY_MIME_TYPE; + // A server can return a child pointing + // anywhere. Descending into one leaves the + // selected skill's tree, and the "Up" control + // only compares against `skillRoot` — so the + // walk could then continue outside it entirely + // (Copilot). Containment is decided on the + // NORMALIZED URI, like every containment check + // in `core/mcp/skills.ts`, so a `..` segment + // cannot walk out while still matching as a + // prefix. + const childUri = normalizeSkillUri(child.uri); + const inRoot = + childUri !== undefined && + skillRoot !== undefined && + (childUri === skillRoot || + childUri.startsWith(`${skillRoot}/`)); // A directory is not a manifest entry in the // first place — a manifest lists files — so it // is neither listed nor unlisted and gets no @@ -2195,31 +2217,43 @@ export function SkillsScreen({ // collapse into one. - - isDir - ? readDirectory( - child.uri, - manifestKey, - ) - : showResource( - child.uri, - manifestKey, - ) - } - > - {isDir ? `${child.name}/` : child.name} - + {!inRoot ? ( + // Shown, never navigable. The reader + // should see what the server sent, and + // a child outside the skill it was + // asked about is itself the finding. + + {child.name} (outside this skill) + + ) : ( + + isDir + ? readDirectory( + child.uri, + manifestKey, + ) + : showResource( + child.uri, + manifestKey, + ) + } + > + {isDir + ? `${child.name}/` + : child.name} + + )} {child.uri} diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 6f3b9018dc..a67a4ea171 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -5,6 +5,7 @@ import { sha256Digest } from "@inspector/core/mcp/skills.js"; import { AuthRecoveryRequiredError } from "@inspector/core/auth/challenge.js"; import { allSkillsVerified, + anySkillFailed, verifySkills, } from "@inspector/core/mcp/skillsVerification.js"; @@ -14,6 +15,45 @@ import { * pin is the fetching policy and the failure handling, since the checks * themselves are covered in `skills.test.ts`. */ + +/** + * A manifest big enough to be truncated, whose files all VERIFY — so the + * report's outcome isolates "incomplete" instead of also tripping a real + * failure. The entry's own SKILL.md carries frontmatter matching the listing, + * since a body of "x" would be `frontmatter-absent` and so genuinely failed. + */ +async function truncatable(options: { + name: string; + count: number; + body?: string; +}): Promise<{ skill: SkillEntry; client: InspectorClientProtocol }> { + const { name, count, body = "x" } = options; + const md = `---\nname: ${name}\ndescription: Big\n---\n\n# ${name}\n`; + const enc = new TextEncoder(); + const selfDigest = await sha256Digest(enc.encode(md)); + const bodyDigest = await sha256Digest(enc.encode(body)); + const selfUri = `skill://${name}/SKILL.md`; + const skill: SkillEntry = { + uri: selfUri, + frontmatter: { name, description: "Big" }, + resources: Array.from({ length: count }, (_, i) => + i === 0 + ? { uri: selfUri, digest: selfDigest, size: enc.encode(md).byteLength } + : { + uri: `skill://${name}/f${i}.md`, + digest: bodyDigest, + size: enc.encode(body).byteLength, + }, + ), + }; + const client = { + readResource: async (uri: string) => ({ + result: { contents: [{ uri, text: uri === selfUri ? md : body }] }, + }), + } as unknown as InspectorClientProtocol; + return { skill, client }; +} + describe("verifySkills (#2248)", () => { const SKILL_MD = "---\nname: demo\ndescription: A demo\n---\n\n# Demo\n"; const REF = "# Reference\n"; @@ -522,100 +562,50 @@ describe("verifySkills (#2248)", () => { // The 512-entry limit is CHECKED but constrains nothing, so a hostile // server advertising far more had the tool perform that many sequential // reads after the report already knew the manifest was over (Copilot). - const skill: SkillEntry = { - uri: "skill://huge/SKILL.md", - frontmatter: { name: "huge", description: "Too many files" }, - resources: Array.from({ length: 900 }, (_, i) => ({ - uri: i === 0 ? "skill://huge/SKILL.md" : `skill://huge/f${i}.md`, - digest: `sha256:${"a".repeat(64)}`, - size: 1, - })), - }; - const readResource = vi.fn(async (uri: string) => ({ - result: { contents: [{ uri, text: "x" }] }, - })); - const client = { readResource } as unknown as InspectorClientProtocol; + const { skill, client } = await truncatable({ name: "many", count: 900 }); + const readResource = vi.spyOn( + client as unknown as { readResource: (u: string) => unknown }, + "readResource", + ); const [report] = await verifySkills(client, [skill]); expect(readResource).toHaveBeenCalledTimes(512); expect(report.files).toHaveLength(512); - expect(report.ok).toBe(false); - // …and the overage is still REPORTED, so bounding the reads does not - // silence the finding that made them unnecessary. + // Incomplete, not failed: every file it read verified. + expect(report.outcome).toBe("incomplete"); expect(report.conformance).toEqual( expect.arrayContaining([ expect.objectContaining({ code: "resource-limit-exceeded" }), ]), ); }); - it("bounds reads by the total-byte limit, not only the entry count", async () => { // A manifest can sit at exactly 512 entries and declare a gigabyte each, // so bounding the count alone still let a server dictate unbounded - // bandwidth after `size-limit-exceeded` had already been reported - // (Copilot). - const huge = 8 * 1024 * 1024; // two of these cross the 16 MiB bound - const skill: SkillEntry = { - uri: "skill://fat/SKILL.md", - frontmatter: { name: "fat", description: "Enormous files" }, - resources: [ - { - uri: "skill://fat/SKILL.md", - digest: `sha256:${"a".repeat(64)}`, - size: huge, - }, - { - uri: "skill://fat/b.md", - digest: `sha256:${"a".repeat(64)}`, - size: huge, - }, - { - uri: "skill://fat/c.md", - digest: `sha256:${"a".repeat(64)}`, - size: huge, - }, - ], - }; - const readResource = vi.fn(async (uri: string) => ({ - result: { contents: [{ uri, text: "x" }] }, - })); - const client = { readResource } as unknown as InspectorClientProtocol; - const [report] = await verifySkills(client, [skill]); - // Two fit exactly; the third would cross, so it is never requested. - expect(readResource).toHaveBeenCalledTimes(2); - // …and a verification that did not finish must not report success. - expect(report.incomplete).toMatch(/2 of 3 manifest entries/); - expect(report.ok).toBe(false); - expect(report.conformance).toEqual( - expect.arrayContaining([ - expect.objectContaining({ code: "size-limit-exceeded" }), - ]), + // bandwidth after `size-limit-exceeded` had been reported (Copilot). + const { skill, client } = await truncatable({ + name: "fat", + count: 4, + body: "y".repeat(7 * 1024 * 1024), + }); + const readResource = vi.spyOn( + client as unknown as { readResource: (u: string) => unknown }, + "readResource", ); + const [report] = await verifySkills(client, [skill]); + // The SKILL.md is small; 7 MiB bodies then cross the 16 MiB bound. + expect(readResource.mock.calls.length).toBeLessThan(4); + expect(report.outcome).toBe("incomplete"); }); - it("does not report success for a manifest it could not finish reading", async () => { - // The trade this bound must NOT make: entries past the cap are never - // fetched and `resource-limit-exceeded` is only a warning, so a manifest - // whose 513th file is tampered with returned `ok: true` and the CLI said - // the skill verified — a denial of service swapped for a false pass - // (Copilot). - const skill: SkillEntry = { - uri: "skill://many/SKILL.md", - frontmatter: { name: "many", description: "Over the entry limit" }, - resources: Array.from({ length: 600 }, (_, i) => ({ - uri: i === 0 ? "skill://many/SKILL.md" : `skill://many/f${i}.md`, - digest: `sha256:${"a".repeat(64)}`, - size: 1, - })), - }; - const readResource = vi.fn(async (uri: string) => ({ - result: { contents: [{ uri, text: "x" }] }, - })); - const client = { readResource } as unknown as InspectorClientProtocol; + // Entries past the cap are never fetched and `resource-limit-exceeded` is + // only a warning, so a manifest whose 513th file is tampered with returned + // `ok: true` and the CLI said the skill verified (Copilot). + const { skill, client } = await truncatable({ name: "many", count: 600 }); const [report] = await verifySkills(client, [skill]); expect(report.incomplete).toBeDefined(); - expect(report.ok).toBe(false); + expect(report.outcome).toBe("incomplete"); + expect(allSkillsVerified([report])).toBe(false); }); - it("verifies the entry's own file even when the cap excluded it", async () => { // The fallback exists for the frontmatter check, but reading the file and // then skipping the digest its manifest advertised would leave the skill's @@ -649,29 +639,23 @@ describe("verifySkills (#2248)", () => { it("stops on bytes ACTUALLY served, not the sizes the manifest declared", async () => { // The declared budget is server-controlled: advertising `size: 1` and then - // serving megabytes sailed straight through it, defeating the 16 MiB - // safeguard entirely (Copilot). - const big = "x".repeat(6 * 1024 * 1024); - const skill: SkillEntry = { - uri: "skill://liar/SKILL.md", - frontmatter: { name: "liar", description: "Understates its sizes" }, - resources: Array.from({ length: 10 }, (_, i) => ({ - uri: i === 0 ? "skill://liar/SKILL.md" : `skill://liar/f${i}.md`, - digest: `sha256:${"a".repeat(64)}`, - size: 1, // a lie - })), - }; - const readResource = vi.fn(async (uri: string) => ({ - result: { contents: [{ uri, text: big }] }, - })); - const client = { readResource } as unknown as InspectorClientProtocol; + // serving megabytes sailed straight through it (Copilot). The fixture'"'"'s + // digests are honest, so the only thing wrong is the unfinished walk. + const { skill, client } = await truncatable({ + name: "liar", + count: 10, + body: "z".repeat(6 * 1024 * 1024), + }); + // …and now understate every non-entry size, which the old budget trusted. + for (const r of skill.resources as { size?: number }[]) r.size = 1; + const readResource = vi.spyOn( + client as unknown as { readResource: (u: string) => unknown }, + "readResource", + ); const [report] = await verifySkills(client, [skill]); - // Three 6 MiB bodies cross 16 MiB; the walk stops rather than reading ten. - expect(readResource).toHaveBeenCalledTimes(3); + expect(readResource.mock.calls.length).toBeLessThan(10); expect(report.incomplete).toMatch(/actually served/); - expect(report.ok).toBe(false); }); - it("still reports the file that crossed the byte budget", async () => { // The crossing file is verified before the walk stops, so its verdict is // not fetched and then thrown away. @@ -748,7 +732,7 @@ describe("verifySkills (#2248)", () => { expect(report.files).toHaveLength(2); // Nothing was skipped, so nothing is reported as incomplete. expect(report.incomplete).toBeUndefined(); - expect(report.ok).toBe(true); + expect(report.outcome).toBe("verified"); }); it("still reads the entry's own file when the cap would exclude it", async () => { @@ -814,3 +798,54 @@ describe("verifySkills (#2248)", () => { expect(allSkillsVerified([])).toBe(true); }); }); + +describe("verification outcomes (#2248)", () => { + const clean = async (): Promise => { + const md = "---\nname: ok\ndescription: Fine\n---\n\n# ok\n"; + const bytes = new TextEncoder().encode(md); + return { + uri: "skill://ok/SKILL.md", + frontmatter: { name: "ok", description: "Fine" }, + resources: [ + { + uri: "skill://ok/SKILL.md", + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }; + }; + + function serving(text: string) { + return { + readResource: async (uri: string) => ({ + result: { contents: [{ uri, text }] }, + }), + } as unknown as InspectorClientProtocol; + } + + it("separates a broken MUST from an unfinished walk", async () => { + // The distinction the tri-state exists for: both are non-zero outcomes, + // but only one of them says the server did something wrong. + const md = "---\nname: ok\ndescription: Fine\n---\n\n# ok\n"; + const good = await verifySkills(serving(md), [await clean()]); + expect(good[0].outcome).toBe("verified"); + expect(allSkillsVerified(good)).toBe(true); + expect(anySkillFailed(good)).toBe(false); + + const bad = await verifySkills(serving("tampered"), [await clean()]); + expect(bad[0].outcome).toBe("failed"); + expect(allSkillsVerified(bad)).toBe(false); + expect(anySkillFailed(bad)).toBe(true); + }); + + it("does not report an incomplete walk as a failure", async () => { + // `anySkillFailed` selects the CLI exit code, so this is what keeps a + // conforming-but-oversized server off exit 7. + const { skill, client } = await truncatable({ name: "many", count: 600 }); + const reports = await verifySkills(client, [skill]); + expect(reports[0].outcome).toBe("incomplete"); + expect(anySkillFailed(reports)).toBe(false); + expect(allSkillsVerified(reports)).toBe(false); + }); +}); diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index e64f1955e6..62219783ea 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -104,15 +104,33 @@ export interface SkillVerifyReport { * pass has traded down. */ incomplete?: string; + /** + * What the verification concluded. **Three outcomes, not two**, because + * "this skill is wrong" and "this skill could not be fully checked" are + * different answers and collapsing them misreports one of them: + * + * - `verified` — everything was checked and everything passed. + * - `failed` — something the SEP makes a MUST was broken. + * - `incomplete` — nothing checked was wrong, but the read bounds stopped + * the walk before it finished. See {@link incomplete} for the reason. + * + * ⚠️ The `incomplete` case exists because both of the obvious two-state + * answers are wrong. Reporting success would be a **false pass** for a + * manifest whose unread 513th file is tampered with. Reporting failure + * would call a server nonconformant for exceeding limits SEP-2640 states as + * SHOULD NOT — with hosts free to support more — which contradicts this + * module's own rule that a warning never fails a report (Copilot). + */ + outcome: "verified" | "failed" | "incomplete"; /** * False when anything the SEP makes a MUST was broken: an error-severity - * finding, a digest or size mismatch, or a file that could not be read — - * **or when {@link incomplete} is set**, since a verification that did not - * finish cannot report success. + * finding, a digest or size mismatch, or a file that could not be read. * * A `warning` does **not** clear it — a `"dynamic"` manifest is legal, and a * report that failed CI for it would be telling server authors their - * conforming skill is broken. + * conforming skill is broken. Neither does {@link incomplete}: an unfinished + * walk is reported through {@link outcome}, so `ok` keeps its narrow meaning + * of "nothing that was checked is wrong". */ ok: boolean; } @@ -419,16 +437,32 @@ export async function verifySkills( frontmatter, files, ...(incomplete ? { incomplete } : {}), - // An unfinished verification is not a passing one. - ok: !hasError && !fileFailed && incomplete === undefined, + ok: !hasError && !fileFailed, + outcome: + hasError || fileFailed + ? "failed" + : incomplete !== undefined + ? "incomplete" + : "verified", }); } return reports; } -/** True when every skill in the report passed. */ +/** + * True when every skill was checked in full and passed. + * + * Deliberately stricter than `every(r => r.ok)`: a report that could not be + * finished has not verified anything about the part it did not read, so it is + * not "verified" even though nothing it *did* read was wrong. + */ export function allSkillsVerified( reports: readonly SkillVerifyReport[], ): boolean { - return reports.every((report) => report.ok); + return reports.every((report) => report.outcome === "verified"); +} + +/** True when any skill broke something the SEP makes a MUST. */ +export function anySkillFailed(reports: readonly SkillVerifyReport[]): boolean { + return reports.some((report) => report.outcome === "failed"); } From 5a1f0efb5a2049067828fefe5ec2248e178b42a4 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 22:58:19 -0400 Subject: [PATCH 18/21] fix: address Copilot review round 16 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **A hole my own round-13 change opened.** The byte budget added a `break`, and `manifestListsSelf` described the bounded slice rather than the rows the walk reached — so breaking before a later self-entry left the flag true, suppressed the fallback, and skipped the mandatory frontmatter check entirely. It is `selfAttempted` now, set inside the loop, and marked before the read: a row the walk reached but could not read has still been attempted, and its failure belongs to the loop rather than to a second fetch. **A preview could overwrite a verification's own bytes.** The digest verdict on screen was computed from the verification's fetch; replacing only the text let the frontmatter findings describe different bytes, recreating the mixed-fetch verdict that state exists to prevent. `entryTextVerified` marks which read produced it, and a preview no longer wins over a verification. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../SkillsScreen/SkillsScreen.test.tsx | 44 ++++++++++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 42 ++++++++++++----- .../test/core/mcp/skillsVerification.test.ts | 45 +++++++++++++++++++ core/mcp/skillsVerification.ts | 33 +++++++++----- 4 files changed, 141 insertions(+), 23 deletions(-) diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index 2da4c57845..ee4b74d3f3 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -2529,6 +2529,50 @@ describe("SkillsScreen frontmatter cross-check (#2248)", () => { ); }); + it("does not let a later preview overwrite a verification's own bytes", async () => { + // The digest verdict on screen was computed from the verification's fetch; + // replacing only the text would let the frontmatter findings describe + // different bytes, recreating the mixed-fetch verdict this state exists to + // prevent (Copilot). + const user = userEvent.setup(); + let served = skillMdFor({ ...CLEAN_FM, description: "As verified" }); + const onReadSkillFile = vi.fn(async (uri: string) => { + if (uri === "skill://data-analysis/reference.md") { + return { text: REF_TEXT }; + } + return { text: served, mimeType: "text/markdown" }; + }); + renderWithMantine( + , + ); + await user.click(screen.getByText("data-analysis")); + await user.click( + screen.getByRole("button", { + name: "Verify skill://data-analysis/SKILL.md", + }), + ); + await waitFor(() => + expect( + within(screen.getByTestId("skill-frontmatter-issues")).getByText( + /As verified/, + ), + ).toBeInTheDocument(), + ); + + // The server changes, and the reader re-opens the file in the viewer. The + // verification's text must survive, since its digest verdict still shows. + served = skillMdFor({ ...CLEAN_FM, description: "Changed after" }); + await user.click( + screen.getByRole("button", { name: "skill://data-analysis/SKILL.md" }), + ); + await waitFor(() => expect(onReadSkillFile).toHaveBeenCalledTimes(3)); + expect( + within(screen.getByTestId("skill-frontmatter-issues")).getByText( + /As verified/, + ), + ).toBeInTheDocument(); + }); + it("keeps a frontmatter finding when the reader opens another file", async () => { // The check ran off whatever the viewer was showing, so opening a // supporting file made `showingSkillMd` false and silently dropped the diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index d237000579..922514df26 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -105,6 +105,18 @@ interface VerificationState { * has not clicked Verify should still get the check. */ entryText?: string; + /** + * True when {@link entryText} came from a **verification** rather than the + * preview read. + * + * A later preview of the same `SKILL.md` must not overwrite text a + * verification produced: the digest verdict on screen was computed from that + * fetch, and replacing only the text would let the frontmatter findings + * describe different bytes — recreating the mixed-fetch verdict this state + * exists to prevent (Copilot). A verification always wins, since it brings a + * matching digest verdict with it. + */ + entryTextVerified?: boolean; } /** @@ -963,9 +975,12 @@ export function SkillsScreen({ // anything not named here is dropped — which silently discarded the // verified `SKILL.md` text the frontmatter check depends on. ...(entryText !== undefined - ? { entryText } + ? { entryText, entryTextVerified: true } : prev.key === key && prev.entryText !== undefined - ? { entryText: prev.entryText } + ? { + entryText: prev.entryText, + entryTextVerified: prev.entryTextVerified, + } : {}), }; }); @@ -1076,16 +1091,19 @@ export function SkillsScreen({ } catch { return; // neither text nor blob; the viewer reports it } - setVerification((prev) => - prev.key !== null && prev.key !== key - ? prev - : { - ...prev, - key, - files: prev.key === key ? prev.files : {}, - entryText: text, - }, - ); + setVerification((prev) => { + if (prev.key !== null && prev.key !== key) return prev; + const sameKey = prev.key === key; + // Never over a verification's own text — see + // `entryTextVerified`. + if (sameKey && prev.entryTextVerified) return prev; + return { + ...prev, + key, + files: sameKey ? prev.files : {}, + entryText: text, + }; + }); } }) .catch((err: unknown) => { diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index a67a4ea171..3b46d24f30 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -718,6 +718,51 @@ describe("verifySkills (#2248)", () => { ); }); + it("runs the fallback when the byte budget broke the loop before the self row", async () => { + // `manifestListsSelf` described the bounded SLICE, not the rows reached — + // so a break on the byte budget left it true, suppressed the fallback, and + // skipped the mandatory frontmatter check entirely (Copilot). + const md = "---\nname: late\ndescription: Served\n---\n\n# late\n"; + const big = "z".repeat(9 * 1024 * 1024); + const enc = new TextEncoder(); + const skill: SkillEntry = { + uri: "skill://late/SKILL.md", + frontmatter: { name: "late", description: "Listed" }, + resources: [ + // Two oversized files cross the budget before the self row is reached. + { + uri: "skill://late/a.md", + digest: await sha256Digest(enc.encode(big)), + size: 1, + }, + { + uri: "skill://late/b.md", + digest: await sha256Digest(enc.encode(big)), + size: 1, + }, + { + uri: "skill://late/SKILL.md", + digest: await sha256Digest(enc.encode(md)), + size: enc.encode(md).byteLength, + }, + ], + }; + const readResource = vi.fn(async (uri: string) => ({ + result: { + contents: [{ uri, text: uri.endsWith("/SKILL.md") ? md : big }], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + // The self file was still fetched, and its frontmatter still compared. + expect(readResource).toHaveBeenCalledWith( + "skill://late/SKILL.md", + undefined, + ); + expect(report.frontmatter).toHaveLength(1); + expect(report.frontmatter[0].code).toBe("frontmatter-mismatch"); + }); + it("does not truncate a conforming manifest", async () => { // A conforming skill totals at most 16 MiB by definition, so the bound // must never shorten one — otherwise it would trade a hostile-server diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index 62219783ea..efd7520a84 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -294,14 +294,25 @@ export async function verifySkills( // oversized set; this is what stops one that lied. let receivedBytes = 0; const entryIdentity = skillUriIdentity(entry.uri); - // Compared by NORMALIZED identity, like every other URI comparison here — + // ⚠️ Whether the self entry was **actually reached**, not merely whether + // the bounded slice contains it. The byte budget can break the loop before + // a later self-entry, and a flag describing the slice stayed true — which + // suppressed the fallback and skipped the mandatory frontmatter check + // entirely (Copilot). Set inside the loop, so it can only be true of a row + // the walk got to. + // + // Compared by NORMALIZED identity, like every other URI comparison here: // `checkSkillConformance` already accepts a manifest self-entry written in // an equivalent form, so a raw string test would disagree with it and read // the same file a second time. - const manifestListsSelf = manifest.some( - (resource) => skillUriIdentity(resource.uri) === entryIdentity, - ); + let selfAttempted = false; for (const resource of manifest) { + if (skillUriIdentity(resource.uri) === entryIdentity) { + // Marked before the read, not after: a row the walk reached but could + // not read has still been attempted, and its failure is recorded here + // rather than re-attempted by the fallback. + selfAttempted = true; + } let contents: ReadContents | undefined; try { const invocation = await client.readResource(resource.uri, metadata); @@ -357,13 +368,13 @@ export async function verifySkills( // being unenumerable; only integrity is. The same applies to a skill whose // manifest omits its own file. // - // Gated on `manifestListsSelf` rather than on `entryBytes`, so a self-entry - // the loop already tried and FAILED to read is not read a second time — its - // failure is recorded there. Note this is computed over the READ slice, so - // a self-entry pushed past the cap by a bloated manifest still reaches the - // fallback: the frontmatter comparison is mandatory and must not be lost to - // a limit that exists to bound unrelated files. - if (!manifestListsSelf) { + // Gated on `selfAttempted` rather than on `entryBytes`, so a self-entry the + // loop already tried and FAILED to read is not read a second time — its + // failure is recorded there. A self-entry the walk never reached, whether + // because a cap excluded it or because the byte budget broke the loop + // first, still gets the fallback: the frontmatter comparison is mandatory + // and must not be lost to a limit that exists to bound unrelated files. + if (!selfAttempted) { // Recorded as a file result, not swallowed. Because a dynamic skill has // no manifest rows, `files` would otherwise stay empty and its only static // finding is a warning — so an unreadable SKILL.md returned `ok: true` From 9117f633aa5e94482112af51ab759714e44094eb Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 23:12:28 -0400 Subject: [PATCH 19/21] fix: address Copilot review round 17 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four of the five findings share one root cause: the tri-state `outcome` added in round 15 was not propagated to consumers, which kept branching on `ok` — true for an `incomplete` report, since nothing that was checked was wrong. - CLI `summarizeSkillVerification` counts off `outcome`, not `ok`, so a truncated walk no longer prints "no conformance errors" one line before exiting SKILL_INCOMPLETE. A mixed failed/incomplete catalog reports both counts rather than letting the louder verdict hide the quieter. - `verifySkills` sets the truncation reason only when entries were actually left unread. A file crossing the byte budget as the FINAL entry stopped nothing, and "Stopped after 3 of 3" both read as a contradiction and demoted a fully-read skill out of `verified`. - The TUI status line switches on `outcome` through a `Record` over the union, so the INCOMPLETE arm is reachable and a fourth outcome would be a type error rather than a silently missing label. - The `incomplete` doc paragraph said it makes `ok` false. It does not, deliberately — corrected, and it now names `outcome` as the thing a consumer must branch on. - The web sidebar composes in the per-URI `duplicate-name` warning, so two colliding skills are badged in the catalog instead of looking clean until one is selected. The suppressed `selfAttempted` finding is stale — fixed in round 16. The existing TUI INCOMPLETE test was passing for the wrong reason: its fixture understated every size, which is itself a mismatch, so the report was `failed` and only the old `ok`-first branch printed INCOMPLETE. It is rebuilt with honest digests and sizes, so truncation comes from the declared-size prefilter and the report is genuinely incomplete. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../cli/__tests__/run-method-skills.test.ts | 29 +++++++++++++ clients/cli/src/handlers/skills-verify.ts | 25 +++++++++-- clients/tui/__tests__/SkillsTab.test.tsx | 42 +++++++++++++++---- clients/tui/src/components/SkillsTab.tsx | 23 +++++++--- .../SkillsScreen/SkillsScreen.test.tsx | 21 ++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 12 +++++- .../test/core/mcp/skillsVerification.test.ts | 29 +++++++++++++ core/mcp/skillsVerification.ts | 31 +++++++++----- 8 files changed, 183 insertions(+), 29 deletions(-) diff --git a/clients/cli/__tests__/run-method-skills.test.ts b/clients/cli/__tests__/run-method-skills.test.ts index e750f97c02..ae791c4856 100644 --- a/clients/cli/__tests__/run-method-skills.test.ts +++ b/clients/cli/__tests__/run-method-skills.test.ts @@ -303,4 +303,33 @@ describe("summarizeSkillVerification (#2248)", () => { "1 of 1 skill failed verification (0 digest/size mismatch across 0 files).", ); }); + + it("does not claim a truncated walk verified", () => { + // An `incomplete` report keeps `ok: true` — nothing checked was wrong — + // so a summary branching on `ok` printed "no conformance errors" one line + // before the run exited SKILL_INCOMPLETE (Copilot). + const cut = report({ + outcome: "incomplete", + incomplete: "Stopped after 2 of 9 manifest entries.", + }); + expect(summarizeSkillVerification([cut])).toBe( + "Checked 1 skill and 1 file: no conformance errors in what was read." + + " 1 of 1 skill could not be fully checked: the read bounds stopped the walk.", + ); + }); + + it("reports a mixed catalog on both counts", () => { + // The louder verdict must not hide the quieter one: a caller told only + // about the failure would think the rest of the catalog was cleared. + const failed = report({ + ok: false, + outcome: "failed", + files: [{ uri: "skill://demo/SKILL.md", status: "mismatch" }], + }); + const cut = report({ outcome: "incomplete", incomplete: "Stopped." }); + expect(summarizeSkillVerification([report(), failed, cut])).toBe( + "1 of 3 skills failed verification (1 digest/size mismatch across 3 files)." + + " 1 of 3 skills could not be fully checked: the read bounds stopped the walk.", + ); + }); }); diff --git a/clients/cli/src/handlers/skills-verify.ts b/clients/cli/src/handlers/skills-verify.ts index 9ef59cd842..7d1910f117 100644 --- a/clients/cli/src/handlers/skills-verify.ts +++ b/clients/cli/src/handlers/skills-verify.ts @@ -21,7 +21,14 @@ import type { SkillVerifyReport } from "@inspector/core/mcp/skillsVerification.j export function summarizeSkillVerification( reports: readonly SkillVerifyReport[], ): string { - const failed = reports.filter((report) => !report.ok).length; + // ⚠️ Counted off `outcome`, never off `ok`. `ok` means "nothing that was + // checked is wrong", which an `incomplete` report satisfies while the walk + // was cut short — so branching on `ok` printed "no conformance errors" one + // line before exiting SKILL_INCOMPLETE (Copilot). + const failed = reports.filter((report) => report.outcome === "failed").length; + const incomplete = reports.filter( + (report) => report.outcome === "incomplete", + ).length; const files = reports.reduce((sum, report) => sum + report.files.length, 0); const mismatched = reports.reduce( (sum, report) => @@ -30,7 +37,17 @@ export function summarizeSkillVerification( ); const skillWord = reports.length === 1 ? "skill" : "skills"; const fileWord = files === 1 ? "file" : "files"; - return failed === 0 - ? `Verified ${reports.length} ${skillWord} and ${files} ${fileWord}: no conformance errors.` - : `${failed} of ${reports.length} ${skillWord} failed verification (${mismatched} digest/size mismatch across ${files} ${fileWord}).`; + // A catalog can be both: some skills broken, others merely cut short. Say so + // rather than letting the louder verdict hide the quieter one. + const incompleteClause = + incomplete === 0 + ? "" + : ` ${incomplete} of ${reports.length} ${skillWord} could not be fully checked: the read bounds stopped the walk.`; + const headline = + failed === 0 + ? incomplete === 0 + ? `Verified ${reports.length} ${skillWord} and ${files} ${fileWord}: no conformance errors.` + : `Checked ${reports.length} ${skillWord} and ${files} ${fileWord}: no conformance errors in what was read.` + : `${failed} of ${reports.length} ${skillWord} failed verification (${mismatched} digest/size mismatch across ${files} ${fileWord}).`; + return `${headline}${incompleteClause}`; } diff --git a/clients/tui/__tests__/SkillsTab.test.tsx b/clients/tui/__tests__/SkillsTab.test.tsx index 2513d374a5..e59c563cc5 100644 --- a/clients/tui/__tests__/SkillsTab.test.tsx +++ b/clients/tui/__tests__/SkillsTab.test.tsx @@ -757,18 +757,35 @@ describe("SkillsTab (#2248)", () => { // with nothing explaining why (Copilot). // // Truncation is triggered by the BYTE budget rather than the 512-entry one - // so the manifest stays three rows long: a 512-row pane pushes the status + // so the manifest stays four rows long: a 512-row pane pushes the status // line off the frame, which would make this assert the test's viewport // rather than the pane's behaviour. + // + // ⚠️ Every digest and size here is HONEST, so the only thing wrong with + // this skill is the unfinished walk. An earlier version understated the + // sizes, which is itself a size mismatch — the report was `failed` and the + // test passed only because the status line branched on `ok` before + // `incomplete`, the very bug this pins (Copilot). const big = "x".repeat(6 * 1024 * 1024); + const bigDigest = await sha256Digest(textToBytes(big)); + const fatMd = "---\nname: fat\ndescription: Four big files\n---\n\n# F\n"; const fat: SkillEntry = { uri: "skill://fat/SKILL.md", - frontmatter: { name: "fat", description: "Understates its sizes" }, - resources: Array.from({ length: 3 }, (_, i) => ({ - uri: i === 0 ? "skill://fat/SKILL.md" : `skill://fat/f${i}.md`, - digest: CLEAN_DIGEST, - size: 1, - })), + frontmatter: { name: "fat", description: "Four big files" }, + resources: [ + { + uri: "skill://fat/SKILL.md", + digest: await sha256Digest(textToBytes(fatMd)), + size: textToBytes(fatMd).byteLength, + }, + // Three 6 MiB files: the third crosses the 16 MiB budget, so the + // manifest is cut before it and one entry is never fetched. + ...Array.from({ length: 3 }, (_, i) => ({ + uri: `skill://fat/f${i + 1}.md`, + digest: bigDigest, + size: textToBytes(big).byteLength, + })), + ], }; const { lastFrame, stdin } = render( { pageCount={1} inspectorClient={mockClient( vi.fn().mockImplementation(async (uri: string) => ({ - result: { contents: [{ uri, text: big }] }, + result: { + contents: [ + { + uri, + text: uri === "skill://fat/SKILL.md" ? fatMd : big, + }, + ], + }, })), )} width={160} @@ -789,7 +813,7 @@ describe("SkillsTab (#2248)", () => { await tick(); const frame = lastFrame() ?? ""; expect(frame).toContain("Incomplete:"); - expect(frame).toContain("actually served"); + expect(frame).toContain("interoperability limits"); expect(frame).toContain("Verification INCOMPLETE"); expect(frame).not.toContain("Verification FAILED"); }); diff --git a/clients/tui/src/components/SkillsTab.tsx b/clients/tui/src/components/SkillsTab.tsx index c76ec46ea2..c09e24c5f6 100644 --- a/clients/tui/src/components/SkillsTab.tsx +++ b/clients/tui/src/components/SkillsTab.tsx @@ -88,6 +88,18 @@ const FILE_COLOR: Record = { "read-error": "red", }; +/** + * The status line for each of the three verification outcomes. + * + * A `Record` over the union rather than a chain of ternaries, so adding a + * fourth outcome is a type error here instead of a silently missing label. + */ +const VERIFY_STATUS: Record = { + verified: "[Verified — Enter to re-verify]", + incomplete: "[Verification INCOMPLETE — Enter to re-verify]", + failed: "[Verification FAILED — Enter to re-verify]", +}; + /** * The explanation printed under a failed file row. * @@ -529,11 +541,12 @@ export function SkillsTab({ {verifying ? "[Verifying…]" : activeReport - ? activeReport.ok - ? "[Verified — Enter to re-verify]" - : activeReport.incomplete - ? "[Verification INCOMPLETE — Enter to re-verify]" - : "[Verification FAILED — Enter to re-verify]" + ? // ⚠️ Switched on `outcome`, not on `ok`. `ok` stays + // true for an `incomplete` report — nothing checked + // was wrong — so an `ok`-first branch printed + // "Verified" for a walk the read bounds cut short and + // the INCOMPLETE arm was unreachable (Copilot). + VERIFY_STATUS[activeReport.outcome] : "[Enter to verify digests and frontmatter]"} diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index ee4b74d3f3..4b6e524c6f 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -2401,6 +2401,27 @@ describe("SkillsScreen name collisions (#2248)", () => { expect(screen.getByTestId("skill-name-collision")).toBeInTheDocument(); }); + it("badges the collision in the sidebar, before either is selected", async () => { + // The collision is a property of the LISTING, so `checkSkillConformance` + // on one entry cannot see it — and a sidebar computed from that alone + // showed both colliding rows as clean until one was clicked, which is + // exactly when a reader most needs to be told two rows share a name + // (Copilot). Nothing is selected here on purpose. + renderWithMantine(); + const rows = [ACME, GLOBEX].map((skill) => + screen.getByText(skill.uri).closest(".mantine-NavLink-root"), + ); + for (const row of rows) { + expect(row).not.toBeNull(); + // One finding, badged — a warning, so yellow rather than the red that + // would call a conforming server broken. + const badge = row?.querySelector(".mantine-Badge-root"); + expect(badge).not.toBeNull(); + expect(badge).toHaveTextContent("1"); + expect(badge?.getAttribute("style") ?? "").toContain("yellow"); + } + }); + it("says nothing when the names are distinct", async () => { const user = userEvent.setup(); renderWithMantine(); diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 922514df26..0d697044f0 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -1624,7 +1624,17 @@ export function SkillsScreen({ No skills listed ) : ( filtered.map((skill) => { - const skillIssues = checkSkillConformance(skill); + // ⚠️ The collision is a property of the LISTING, not of the + // entry, so `checkSkillConformance` alone cannot see it — and + // a sidebar computed from that alone showed both colliding + // skills as clean until one was selected, which is exactly + // when a reader most needs to be told two rows are the same + // name (Copilot). Same composition as `conformance` above. + const collision = collisions.get(skillUriIdentity(skill.uri)); + const skillIssues = [ + ...checkSkillConformance(skill), + ...(collision ? [collision] : []), + ]; const errors = skillIssues.filter( (i) => i.severity === "error", ).length; diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 3b46d24f30..0bbdc8226e 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -656,6 +656,35 @@ describe("verifySkills (#2248)", () => { expect(readResource.mock.calls.length).toBeLessThan(10); expect(report.incomplete).toMatch(/actually served/); }); + + it("is not incomplete when the budget is crossed by the LAST entry", async () => { + // Crossing the line on the final row stopped nothing: every manifest entry + // was fetched and checked. Reporting "Stopped after 4 of 4" there both + // reads as a contradiction and demotes a fully-read skill out of + // `verified` (Copilot). + // + // The sizes are understated for the same reason as the test above — with + // honest ones the *declared* prefilter stops first and the received-bytes + // guard is never reached at all. That understatement is itself a size + // mismatch, so this fixture is `failed`; what it pins is that the walk is + // not ALSO reported as cut short. + const { skill, client } = await truncatable({ + name: "edge", + count: 4, + body: "z".repeat(6 * 1024 * 1024), + }); + for (const r of skill.resources as { size?: number }[]) r.size = 1; + const readResource = vi.spyOn( + client as unknown as { readResource: (u: string) => unknown }, + "readResource", + ); + const [report] = await verifySkills(client, [skill]); + // All four read — the fourth is what crosses the 16 MiB budget. + expect(readResource.mock.calls.length).toBe(4); + expect(report.files).toHaveLength(4); + expect(report.incomplete).toBeUndefined(); + expect(report.outcome).toBe("failed"); + }); it("still reports the file that crossed the byte budget", async () => { // The crossing file is verified before the walk stops, so its verdict is // not fetched and then thrown away. diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index efd7520a84..fa65057ae5 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -94,14 +94,17 @@ export interface SkillVerifyReport { /** * Why the manifest could not be checked in full, or `undefined` when it was. * - * Set when the read bounds truncated the manifest. It is reported separately - * from `ok` being false so a consumer can tell "this skill is wrong" from - * "this skill was not fully checked" — but it does make `ok` false, because - * the alternative is worse: entries past the cap are never fetched, and - * `resource-limit-exceeded` is only a WARNING, so a manifest whose 513th - * file was tampered with reported `ok: true` and the CLI said the skill - * verified (Copilot). A bound that turns a denial of service into a false - * pass has traded down. + * Set when the read bounds truncated the manifest, and **only** when entries + * were actually left unread — a file that crosses the byte budget as the + * last entry checked nothing short, so it is not incomplete. + * + * ⚠️ It does **not** make {@link ok} false. `ok` keeps the narrow meaning of + * "nothing that was checked is wrong", and a truncated walk checked nothing + * that was wrong. The signal a consumer must branch on is {@link outcome} + * being `"incomplete"`, never `!ok` — a manifest whose unread 513th file was + * tampered with reports `ok: true`, and a consumer that prints "verified" on + * `ok` alone turns a denial of service into a false pass (Copilot). Both the + * CLI summary and the TUI status line had exactly that bug. */ incomplete?: string; /** @@ -306,7 +309,7 @@ export async function verifySkills( // an equivalent form, so a raw string test would disagree with it and read // the same file a second time. let selfAttempted = false; - for (const resource of manifest) { + for (const [index, resource] of manifest.entries()) { if (skillUriIdentity(resource.uri) === entryIdentity) { // Marked before the read, not after: a row the walk reached but could // not read has still been attempted, and its failure is recorded here @@ -357,7 +360,15 @@ export async function verifySkills( // prevent and is not something this API exposes. receivedBytes += bytes.byteLength; if (receivedBytes > SKILL_MAX_TOTAL_BYTES) { - incomplete = `Stopped after ${files.length} of ${declared.length} manifest entries: the files actually served exceed the ${SKILL_MAX_TOTAL_BYTES}-byte interoperability limit, whatever sizes the manifest declared.`; + // ⚠️ Only *incomplete* when the budget actually cost a read. Crossing + // the line on the final entry stopped nothing — every manifest row was + // fetched and checked — and reporting "Stopped after 3 of 3" there + // both reads as a contradiction and demotes a fully-checked skill out + // of `verified` (Copilot). The prefilter's own reason, if it dropped + // entries before the walk, is already set and is not overwritten. + if (index < manifest.length - 1) { + incomplete = `Stopped after ${index + 1} of ${declared.length} manifest entries: the files actually served exceed the ${SKILL_MAX_TOTAL_BYTES}-byte interoperability limit, whatever sizes the manifest declared.`; + } break; } } From a3f36ce86388772fabe6c24643c8e9d9e21503f2 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 23:33:30 -0400 Subject: [PATCH 20/21] fix: address Copilot review round 18 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings, all in paths hardened by earlier rounds. - The 16 MiB read budget was charged only after a content block had matched by URI *and* decoded. A server could answer every manifest row with one enormous block labelled a different URI (so `contentsFor` found nothing) or one enormous invalid-base64 blob (so `skillFileBytes` threw), bank zero against the cap, and have the walk issue up to 512 more. `responseBytes` now charges the raw response before selection or decoding; the exact decoded length is substituted when there is one, so the common path is still measured precisely. - The Directory section checked that a child was inside the skill root but not that it was a child of the directory actually being read. A grandchild, or the directory echoing itself back, passed containment and was rendered as navigable. Non-direct entries are now shown and labelled rather than linked — `resources/directory/read` answers with direct children, so an entry that is not one is itself the finding. - Containment was decided on the normalized URI while navigation sent and stored the raw one. For `skill://r/a/../templates` the first Up produced `skill://r/a/..` and a second walked into `skill://r/a`, a directory nothing had validated. Directory descent passes the normalized URI, and the Up arithmetic moved into `parentOfSkillUri`, documented as valid only on a normalized URI. The suppressed comment (two stderr lines on a failing `--verify`) is declined again: `--strict` produces the same shape in `emit-result.ts`, and clients/cli/README.md already documents it as such. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- .../SkillsScreen/SkillsScreen.test.tsx | 84 +++++++++++++ .../screens/SkillsScreen/SkillsScreen.tsx | 60 +++++++-- .../test/core/mcp/skillsVerification.test.ts | 34 +++++ core/mcp/skillsVerification.ts | 117 +++++++++++++----- 4 files changed, 253 insertions(+), 42 deletions(-) diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx index 4b6e524c6f..a4855e5596 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.test.tsx @@ -2134,6 +2134,90 @@ describe("SkillsScreen directory browsing (#2248)", () => { ).toBeInTheDocument(); }); + it("refuses a child that is not a DIRECT child of the directory read", async () => { + // `resources/directory/read` answers with the directory's direct children. + // A grandchild, or the directory itself echoed back, is still inside the + // root — so it passed the containment check and was rendered as though the + // server had said it lives here (Copilot). The row is shown, because that + // is the finding, but it is not a link. + const user = userEvent.setup(); + const GRANDCHILD = { + uri: "skill://data-analysis/templates/invoice.md", + name: "invoice.md", + mimeType: "text/markdown", + }; + await openRoot( + user, + directoryReader({ + [ROOT]: { resources: [CHILD_FILE, GRANDCHILD, CHILD_DIR] }, + // The second page lists the directory ITSELF alongside its child. + [CHILD_DIR.uri]: { resources: [CHILD_DIR, NESTED] }, + }), + ); + const table = () => within(screen.getByTestId("skill-directory")); + expect(table().getByText(/not a direct child/)).toBeInTheDocument(); + // Named for what is wrong with it — it is inside the skill, so calling it + // "outside this skill" would send the reader after the wrong defect. + expect(table().queryByText(/outside this skill/)).not.toBeInTheDocument(); + expect( + table().queryByRole("button", { name: `View ${GRANDCHILD.uri}` }), + ).not.toBeInTheDocument(); + // The real direct children are unaffected. + expect( + table().getByRole("button", { name: `View ${CHILD_FILE.uri}` }), + ).toBeInTheDocument(); + + // …and the same holds one level down, where the offender is the directory + // being read. Left navigable it would be a link back to the page you are + // already on. + await user.click( + screen.getByRole("button", { name: `Open directory ${CHILD_DIR.uri}` }), + ); + await waitFor(() => + expect(table().getByText(NESTED.uri)).toBeInTheDocument(), + ); + expect(table().getByText(/not a direct child/)).toBeInTheDocument(); + expect( + table().queryByRole("button", { + name: `Open directory ${CHILD_DIR.uri}`, + }), + ).not.toBeInTheDocument(); + }); + + it("navigates on the normalized URI, so Up cannot walk into a `..` segment", async () => { + // Containment was decided on the normalized URI while navigation sent and + // stored the raw one, so for `skill://root/a/../templates` the first Up + // produced `skill://root/a/..` and a second walked into `skill://root/a` + // — a directory the check never validated (Copilot). + const user = userEvent.setup(); + const DOTTED_DIR = { + uri: "skill://data-analysis/nested/../templates", + name: "templates", + mimeType: "inode/directory", + }; + const reader = directoryReader({ + [ROOT]: { resources: [DOTTED_DIR] }, + // Keyed by the NORMALIZED URI: that is what must be sent. + "skill://data-analysis/templates": { resources: [NESTED] }, + }); + await openRoot(user, reader); + await user.click( + screen.getByRole("button", { name: `Open directory ${DOTTED_DIR.uri}` }), + ); + await waitFor(() => + expect(reader).toHaveBeenCalledWith( + "skill://data-analysis/templates", + undefined, + ), + ); + await user.click(screen.getByRole("button", { name: "Up" })); + // One hop, straight back to the root — not to `skill://data-analysis/nested`. + await waitFor(() => expect(reader).toHaveBeenCalledWith(ROOT, undefined)); + expect(reader.mock.calls.map((call) => call[0])).not.toContain( + "skill://data-analysis/nested", + ); + }); + it("refuses a sibling whose path merely starts with the same characters", async () => { // The reason the check appends a separator: a bare `startsWith(skillRoot)` // would accept `skill://data-analysis-other/...` as a child of diff --git a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx index 0d697044f0..ac0e2173ff 100644 --- a/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx +++ b/clients/web/src/components/screens/SkillsScreen/SkillsScreen.tsx @@ -698,6 +698,18 @@ function resourceFileName(uri: string): string { } /** `sha256:abcd…wxyz`, so a long digest stays readable in a table cell. */ +/** + * The parent directory of a skill URI, by path arithmetic. + * + * Only ever applied to a NORMALIZED URI — one with no `.`/`..` segments left — + * so the last `/` really is the boundary between a directory and its child. On + * a raw URI the same slice is meaningless: the parent of + * `skill://r/a/../templates` is `skill://r`, not `skill://r/a/..`. + */ +function parentOfSkillUri(uri: string): string { + return uri.slice(0, uri.lastIndexOf("/")); +} + function shortDigest(digest: string | undefined): string { if (!digest) return "—"; return digest.length <= 24 ? digest : `${digest.slice(0, 16)}…`; @@ -1351,6 +1363,9 @@ export function SkillsScreen({ // same guard every other async slot on this screen uses. const directoryCurrent = directory.key === manifestKey; const directoryUri = directoryCurrent ? directory.uri : undefined; + // The directory these children were read FROM — the root until the reader + // descends. Every child row is judged against this, so the two cannot drift. + const readingUri = directoryUri ?? skillRoot; const directoryChildren = directoryCurrent ? directory.children : undefined; const directoryError = directoryCurrent ? directory.message : undefined; const directoryLoading = directoryCurrent && directory.loading === true; @@ -2154,10 +2169,7 @@ export function SkillsScreen({ readDirectory( - directoryUri.slice( - 0, - directoryUri.lastIndexOf("/"), - ), + parentOfSkillUri(directoryUri), manifestKey, ) } @@ -2231,6 +2243,19 @@ export function SkillsScreen({ skillRoot !== undefined && (childUri === skillRoot || childUri.startsWith(`${skillRoot}/`)); + // `resources/directory/read` answers with the + // directory's DIRECT children. A grandchild, a + // sibling's file, or the directory itself is + // inside the root and so passed `inRoot`, and + // was then rendered as though the server had + // said it lives here (Copilot). This screen + // exists to report what a server sent, so an + // entry that is not a direct child is shown + // and named rather than quietly navigable. + const directChild = + childUri !== undefined && + readingUri !== undefined && + parentOfSkillUri(childUri) === readingUri; // A directory is not a manifest entry in the // first place — a manifest lists files — so it // is neither listed nor unlisted and gets no @@ -2245,13 +2270,18 @@ export function SkillsScreen({ // collapse into one. - {!inRoot ? ( + {!inRoot || !directChild ? ( // Shown, never navigable. The reader // should see what the server sent, and // a child outside the skill it was - // asked about is itself the finding. + // asked about — or one that is not a + // child of this directory at all — is + // itself the finding. - {child.name} (outside this skill) + {child.name} + {!inRoot + ? " (outside this skill)" + : " (not a direct child)"} ) : ( isDir - ? readDirectory( - child.uri, + ? // ⚠️ The NORMALIZED URI, which + // is what `directChild` and + // `inRoot` were decided on. + // Storing the raw one instead + // meant "Up" did its path + // arithmetic on an identity + // nothing had validated — from + // `skill://r/a/../templates` + // the first Up produced + // `skill://r/a/..` and the + // second walked into + // `skill://r/a` (Copilot). + readDirectory( + childUri, manifestKey, ) : showResource( diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 0bbdc8226e..0f874339b4 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -657,6 +657,40 @@ describe("verifySkills (#2248)", () => { expect(report.incomplete).toMatch(/actually served/); }); + it("charges the budget for a response whose block never matched", async () => { + // The budget was charged only after a matching block had been decoded, so + // a server could answer every row with one enormous block labelled some + // OTHER URI: `contentsFor` found nothing, zero was banked, and the walk + // went on to issue up to 512 more of them (Copilot). The bytes crossed the + // wire either way, so the transfer is what pays. + const junk = "z".repeat(6 * 1024 * 1024); + const { skill } = await truncatable({ name: "junk", count: 20 }); + const readResource = vi.fn(async () => ({ + // Labelled a URI nobody asked for — the whole point. + result: { contents: [{ uri: "skill://elsewhere/huge.md", text: junk }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + // Three reads of 6 MiB crosses 16 MiB; without the fix all 20 were issued. + expect(readResource.mock.calls.length).toBe(3); + expect(report.incomplete).toMatch(/actually served/); + expect(report.outcome).toBe("failed"); + }); + + it("charges the budget for a response that could not be decoded", async () => { + // The second free route: an enormous `blob` that is not valid base64, so + // `skillFileBytes` throws before anything is counted. + const junk = "!".repeat(6 * 1024 * 1024); + const { skill } = await truncatable({ name: "junk", count: 20 }); + const readResource = vi.fn(async (uri: string) => ({ + result: { contents: [{ uri, blob: junk }] }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(readResource.mock.calls.length).toBe(3); + expect(report.incomplete).toMatch(/actually served/); + }); + it("is not incomplete when the budget is crossed by the LAST entry", async () => { // Crossing the line on the final row stopped nothing: every manifest entry // was fetched and checked. Reporting "Stopped after 4 of 4" there both diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index fa65057ae5..c4ee2da9d7 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -219,6 +219,37 @@ function contentsFor(result: unknown, uri: string): ReadContents | undefined { return undefined; } +/** + * What a `resources/read` response cost to receive, charged against the byte + * budget regardless of whether any of it is usable. + * + * ⚠️ Deliberately measured on the RAW result rather than on decoded content. + * The budget exists to stop a server from making this walk transfer unbounded + * data, and a server that wants to do that has two free routes if only decoded + * bytes are counted: label an enormous block with a URI that was not asked for, + * or send an enormous blob that is not valid base64. Both leave the decode + * paths empty-handed while the bytes have already crossed the wire. + * + * The figure is an **approximation, and deliberately never an undercount by + * more than a small factor**: `text` is charged in UTF-16 code units (UTF-8 is + * between 1× and 3× that for the same string) and `blob` in base64 characters + * (roughly 4/3 of the bytes it decodes to, so an overcharge). Exactness is not + * the point — this is a safety limit, not an accounting figure, and the caller + * substitutes the exact decoded length whenever it has one. + */ +function responseBytes(result: unknown): number { + const contents = (result as { contents?: unknown })?.contents; + if (!Array.isArray(contents)) return 0; + let total = 0; + for (const block of contents) { + if (typeof block !== "object" || block === null) continue; + const { text, blob } = block as { text?: unknown; blob?: unknown }; + if (typeof text === "string") total += text.length; + if (typeof blob === "string") total += blob.length; + } + return total; +} + /** * Verify every skill in `entries` against the connected server. * @@ -316,49 +347,69 @@ export async function verifySkills( // rather than re-attempted by the fallback. selfAttempted = true; } - let contents: ReadContents | undefined; + // Bytes attributed to THIS response, charged whether or not any of them + // turn out to be usable — see `responseBytes`. + let charged = 0; try { const invocation = await client.readResource(resource.uri, metadata); - contents = contentsFor(invocation.result, resource.uri); + // ⚠️ Charged from the RAW response, BEFORE the block is selected and + // before it is decoded. Charging only the decoded bytes let a server + // spend the budget for free: return one enormous block labelled some + // other URI (so `contentsFor` finds nothing) or one enormous invalid + // base64 blob (so `skillFileBytes` throws), and the walk banked zero + // against the cap and went on to issue up to 512 more of them + // (Copilot). The safeguard has to be paid for by the transfer, not by + // the parse. + charged = responseBytes(invocation.result); + const contents = contentsFor(invocation.result, resource.uri); + if (!contents) { + files.push({ + uri: resource.uri, + status: "read-error", + reason: + "resources/read returned no content block for this URI, so there are no bytes that can be checked against its digest.", + }); + } else { + let bytes: Uint8Array | undefined; + try { + bytes = skillFileBytes(contents); + } catch (err) { + files.push({ + uri: resource.uri, + status: "read-error", + reason: reasonOf(err), + }); + } + if (bytes) { + // The decoded length is exact where `responseBytes` is only an + // estimate, so the larger of the two is charged: never less than + // what this file actually cost, and never less than what the rest + // of the response was estimated to cost. + charged = Math.max(charged, bytes.byteLength); + if (skillUriIdentity(resource.uri) === entryIdentity) + entryBytes = bytes; + const verification = await verifySkillResource(resource, bytes); + files.push({ uri: resource.uri, ...verification }); + } + } } catch (err) { if (err instanceof AuthRecoveryRequiredError) throw err; + // Nothing to charge: a rejected read never handed us a payload to + // measure. Whatever the transport moved before failing is invisible + // at this layer. files.push({ uri: resource.uri, status: "read-error", reason: reasonOf(err), }); - continue; - } - if (!contents) { - files.push({ - uri: resource.uri, - status: "read-error", - reason: - "resources/read returned no content block for this URI, so there are no bytes that can be checked against its digest.", - }); - continue; - } - let bytes: Uint8Array; - try { - bytes = skillFileBytes(contents); - } catch (err) { - files.push({ - uri: resource.uri, - status: "read-error", - reason: reasonOf(err), - }); - continue; } - if (skillUriIdentity(resource.uri) === entryIdentity) entryBytes = bytes; - const verification = await verifySkillResource(resource, bytes); - files.push({ uri: resource.uri, ...verification }); - // Counted AFTER verifying this file, so the one that crosses the line is - // still reported rather than fetched and discarded. The next read is what - // stops. ⚠️ This bounds the total across responses, not the size of any - // single one: a first response larger than the cap is already in memory - // by the time it can be measured, which would need a streaming read to - // prevent and is not something this API exposes. - receivedBytes += bytes.byteLength; + // Counted AFTER this row is recorded, so the response that crosses the + // line is still reported rather than fetched and discarded. The next + // read is what stops. ⚠️ This bounds the total across responses, not the + // size of any single one: a first response larger than the cap is + // already in memory by the time it can be measured, which would need a + // streaming read to prevent and is not something this API exposes. + receivedBytes += charged; if (receivedBytes > SKILL_MAX_TOTAL_BYTES) { // ⚠️ Only *incomplete* when the budget actually cost a read. Crossing // the line on the final entry stopped nothing — every manifest row was From 62fd7d1bd49e14a16c6338f0da33185ad2c4bdc9 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Mon, 7 Sep 2026 23:55:53 -0400 Subject: [PATCH 21/21] fix: address Copilot review round 19 on #2248 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings, all real. - No catalog-level bound. The per-skill caps bound what ONE entry can cost; nothing bounded how many entries there are, and SEP-2640 puts no ceiling on a catalog — every entry costs at least one resources/read, so a large listing made `--verify` run indefinitely and transfer unboundedly. Adds SKILL_MAX_CATALOG_SKILLS / SKILL_MAX_CATALOG_BYTES. Entries past the budget are still reported, with their static findings and an `incomplete` reason, rather than dropped or failed: they were not checked, which is neither a pass nor a verdict against the server. - `responseBytes` charged `text.length`, which counts UTF-16 code units. That undercharges non-ASCII by up to 3x, so a decoy block of emoji kept the counter under 16 MiB while the wire carried far more. Adds `utf8Length`, an allocation-free UTF-8 byte count — deliberately not TextEncoder, which would copy a payload the server chose the size of. - `checkSkillNameCollisions` was O(N^2) in both work and output for a group of N: every entry filtered all N URIs and embedded the other N-1. Duplicate names are legal and a server controls N. Now names a bounded sample and counts the rest. - `--verify` help documented exit 7 but not exit 8. The three-tier bounding (per skill, per skill on the wire, per run) is now a table in clients/cli/README.md, since the run bound is this tool's limit rather than the spec's and should not read as a conformance rule. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall --- clients/cli/README.md | 15 +++ clients/cli/src/cli.ts | 2 +- clients/web/src/test/core/mcp/skills.test.ts | 21 ++++ .../test/core/mcp/skillsVerification.test.ts | 101 +++++++++++++++++- core/mcp/skills.ts | 57 +++++++++- core/mcp/skillsVerification.ts | 83 ++++++++++++-- 6 files changed, 264 insertions(+), 15 deletions(-) diff --git a/clients/cli/README.md b/clients/cli/README.md index aeacebd6fb..4e2336fe95 100644 --- a/clients/cli/README.md +++ b/clients/cli/README.md @@ -381,6 +381,21 @@ wrong" and "this skill could not be fully checked" are different answers: | `failed` | `7` | Something SEP-2640 makes a MUST was broken — an error-severity finding, a digest or size mismatch, or an unreadable manifest file. | | `incomplete` | `8` | Nothing checked was wrong, but the read bounds stopped the walk before it finished. See `incomplete` in the report for the reason. | +**The run is bounded, and says when a bound bit.** Three limits, all reported as +`incomplete` (`8`) rather than as a pass or a failure, because an entry that was +not read has not been cleared of anything: + +| Bound | Limit | Why | +| --- | --- | --- | +| Per skill | 512 manifest entries / 16 MiB | SEP-2640's own interoperability limits. | +| Per skill, on the wire | 16 MiB actually served | The declared sizes are server-controlled; this one cannot be lied past. | +| Per run | 256 skills / 64 MiB | SEP-2640 bounds a skill and deliberately does not bound a *catalog*. Every entry costs at least one `resources/read`, so without this a large listing — hostile or merely big — is unbounded work against the tool inspecting it. | + +The run bound is this tool's, not the spec's. A skill past it is still reported, +with its static conformance findings and an `incomplete` reason saying nothing +about its files was checked; verify it on its own with `--method skills/get +--uri ` to get a verdict for it. + A **warning** never produces `7`. That distinction matters most for `resources: "dynamic"`, which is a *conforming* wire form for generated content: it means integrity cannot be verified, which is worth reporting, but failing CI for it would tell server diff --git a/clients/cli/src/cli.ts b/clients/cli/src/cli.ts index fe19ee84ad..28ccd0dc76 100644 --- a/clients/cli/src/cli.ts +++ b/clients/cli/src/cli.ts @@ -752,7 +752,7 @@ async function parseArgs(argv?: string[]): Promise { ) .option( "--verify", - "Run the SEP-2640 conformance and digest checks over the skills returned, emit one JSON report per skill on stdout, and exit 7 if any fails. Use with --method skills/list or --method skills/get.", + "Run the SEP-2640 conformance and digest checks over the skills returned, emit one JSON report per skill on stdout, and exit 7 if any fails or 8 if any could not be fully checked within the read bounds. Use with --method skills/list or --method skills/get.", ) .option( "--connect-timeout ", diff --git a/clients/web/src/test/core/mcp/skills.test.ts b/clients/web/src/test/core/mcp/skills.test.ts index ebcf1a1111..04a250b5c3 100644 --- a/clients/web/src/test/core/mcp/skills.test.ts +++ b/clients/web/src/test/core/mcp/skills.test.ts @@ -1154,6 +1154,27 @@ describe("checkSkillNameCollisions (#2248)", () => { expect(first?.message).toContain("skill://c/r/SKILL.md"); }); + it("bounds a large collision group instead of transcribing it", () => { + // Duplicate names are legal and SEP-2640 puts no ceiling on a catalog, so + // naming every other member made both the work and the generated text + // O(N²) — a server controls N, which turns a legal listing into a denial + // of service against the tool sent to inspect it (Copilot). + const N = 500; + const collisions = checkSkillNameCollisions( + Array.from({ length: N }, (_, i) => at(`skill://s${i}/r/SKILL.md`, "r")), + ); + expect(collisions.size).toBe(N); + const message = collisions.get("skill://s0/r/SKILL.md")?.message ?? ""; + // Three named, the rest counted — enough to see what the collision IS and + // where to look, without a transcript of the catalog. + expect(message).toMatch(/and 496 more/); + expect(message).toContain("499 other skills in this listing also declare"); + // The bound is on the message, so its length cannot grow with the catalog. + expect(message.length).toBeLessThan(400); + // Still never names itself. + expect(message).not.toContain("skill://s0/r/SKILL.md"); + }); + it("does not report the SAME skill listed twice as a collision", () => { // A repeated entry is a different defect from two skills sharing a name, // and calling it this one would be a wrong diagnosis rather than a missing diff --git a/clients/web/src/test/core/mcp/skillsVerification.test.ts b/clients/web/src/test/core/mcp/skillsVerification.test.ts index 0f874339b4..e6ff5b0775 100644 --- a/clients/web/src/test/core/mcp/skillsVerification.test.ts +++ b/clients/web/src/test/core/mcp/skillsVerification.test.ts @@ -1,11 +1,15 @@ import { describe, it, expect, vi } from "vitest"; import type { InspectorClientProtocol } from "@inspector/core/mcp/inspectorClientProtocol.js"; import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js"; -import { sha256Digest } from "@inspector/core/mcp/skills.js"; +import { + SKILL_MAX_CATALOG_SKILLS, + sha256Digest, +} from "@inspector/core/mcp/skills.js"; import { AuthRecoveryRequiredError } from "@inspector/core/auth/challenge.js"; import { allSkillsVerified, anySkillFailed, + utf8Length, verifySkills, } from "@inspector/core/mcp/skillsVerification.js"; @@ -691,6 +695,101 @@ describe("verifySkills (#2248)", () => { expect(report.incomplete).toMatch(/actually served/); }); + it("charges non-ASCII text in UTF-8 bytes, not UTF-16 units", async () => { + // `text.length` undercharged every non-ASCII payload by up to 3×, so a + // decoy block of emoji kept the counter under 16 MiB while the wire + // carried twice that, and the walk read on (Copilot). Each block below is + // 3 MiB of UTF-16 units and 12 MiB of UTF-8 bytes, so two cross the limit + // under correct accounting and six would be needed under the old one. + const emoji = "🙂".repeat(1.5 * 1024 * 1024); // 2 units each, 4 bytes each + const { skill } = await truncatable({ name: "emoji", count: 20 }); + const readResource = vi.fn(async () => ({ + result: { + contents: [{ uri: "skill://elsewhere/decoy.md", text: emoji }], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const [report] = await verifySkills(client, [skill]); + expect(readResource.mock.calls.length).toBe(3); + expect(report.incomplete).toMatch(/actually served/); + }); + + it("counts UTF-8 length exactly as TextEncoder does", async () => { + // The counter is hand-rolled to avoid allocating a copy of a payload a + // hostile server sized, so it is pinned against the reference encoder — + // including the surrogate cases that are the only reason it is not a + // one-liner. + const cases = [ + "", + "plain ascii", + "café", // 2-byte + "日本語", // 3-byte + "🙂👍", // surrogate pairs, 4-byte + "a🙂b", + "\ud83d", // lone HIGH surrogate — U+FFFD, 3 bytes + "\udc4d", // lone LOW surrogate + "\ud83d\ud83d", // two highs in a row: neither pairs + "end\ud83d", // unpaired high at the very end + ]; + const encoder = new TextEncoder(); + for (const value of cases) { + expect(utf8Length(value)).toBe(encoder.encode(value).byteLength); + } + }); + + it("stops reading once the run's catalog budget is spent", async () => { + // The per-skill caps bound what ONE entry costs; nothing bounded how many + // entries there are, and SEP-2640 puts no ceiling on a catalog — so a + // listing of a hundred thousand conforming skills made `--verify` run + // indefinitely (Copilot). + // Each skill carries its OWN SKILL.md, whose frontmatter matches its + // listing entry — otherwise every report is `failed` on a frontmatter + // mismatch and the budget is not what the test is measuring. + const enc = new TextEncoder(); + const mdFor = (i: number) => + `---\nname: s${i}\ndescription: A demo\n---\n\n# s${i}\n`; + const many = await Promise.all( + Array.from( + { length: SKILL_MAX_CATALOG_SKILLS + 5 }, + async (_, i): Promise => { + const bytes = enc.encode(mdFor(i)); + return entry({ + uri: `skill://s${i}/SKILL.md`, + frontmatter: { name: `s${i}`, description: "A demo" }, + resources: [ + { + uri: `skill://s${i}/SKILL.md`, + digest: await sha256Digest(bytes), + size: bytes.byteLength, + }, + ], + }); + }, + ), + ); + const readResource = vi.fn(async (uri: string) => ({ + result: { + contents: [ + { uri, text: mdFor(Number(/s(\d+)/.exec(uri)?.[1] ?? "0")) }, + ], + }, + })); + const client = { readResource } as unknown as InspectorClientProtocol; + const reports = await verifySkills(client, many); + // Every entry is still REPORTED — the static checks cost no I/O, so a + // skill past the budget is not silently dropped from the output. + expect(reports).toHaveLength(SKILL_MAX_CATALOG_SKILLS + 5); + expect(readResource.mock.calls.length).toBe(SKILL_MAX_CATALOG_SKILLS); + // …and the remainder says so, rather than passing or failing. + const past = reports.slice(SKILL_MAX_CATALOG_SKILLS); + for (const report of past) { + expect(report.outcome).toBe("incomplete"); + expect(report.incomplete).toMatch(/catalog budget/); + expect(report.files).toHaveLength(0); + } + expect(reports[0].outcome).toBe("verified"); + }); + it("is not incomplete when the budget is crossed by the LAST entry", async () => { // Crossing the line on the final row stopped nothing: every manifest entry // was fetched and checked. Reporting "Stopped after 4 of 4" there both diff --git a/core/mcp/skills.ts b/core/mcp/skills.ts index 8ae718d554..61dd759379 100644 --- a/core/mcp/skills.ts +++ b/core/mcp/skills.ts @@ -54,6 +54,28 @@ export const SKILL_MAX_RESOURCE_ENTRIES = 512; /** Maximum total size, in bytes, of a single skill's resources (16 MiB). */ export const SKILL_MAX_TOTAL_BYTES = 16 * 1024 * 1024; +/** + * Maximum skills one verification run will actually read from, and the byte + * ceiling across all of them. + * + * ⚠️ **Not SEP-2640 limits — they are this tool's own.** The SEP bounds a + * single skill and deliberately does not bound a catalog: `skills/list` may be + * arbitrarily long, and a page may hold arbitrarily many entries, so the + * cursor-walk's page cap constrains nothing here. Every entry costs at least + * one `resources/read`, so an unbounded catalog is unbounded work and + * unbounded transfer against the tool sent to inspect it — a `--verify` in CI + * that never returns (Copilot). + * + * Entries past either bound are reported as `incomplete` rather than dropped + * or failed: they were not checked, which is neither a pass nor a verdict + * against the server. A host wanting more is not wrong — these are safety + * limits, not conformance ones — which is why the reason names them. + */ +export const SKILL_MAX_CATALOG_SKILLS = 256; + +/** @see {@link SKILL_MAX_CATALOG_SKILLS} — 64 MiB across the whole run. */ +export const SKILL_MAX_CATALOG_BYTES = 64 * 1024 * 1024; + /** The suffix every skill URI ends with; the segment before it is the name. */ export const SKILL_FILE_SUFFIX = "/SKILL.md"; @@ -971,6 +993,13 @@ export function checkSkillFrontmatterMatch( return issues; } +/** + * How many colliding URIs a `duplicate-name` message names before it counts the + * rest. Three is enough to show the shape of the collision; the count carries + * the scale. + */ +const COLLISION_SAMPLE = 3; + /** * Findings that can only be computed over the **whole listing**, keyed by the * entry they belong to (its normalized URI identity). @@ -1025,11 +1054,35 @@ export function checkSkillNameCollisions( const uris = [...identities].sort(); for (const entry of group) { const self = skillUriIdentity(entry.uri); - const others = uris.filter((uri) => uri !== self); + // ⚠️ A bounded SAMPLE, taken with an early exit — not + // `uris.filter(...)` and not the whole list in the message. Duplicate + // names are legal and SEP-2640 puts no ceiling on a catalog, so a group + // of N made both the work and the generated text O(N²): every one of N + // entries scanned all N URIs and embedded the other N−1 (Copilot). A + // server controls N, which turns a legal listing into a denial of + // service against the tool meant to inspect it. + const sample: string[] = []; + for (const uri of uris) { + if (uri === self) continue; + sample.push(uri); + if (sample.length === COLLISION_SAMPLE) break; + } + const unshown = identities.size - 1 - sample.length; + // Naming a few and counting the rest keeps the finding actionable — a + // reader needs to see that it IS a collision and where to look, not a + // transcript of the catalog. + const others = + unshown > 0 + ? `${sample.join(", ")}, and ${unshown} more` + : sample.join(", "); + const subject = + identities.size === 2 + ? "Another skill in this listing also declares" + : `${identities.size - 1} other skills in this listing also declare`; issues.set(self, { code: "duplicate-name", severity: "warning", - message: `Another skill in this listing also declares the name "${name}" (${others.join(", ")}). This is legal — a consumer must tell them apart by their URIs rather than collapsing or preferring one.`, + message: `${subject} the name "${name}" (${others}). This is legal — a consumer must tell them apart by their URIs rather than collapsing or preferring one.`, }); } } diff --git a/core/mcp/skillsVerification.ts b/core/mcp/skillsVerification.ts index c4ee2da9d7..c034b465ef 100644 --- a/core/mcp/skillsVerification.ts +++ b/core/mcp/skillsVerification.ts @@ -33,6 +33,8 @@ import { checkSkillFrontmatterMatch, checkSkillNameCollisions, skillDisplayName, + SKILL_MAX_CATALOG_BYTES, + SKILL_MAX_CATALOG_SKILLS, SKILL_MAX_RESOURCE_ENTRIES, SKILL_MAX_TOTAL_BYTES, skillFileBytes, @@ -219,6 +221,42 @@ function contentsFor(result: unknown, uri: string): ReadContents | undefined { return undefined; } +/** + * How many bytes a string occupies as UTF-8, without encoding a copy of it. + * + * `TextEncoder` would be the obvious answer and allocates a second buffer for + * a payload that may already be megabytes — and this is called on responses a + * hostile server chose the size of, which is the case the count exists to + * bound. Counting is O(n) and allocates nothing. + * + * A high surrogate is only worth 4 bytes when a low surrogate actually follows + * it. An unpaired one encodes as U+FFFD, which is 3 — the same as any other + * BMP character in that range, so it needs no special case beyond not + * consuming the next unit. + */ +export function utf8Length(value: string): number { + let total = 0; + for (let i = 0; i < value.length; i += 1) { + const code = value.charCodeAt(i); + if (code < 0x80) { + total += 1; + } else if (code < 0x800) { + total += 2; + } else if (code >= 0xd800 && code <= 0xdbff) { + const next = i + 1 < value.length ? value.charCodeAt(i + 1) : 0; + if (next >= 0xdc00 && next <= 0xdfff) { + total += 4; + i += 1; + } else { + total += 3; + } + } else { + total += 3; + } + } + return total; +} + /** * What a `resources/read` response cost to receive, charged against the byte * budget regardless of whether any of it is usable. @@ -230,12 +268,13 @@ function contentsFor(result: unknown, uri: string): ReadContents | undefined { * or send an enormous blob that is not valid base64. Both leave the decode * paths empty-handed while the bytes have already crossed the wire. * - * The figure is an **approximation, and deliberately never an undercount by - * more than a small factor**: `text` is charged in UTF-16 code units (UTF-8 is - * between 1× and 3× that for the same string) and `blob` in base64 characters - * (roughly 4/3 of the bytes it decodes to, so an overcharge). Exactness is not - * the point — this is a safety limit, not an accounting figure, and the caller - * substitutes the exact decoded length whenever it has one. + * `text` is charged in **UTF-8 bytes**, the unit the limit is written in. + * Charging `text.length` instead — UTF-16 code units — undercharged every + * non-ASCII payload by up to 3×, so a decoy block of emoji or CJK kept the + * counter under 16 MiB while the wire carried far more, and the walk read on + * (Copilot). `blob` is charged in base64 characters, which is ~4/3 of what it + * decodes to: an OVERcharge, and deliberately left as one, since a blob that + * fails to decode has no byte count to be exact about. */ function responseBytes(result: unknown): number { const contents = (result as { contents?: unknown })?.contents; @@ -244,7 +283,7 @@ function responseBytes(result: unknown): number { for (const block of contents) { if (typeof block !== "object" || block === null) continue; const { text, blob } = block as { text?: unknown; blob?: unknown }; - if (typeof text === "string") total += text.length; + if (typeof text === "string") total += utf8Length(text); if (typeof blob === "string") total += blob.length; } return total; @@ -278,7 +317,21 @@ export async function verifySkills( // reports no collision: there is no listing to collide within. const collisions = checkSkillNameCollisions(entries); const reports: SkillVerifyReport[] = []; + // ⚠️ Run-level budgets, on top of the per-skill ones below. The per-skill + // caps bound what ONE entry can cost; nothing bounded how many entries there + // are, and SEP-2640 puts no ceiling on a catalog — so a listing of a hundred + // thousand skills, each individually conforming, made `--verify` run + // indefinitely and transfer unboundedly (Copilot). See + // {@link SKILL_MAX_CATALOG_SKILLS}. + let walkedSkills = 0; + let catalogBytes = 0; for (const entry of entries) { + // Static checks still run for every entry — they cost no I/O, so a skill + // past the budget is still reported on, just not read. What stops is the + // reading. + const withinBudget = + walkedSkills < SKILL_MAX_CATALOG_SKILLS && + catalogBytes <= SKILL_MAX_CATALOG_BYTES; // The entry's own SKILL.md, read once and used twice — for its digest and // for the frontmatter cross-check. Reading it twice would double the load // on the server and, worse, could compare a digest against one snapshot @@ -313,9 +366,10 @@ export async function verifySkills( // is bounded by the count cap regardless. A read is skipped only when the // running total would CROSS the limit, so a conforming skill (≤ 16 MiB in // total, by definition) is never truncated. - const manifest = boundedManifest(declared); - let incomplete = - manifest.length < declared.length + const manifest = withinBudget ? boundedManifest(declared) : []; + let incomplete = !withinBudget + ? `Not read: this run already reached its catalog budget of ${SKILL_MAX_CATALOG_SKILLS} skills / ${SKILL_MAX_CATALOG_BYTES} bytes. Nothing about this skill's files has been checked — verify it on its own with \`--method skills/get --uri\` to get a verdict.` + : manifest.length < declared.length ? `Only ${manifest.length} of ${declared.length} manifest entries were read: the skill exceeds the ${SKILL_MAX_RESOURCE_ENTRIES}-entry / ${SKILL_MAX_TOTAL_BYTES}-byte interoperability limits, so the rest were not fetched and cannot be reported on.` : undefined; // ⚠️ Bytes ACTUALLY RECEIVED, which is the only budget a server cannot @@ -436,7 +490,7 @@ export async function verifySkills( // because a cap excluded it or because the byte budget broke the loop // first, still gets the fallback: the frontmatter comparison is mandatory // and must not be lost to a limit that exists to bound unrelated files. - if (!selfAttempted) { + if (withinBudget && !selfAttempted) { // Recorded as a file result, not swallowed. Because a dynamic skill has // no manifest rows, `files` would otherwise stay empty and its only static // finding is a warning — so an unreadable SKILL.md returned `ok: true` @@ -485,6 +539,13 @@ export async function verifySkills( } } + // Charged after this entry's reads, so the skill that crosses the run + // budget is still fully reported rather than half-read. The NEXT one stops. + if (withinBudget) { + walkedSkills += 1; + catalogBytes += receivedBytes; + } + const entryText = entryBytes === undefined ? undefined : bytesToText(entryBytes);