diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 375d7010..cf43bda3 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -60,6 +60,8 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; +import { EventsDiscoveryScenario } from './server/events/discovery'; +import { EventsPollScenario } from './server/events/poll'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; @@ -165,7 +167,16 @@ const pendingClientScenariosList: ClientScenario[] = [ // `npm start -- server --scenario sep-2640-skills-* --url `. new SkillsDirectoryReadScenario(), new SkillsEnumerationScenario(), - new SkillsManifestScenario() + new SkillsManifestScenario(), + + // MCP Events. Pending because the everything-server does not implement the + // events capability; targeted runs point at an events-capable fixture via + // `npm start -- server --scenario events-* --url `. The suite + // scores against the merged design sketch in + // modelcontextprotocol/experimental-ext-triggers-events, which has no SEP + // number yet — see the header of src/seps/sep-9999.yaml. + new EventsDiscoveryScenario(), + new EventsPollScenario() ]; // All client scenarios @@ -223,6 +234,11 @@ const allClientScenariosList: ClientScenario[] = [ new SkillsEnumerationScenario(), new SkillsManifestScenario(), + // MCP Events. Fixture-dependent (needs a server declaring `capabilities.events`); + // each scenario SKIPs cleanly when the capability is not declared. + new EventsDiscoveryScenario(), + new EventsPollScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/events/discovery.ts b/src/scenarios/server/events/discovery.ts new file mode 100644 index 00000000..16cb0a15 --- /dev/null +++ b/src/scenarios/server/events/discovery.ts @@ -0,0 +1,548 @@ +/** + * MCP Events — capability declaration, `events/list`, and the error-code + * contract. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec excerpt lives next to its check ID in + * src/seps/sep-9999.yaml, keeping the yaml and this scenario in lock-step. + * 9999 is a placeholder SEP number; see that file's header. + * + * This is the gate scenario for the suite: the delivery-mode scenarios all + * start from a descriptor found here, so a server that fails `events/list` + * fails everything downstream for a reason this scenario names. + * + * Discovery is dynamic and brand-neutral. The scenario enumerates whatever the + * server serves and validates the descriptors it finds, hardcoding no event + * name. An empty catalog is permitted by the document, so descriptor-level + * checks report the unmet prerequisite via `untestableCheck` rather than + * passing vacuously. + * + * The capability gate is two-sided rather than a plain SKIP. An optional + * capability a server never declared is not a defect, so a server that also + * does not implement `events/list` skips the whole scenario. A server that + * answers `events/list` while declaring nothing is a different thing: it has + * an events surface that a client following the spec would never call, and + * SKIP would report that as a clean run. So the scenario asks before it + * skips. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_EXTENSION_ID, + EVENTS_CAPABILITY, + EVENTS_LIST_METHOD, + EVENTS_POLL_METHOD, + EVENTS_SPEC_REF, + EVENTS_NOT_FOUND, + EVENTS_UNSUPPORTED, + DELIVERY_MODES, + JSONRPC_METHOD_NOT_FOUND, + type EventDescriptor, + declaredEventsCapability, + eventsCapability, + eventsCheck, + eventsListPage, + eventsListAll, + eventsPoll, + deliveryModes, + descriptorName, + descriptorLabel, + describeValue, + isObject, + inServerErrorRange, + unknownEventName +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-9999-capability-events-object', + 'sep-9999-capability-list-changed-flag' +] as const; + +const LIST_IDS = [ + 'sep-9999-list-implemented', + 'sep-9999-list-pagination' +] as const; + +const DESCRIPTOR_IDS = [ + 'sep-9999-descriptor-name', + 'sep-9999-descriptor-description', + 'sep-9999-descriptor-delivery-subset', + 'sep-9999-descriptor-input-schema', + 'sep-9999-descriptor-payload-schema', + 'sep-9999-descriptor-meta' +] as const; + +const ERROR_IDS = [ + 'sep-9999-error-not-found', + 'sep-9999-error-server-range' +] as const; + +const ALL_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...DESCRIPTOR_IDS, + ...ERROR_IDS +]; + +/** Every check this scenario can emit, as SKIPPED with one shared reason. */ +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsDiscoveryScenario implements ClientScenario { + name = 'events-discovery'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: capability declaration, \`events/list\` enumeration, and the error-code contract. + +**Methods**: \`events/list\` (mandatory for a server declaring \`capabilities.events\`), \`events/poll\` (probed only for its error path) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-capability-events-object\` — \`events\` is declared top-level under \`capabilities\` as an object +- \`sep-9999-capability-list-changed-flag\` — \`listChanged\`, when present, is a boolean +- \`sep-9999-list-implemented\` — \`events/list\` is implemented and returns an \`events\` array +- \`sep-9999-list-pagination\` — \`nextCursor\` is honoured as a cursor on the next request +- \`sep-9999-descriptor-*\` — the descriptor fields: \`name\`, \`description\`, \`delivery\` as a non-empty subset of poll/push/webhook, \`inputSchema\`, \`payloadSchema\`, and \`_meta\` when present +- \`sep-9999-error-not-found\` — an unknown event name answers \`-32011 NotFound\` (the poll-specific restatement of the same rule is graded by \`events-poll\`) +- \`sep-9999-error-server-range\` — the extension's codes sit in the JSON-RPC implementation-defined server range + +**Discovery is dynamic**: a server that neither declares the capability nor implements \`events/list\` SKIPs everything. One that answers \`events/list\` without declaring the capability is graded, and fails the declaration check, because that surface is unreachable for a client that reads capabilities first. An empty catalog reports the descriptor checks as untestable rather than passing them.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + return await this.checks(conn); + } finally { + await conn.close(); + } + } + + private async checks( + conn: Awaited> + ): Promise { + const checks: ConformanceCheck[] = []; + + // --- Capability ------------------------------------------------------ + const capDescription = + 'Servers advertise event support in their capabilities as an object under `capabilities.events`.'; + const { declared, value } = await declaredEventsCapability(conn); + + if (!declared) { + // An undeclared optional capability is normally a SKIP. It is not one + // when the server answers `events/list` anyway: that server has an + // events surface no spec-following client can discover, and SKIP would + // report it as a clean run. Distinguish the two by asking. + const probe = await eventsListPage(conn); + if ('error' in probe && probe.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + checks.push( + eventsCheck( + 'sep-9999-capability-events-object', + capDescription, + 'FAILURE', + { + errorMessage: `Server answers \`${EVENTS_LIST_METHOD}\` but declares no \`capabilities.${EVENTS_CAPABILITY}\`. A client that follows the spec reads capabilities to decide whether to call it, so this surface is unreachable.`, + details: { capabilities: EVENTS_CAPABILITY, declared: false } + } + ) + ); + } else if (isObject(value)) { + checks.push( + eventsCheck( + 'sep-9999-capability-events-object', + capDescription, + 'SUCCESS' + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-9999-capability-events-object', + capDescription, + 'FAILURE', + { + errorMessage: `\`capabilities.${EVENTS_CAPABILITY}\` is ${describeValue(value)}, expected an object.`, + details: { declared: value } + } + ) + ); + } + + const caps = declared ? await eventsCapability(conn) : undefined; + const listChanged = caps?.listChanged; + if (listChanged === undefined) { + checks.push( + eventsCheck( + 'sep-9999-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'SKIPPED', + { + errorMessage: declared + ? 'Server did not declare `listChanged`; the flag is optional and its absence means the notification is not advertised.' + : 'Server declared no `capabilities.events` object for the flag to sit in; see sep-9999-capability-events-object.' + } + ) + ); + } else if (typeof listChanged === 'boolean') { + checks.push( + eventsCheck( + 'sep-9999-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'SUCCESS', + { details: { listChanged } } + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-9999-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'FAILURE', + { + errorMessage: `\`capabilities.events.listChanged\` is ${describeValue(listChanged)}, expected a boolean.`, + details: { listChanged } + } + ) + ); + } + + // --- events/list ----------------------------------------------------- + const firstPage = await eventsListPage(conn); + if ('error' in firstPage) { + const err = firstPage.error; + const unimplemented = err.code === JSONRPC_METHOD_NOT_FOUND; + checks.push( + eventsCheck( + 'sep-9999-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'FAILURE', + { + errorMessage: unimplemented + ? `Server declares \`capabilities.events\` but \`${EVENTS_LIST_METHOD}\` is not implemented (-32601).` + : `\`${EVENTS_LIST_METHOD}\` failed: ${err.code} ${err.message}`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + // Nothing downstream can be graded without a catalog. + const reason = `\`${EVENTS_LIST_METHOD}\` did not return a result (${err.code} ${err.message}).`; + for (const id of [...LIST_IDS.slice(1), ...DESCRIPTOR_IDS]) { + checks.push( + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], 'FAILURE') + ); + } + checks.push(...(await this.errorChecks(conn, []))); + return checks; + } + + const eventsField = firstPage.result.events; + if (Array.isArray(eventsField)) { + checks.push( + eventsCheck( + 'sep-9999-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'SUCCESS', + { details: { firstPageCount: firstPage.descriptors.length } } + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-9999-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'FAILURE', + { + errorMessage: `\`${EVENTS_LIST_METHOD}\` result \`events\` is ${describeValue(eventsField)}, expected an array.`, + details: { events: eventsField } + } + ) + ); + } + + checks.push(await this.paginationCheck(conn, firstPage.result.nextCursor)); + + const all = await eventsListAll(conn); + const descriptors = + 'error' in all ? firstPage.descriptors : all.descriptors; + checks.push(...this.descriptorChecks(descriptors)); + checks.push(...(await this.errorChecks(conn, descriptors))); + + return checks; + } + + /** + * `nextCursor` is only gradeable when the server actually paginates. A + * single-page catalog leaves nothing to follow, which is a missing + * prerequisite rather than a pass: the document defers the semantics to the + * base protocol, so the only thing this check can establish is that + * `events/list` participates in the scheme at all. + */ + private async paginationCheck( + conn: Awaited>, + nextCursor: unknown + ): Promise { + const id = 'sep-9999-list-pagination'; + const description = + '`nextCursor` is present when more pages are available; same semantics as tools/list.'; + + if (nextCursor === undefined || nextCursor === null) { + return untestableCheck( + id, + id, + description, + `Server returned a single page from \`${EVENTS_LIST_METHOD}\` with no \`nextCursor\`, so cursor round-tripping could not be exercised.`, + [EVENTS_SPEC_REF], + 'WARNING' + ); + } + + if (typeof nextCursor !== 'string' || nextCursor.length === 0) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `\`nextCursor\` is ${describeValue(nextCursor)}, expected a non-empty string.`, + details: { nextCursor } + }); + } + + const second = await eventsListPage(conn, nextCursor); + if ('error' in second) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Server returned \`nextCursor\` but rejected it on the next \`${EVENTS_LIST_METHOD}\`: ${second.error.code} ${second.error.message}`, + details: { nextCursor, code: second.error.code } + }); + } + + if (second.result.nextCursor === nextCursor) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: + 'Server echoed the same `nextCursor` on the following page, which never terminates.', + details: { nextCursor } + }); + } + + return eventsCheck(id, description, 'SUCCESS', { + details: { secondPageCount: second.descriptors.length } + }); + } + + /** + * Grade every descriptor and report one check per field, naming the first + * offender. One check per field rather than per descriptor keeps the check + * IDs stable across servers with different catalog sizes. + */ + private descriptorChecks(descriptors: EventDescriptor[]): ConformanceCheck[] { + if (descriptors.length === 0) { + const reason = `Server's \`${EVENTS_LIST_METHOD}\` returned an empty catalog, so no descriptor could be validated.`; + return DESCRIPTOR_IDS.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], 'FAILURE') + ); + } + + const out: ConformanceCheck[] = []; + + const field = ( + id: string, + description: string, + severity: 'FAILURE' | 'WARNING', + predicate: (d: EventDescriptor) => string | undefined + ) => { + for (const [i, d] of descriptors.entries()) { + const problem = predicate(d); + if (problem) { + out.push( + eventsCheck(id, description, severity, { + errorMessage: `${descriptorLabel(d, i)}: ${problem}`, + details: { descriptor: d } + }) + ); + return; + } + } + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { descriptorsChecked: descriptors.length } + }) + ); + }; + + field( + 'sep-9999-descriptor-name', + 'Each event descriptor carries a `name` identifying the event type.', + 'FAILURE', + (d) => + descriptorName(d) === undefined + ? `\`name\` is ${describeValue(d.name)}, expected a non-empty string.` + : undefined + ); + + field( + 'sep-9999-descriptor-description', + 'Each event descriptor carries a `description` of when the event fires.', + 'WARNING', + (d) => + typeof d.description === 'string' && d.description.length > 0 + ? undefined + : `\`description\` is ${describeValue(d.description)}, expected a non-empty string.` + ); + + field( + 'sep-9999-descriptor-delivery-subset', + '`delivery` lists the delivery modes this event type supports — any non-empty subset of `poll`, `push`, `webhook`.', + 'FAILURE', + (d) => { + if (!Array.isArray(d.delivery)) { + return `\`delivery\` is ${describeValue(d.delivery)}, expected an array.`; + } + const modes = deliveryModes(d); + if (modes.length === 0) + return '`delivery` is empty; the subset must be non-empty.'; + const unknown = modes.filter( + (m) => !(DELIVERY_MODES as readonly string[]).includes(m) + ); + if (unknown.length > 0) { + return `\`delivery\` contains ${unknown.map((m) => `\`${m}\``).join(', ')}, outside the poll/push/webhook set.`; + } + if (new Set(modes).size !== modes.length) { + return '`delivery` repeats a mode; it is a subset, not a list.'; + } + return undefined; + } + ); + + field( + 'sep-9999-descriptor-input-schema', + '`inputSchema` is a JSON Schema describing valid subscription arguments.', + 'FAILURE', + (d) => + isObject(d.inputSchema) + ? undefined + : `\`inputSchema\` is ${describeValue(d.inputSchema)}, expected a JSON Schema object.` + ); + + field( + 'sep-9999-descriptor-payload-schema', + '`payloadSchema` describes the shape of `data` in delivered events.', + 'FAILURE', + (d) => + isObject(d.payloadSchema) + ? undefined + : `\`payloadSchema\` is ${describeValue(d.payloadSchema)}, expected a JSON Schema object.` + ); + + field( + 'sep-9999-descriptor-meta', + '`_meta` on an event descriptor is optional; same semantics as on Tool/Resource/Prompt.', + 'WARNING', + (d) => + d._meta === undefined || isObject(d._meta) + ? undefined + : `\`_meta\` is ${describeValue(d._meta)}, expected an object when present.` + ); + + return out; + } + + /** + * Probe the unknown-name error path through `events/poll`. + * + * Poll is the cheapest probe: it holds no server-side state, so a rejected + * call leaves nothing behind. The document states the same obligation twice, + * once as the general `-32011 NotFound` code and once as the poll-specific + * consequence of an event type having been removed, so both rows are graded + * from this one exchange rather than by poking the server twice. + */ + private async errorChecks( + conn: Awaited>, + descriptors: EventDescriptor[] + ): Promise { + const notFoundDesc = + '`-32011 NotFound` — a referenced entity does not exist, such as an unknown event name.'; + const rangeDesc = + "The extension's codes are carried in the JSON-RPC implementation-defined server range `[-32000, -32099]`."; + + // A server offering no poll-capable event type may legitimately not route + // events/poll at all, which would make -32601 the honest answer and this + // probe meaningless. + const pollable = descriptors.some((d) => deliveryModes(d).includes('poll')); + if (descriptors.length > 0 && !pollable) { + const reason = `No event type advertises \`poll\` delivery, so \`${EVENTS_POLL_METHOD}\` could not be used to probe the unknown-name error path.`; + return [ + untestableCheck( + 'sep-9999-error-not-found', + 'sep-9999-error-not-found', + notFoundDesc, + reason, + [EVENTS_SPEC_REF], + 'FAILURE' + ), + untestableCheck( + 'sep-9999-error-server-range', + 'sep-9999-error-server-range', + rangeDesc, + reason, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + ]; + } + + const name = unknownEventName(); + const probe = await eventsPoll(conn, { name, arguments: {}, cursor: null }); + + if (!('error' in probe)) { + const errorMessage = `\`${EVENTS_POLL_METHOD}\` for unknown event name \`${name}\` returned a result instead of an error.`; + return [ + eventsCheck('sep-9999-error-not-found', notFoundDesc, 'FAILURE', { + errorMessage, + details: { result: probe.result } + }), + eventsCheck('sep-9999-error-server-range', rangeDesc, 'FAILURE', { + errorMessage + }) + ]; + } + + const { code, message, data } = probe.error; + const isNotFound = code === EVENTS_NOT_FOUND; + const details = { code, message, data, probedName: name }; + + const out: ConformanceCheck[] = []; + + out.push( + isNotFound + ? eventsCheck('sep-9999-error-not-found', notFoundDesc, 'SUCCESS', { + details + }) + : eventsCheck('sep-9999-error-not-found', notFoundDesc, 'FAILURE', { + errorMessage: `Unknown event name answered ${code}, expected ${EVENTS_NOT_FOUND} NotFound.${ + code === EVENTS_UNSUPPORTED + ? ' `-32014 Unsupported` is for a well-formed request naming an option the server does not offer, not for a name it does not have.' + : '' + }`, + details + }) + ); + + out.push( + inServerErrorRange(code) + ? eventsCheck('sep-9999-error-server-range', rangeDesc, 'SUCCESS', { + details + }) + : eventsCheck('sep-9999-error-server-range', rangeDesc, 'FAILURE', { + errorMessage: `Error code ${code} is outside the implementation-defined server range [-32099, -32000].`, + details + }) + ); + + return out; + } +} + +/** Exported for the negative tests, which assert the full emitted set. */ +export const EVENTS_DISCOVERY_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts new file mode 100644 index 00000000..22cfa7c6 --- /dev/null +++ b/src/scenarios/server/events/helpers.ts @@ -0,0 +1,391 @@ +/** + * Shared helpers for the MCP Events server-conformance scenarios under this + * directory. + * + * Extracted against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events (merged 2026-09-08). + * Each check's verbatim excerpt lives next to its check ID in + * src/seps/sep-9999.yaml, and 9999 is a placeholder SEP number — see that + * file's header before renaming anything here. + * + * Two things about Events differ from every other extension suite here and are + * worth knowing before reading the scenarios: + * + * 1. The capability is declared at the top level as `capabilities.events`, not + * under `capabilities.extensions`. SEP-2133's extensions map does not come + * into it. `EVENTS_EXTENSION_ID` exists only as a `ScenarioSource` key so + * the runner keeps these scenarios off the `--spec-version` timeline; it is + * never a path into the capability object. + * + * 2. No delivery mode is mandatory. A descriptor's `delivery` array is any + * non-empty subset of poll/push/webhook, so a scenario for one mode has to + * discover whether any event type offers it before it can probe anything. + * Nothing is hardcoded to a fixture's event names. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; + +/** + * Suite-selection key for the Events scenarios. + * + * Events has no SEP-2133 extension identifier, because it declares its + * capability top-level rather than inside `capabilities.extensions`. This + * string exists so `ScenarioSource` can carry `{ extensionId }`, which is what + * keeps the scenarios out of `--spec-version` selection (see + * `matchesSpecVersion` in src/scenarios/index.ts). Do not read the capability + * at this key; read `capabilities.events`. + */ +export const EVENTS_EXTENSION_ID = 'io.modelcontextprotocol/events'; + +/** The capability key, top-level under `capabilities`. */ +export const EVENTS_CAPABILITY = 'events'; + +export const EVENTS_LIST_METHOD = 'events/list'; +export const EVENTS_POLL_METHOD = 'events/poll'; +export const EVENTS_STREAM_METHOD = 'events/stream'; +export const EVENTS_SUBSCRIBE_METHOD = 'events/subscribe'; +export const EVENTS_UNSUBSCRIBE_METHOD = 'events/unsubscribe'; + +export const EVENTS_LIST_CHANGED_NOTIFICATION = + 'notifications/events/list_changed'; +export const EVENTS_EVENT_NOTIFICATION = 'notifications/events/event'; +export const EVENTS_ACTIVE_NOTIFICATION = 'notifications/events/active'; +export const EVENTS_HEARTBEAT_NOTIFICATION = 'notifications/events/heartbeat'; +export const EVENTS_ERROR_NOTIFICATION = 'notifications/events/error'; +export const EVENTS_TERMINATED_NOTIFICATION = 'notifications/events/terminated'; + +/** + * The `_meta` key carrying the parent `events/stream` request id on every + * `notifications/events/*` message, per SEP-2575's correlation convention. + */ +export const SUBSCRIPTION_ID_META = 'io.modelcontextprotocol/subscriptionId'; + +/** The three delivery modes, as they appear in a descriptor's `delivery`. */ +export const DELIVERY_MODES = ['poll', 'push', 'webhook'] as const; +export type DeliveryMode = (typeof DELIVERY_MODES)[number]; + +/** Standard JSON-RPC. */ +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; + +/** + * The general-purpose codes this document defines, carried in the JSON-RPC + * implementation-defined server range. Named for reuse across MCP rather than + * scoped to events, and each conveys its specifics through a typed `data` + * payload rather than by minting more numbers. + */ +export const EVENTS_NOT_FOUND = -32011; +export const EVENTS_FORBIDDEN = -32012; +export const EVENTS_RESOURCE_EXHAUSTED = -32013; +export const EVENTS_UNSUPPORTED = -32014; +export const EVENTS_CALLBACK_ENDPOINT_ERROR = -32015; + +/** Inclusive bounds of the JSON-RPC implementation-defined server range. */ +export const SERVER_ERROR_RANGE_MIN = -32099; +export const SERVER_ERROR_RANGE_MAX = -32000; + +export const EVENTS_SPEC_REF: SpecReference = { + id: 'MCP-Events', + url: 'https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md' +}; + +/** One entry of the `events` array returned by `events/list`. */ +export interface EventDescriptor { + name?: unknown; + description?: unknown; + delivery?: unknown; + inputSchema?: unknown; + payloadSchema?: unknown; + _meta?: unknown; + [key: string]: unknown; +} + +export interface EventsListResult { + events?: unknown; + nextCursor?: unknown; + [key: string]: unknown; +} + +/** One entry of a poll response's `events` array, or a pushed notification. */ +export interface EventOccurrence { + eventId?: unknown; + name?: unknown; + timestamp?: unknown; + data?: unknown; + cursor?: unknown; + _meta?: unknown; + [key: string]: unknown; +} + +export interface EventsPollResult { + events?: unknown; + cursor?: unknown; + truncated?: unknown; + hasMore?: unknown; + nextPollMs?: unknown; + [key: string]: unknown; +} + +/** + * Build a check carrying the Events spec reference. Per AGENTS.md the same + * `id` flips `status` + `errorMessage` between SUCCESS and FAILURE rather than + * branching into distinct slugs. + */ +export function eventsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [EVENTS_SPEC_REF], + ...extras + }; +} + +/** A JSON object, as opposed to an array, `null`, or a primitive. */ +export function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * How to name an observed value in an error message. + * + * `absent` rather than `a undefined`, because a reader chasing a failure needs + * to know the field was missing, and the JavaScript spelling of that is noise. + */ +export function describeValue(value: unknown): string { + if (value === undefined) return 'absent'; + if (value === null) return 'null'; + if (Array.isArray(value)) return 'an array'; + return `a ${typeof value}`; +} + +/** + * Whether the server declared the events capability at all, and the raw value + * it declared it with, before any shape coercion. + * + * Kept separate from `eventsCapability` so callers can tell "absent" from + * "declared with the wrong type" apart. Folding the two together would turn a + * server that declares `events: true` into a clean SKIP of the whole suite, + * which reads as a green run against a server that is plainly wrong. + */ +export async function declaredEventsCapability( + conn: Connection +): Promise<{ declared: boolean; value: unknown }> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + if (!(EVENTS_CAPABILITY in caps)) + return { declared: false, value: undefined }; + return { declared: true, value: caps[EVENTS_CAPABILITY] }; +} + +/** + * The events capability object, or `undefined` when the server did not declare + * it — or declared it with something that is not an object, which callers + * treat the same way. An undeclared optional capability is a SKIP. + */ +export async function eventsCapability( + conn: Connection +): Promise | undefined> { + const { value } = await declaredEventsCapability(conn); + return isObject(value) ? value : undefined; +} + +/** A single `events/list` page, kept separate so pagination can be inspected. */ +export interface EventsListPage { + result: EventsListResult; + descriptors: EventDescriptor[]; +} + +/** + * Call `events/list` once, optionally with a cursor. Returns the `JsonRpcError` + * rather than throwing, so a scenario can grade the error instead of aborting. + */ +export async function eventsListPage( + conn: Connection, + cursor?: string +): Promise { + try { + const result = await conn.request( + EVENTS_LIST_METHOD, + cursor ? { cursor } : undefined + ); + const raw = result?.events; + return { + result: result ?? {}, + descriptors: Array.isArray(raw) ? (raw as EventDescriptor[]) : [] + }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } +} + +/** + * Every descriptor from `events/list`, paginating until `nextCursor` clears. + * + * Bounded at `maxPages` because a server that echoes the same `nextCursor` + * forever would otherwise hang the scenario rather than fail it. Hitting the + * bound is reported through `truncatedByBound` so the caller can say so instead + * of silently grading a partial catalog. + */ +export async function eventsListAll( + conn: Connection, + maxPages = 20 +): Promise< + | { descriptors: EventDescriptor[]; pages: number; truncatedByBound: boolean } + | { error: JsonRpcError } +> { + const out: EventDescriptor[] = []; + const seen = new Set(); + let cursor: string | undefined; + let pages = 0; + + do { + const page = await eventsListPage(conn, cursor); + if ('error' in page) return page; + pages += 1; + out.push(...page.descriptors); + + const next = page.result.nextCursor; + if (typeof next !== 'string' || next.length === 0) break; + // A repeated cursor is a server bug; stop rather than loop forever. The + // pagination check grades it, this helper just refuses to hang. + if (seen.has(next)) break; + seen.add(next); + cursor = next; + } while (pages < maxPages); + + return { descriptors: out, pages, truncatedByBound: pages >= maxPages }; +} + +/** The `delivery` array of a descriptor, or `[]` when it is missing/malformed. */ +export function deliveryModes(descriptor: EventDescriptor): string[] { + const d = descriptor.delivery; + return Array.isArray(d) + ? d.filter((m): m is string => typeof m === 'string') + : []; +} + +/** The first descriptor advertising `mode`, or `undefined` when none does. */ +export function firstSupporting( + descriptors: EventDescriptor[], + mode: DeliveryMode +): EventDescriptor | undefined { + return descriptors.find((d) => deliveryModes(d).includes(mode)); +} + +/** A descriptor's `name` when it is a usable string, else `undefined`. */ +export function descriptorName( + descriptor: EventDescriptor +): string | undefined { + return typeof descriptor.name === 'string' && descriptor.name.length > 0 + ? descriptor.name + : undefined; +} + +/** + * How to refer to a descriptor in an error message without assuming it has a + * usable `name` — the checks that grade `name` itself run against descriptors + * that may not. + */ +export function descriptorLabel( + descriptor: EventDescriptor, + index: number +): string { + const name = descriptorName(descriptor); + return name ? `\`${name}\`` : `events[${index}]`; +} + +/** + * Arguments that satisfy a descriptor's `inputSchema` well enough to poll with. + * + * Deliberately minimal: an empty object. Every `inputSchema` in the document is + * an object schema whose properties are filters and transforms, none of them + * required, so `{}` means "no filtering" and is valid against all of them. A + * schema that does declare `required` is the one case this cannot satisfy, and + * the caller reports that as an unmet prerequisite rather than guessing values + * a server would then reject for the wrong reason. + */ +export function minimalArguments( + descriptor: EventDescriptor +): Record | undefined { + const schema = descriptor.inputSchema; + if (!isObject(schema)) return {}; + const required = schema.required; + if (Array.isArray(required) && required.length > 0) return undefined; + return {}; +} + +/** + * Call `events/poll`, returning the `JsonRpcError` rather than throwing so the + * caller can grade error codes. + */ +export async function eventsPoll( + conn: Connection, + params: Record +): Promise<{ result: EventsPollResult } | { error: JsonRpcError }> { + try { + const result = await conn.request( + EVENTS_POLL_METHOD, + params + ); + return { result: result ?? {} }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } +} + +/** The `events` array of a poll result, or `[]` when missing/malformed. */ +export function occurrences(result: EventsPollResult): EventOccurrence[] { + return Array.isArray(result.events) + ? (result.events as EventOccurrence[]) + : []; +} + +/** + * Whether a value is an acceptable cursor: a string, `null`, or absent. + * + * "Absent means null" is normative in both directions, so a missing field is + * not a defect and callers must not treat it as one. + */ +export function isValidCursor(value: unknown): boolean { + return value === undefined || value === null || typeof value === 'string'; +} + +/** Whether a value parses as an ISO 8601 instant. */ +export function isIso8601(value: unknown): boolean { + if (typeof value !== 'string') return false; + const t = Date.parse(value); + if (Number.isNaN(t)) return false; + // Date.parse accepts bare dates and a few non-ISO forms; require at least a + // date and a time separated by `T`, which every example in the document has. + return /^\d{4}-\d{2}-\d{2}T/.test(value); +} + +/** Whether `code` sits in the JSON-RPC implementation-defined server range. */ +export function inServerErrorRange(code: number): boolean { + return code >= SERVER_ERROR_RANGE_MIN && code <= SERVER_ERROR_RANGE_MAX; +} + +/** + * A name no conformant server should be serving, for probing the "unknown + * event name" error path. Randomised so a fixture cannot accidentally define + * it, and prefixed so a human reading server logs knows where it came from. + */ +export function unknownEventName(): string { + return `conformance.nonexistent.${Math.random().toString(36).slice(2, 10)}`; +} diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts new file mode 100644 index 00000000..30d0622c --- /dev/null +++ b/src/scenarios/server/events/negative.test.ts @@ -0,0 +1,599 @@ +import { describe, test, expect } from 'vitest'; +import { createServer, type IncomingMessage, type Server } from 'http'; +import type { AddressInfo } from 'net'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { EventsDiscoveryScenario } from './discovery'; +import { EventsPollScenario } from './poll'; + +/** + * Negative controls for the MCP Events scenarios. + * + * A passing run against a conformant fixture proves a check does not + * false-positive. It does not prove the check catches anything, which is what + * these tests are for: every assertion below pairs a conformant server with a + * server broken in exactly one way, and asserts the check flips. + * + * The three divergences the suite is expected to find in mcpkit each get a + * test here, so the checks that will report them are known to work before + * anyone reads a red run and wonders whether the harness is wrong: + * `nextPollSeconds` (gap G31), and the two error-code paths behind event-type + * removal (gap G30). + * + * The fixture is a minimal SEP-2575 stateless server built per test rather + * than a checked-in example file, matching the SEP-2640 negative tests. An + * events-capable example server is a larger piece of work and belongs with the + * push and webhook scenarios, which genuinely need one. + */ + +/** A descriptor that is well formed apart from whatever a test overrides. */ +function descriptor(overrides: Record = {}) { + return { + name: 'test.event', + description: 'A negative-control fixture event type.', + delivery: ['poll'], + inputSchema: { + type: 'object', + properties: { channel: { type: 'string' } } + }, + payloadSchema: { type: 'object', properties: { id: { type: 'string' } } }, + ...overrides + }; +} + +/** A poll result that is well formed apart from whatever a test overrides. */ +function pollResult(overrides: Record = {}) { + return { + events: [], + cursor: 'cursor_001', + truncated: false, + hasMore: false, + nextPollMs: 30000, + ...overrides + }; +} + +/** An occurrence that is well formed apart from whatever a test overrides. */ +function occurrence(overrides: Record = {}) { + return { + eventId: 'evt_001', + name: 'test.event', + timestamp: '2026-09-15T12:00:00Z', + data: { id: 'x' }, + ...overrides + }; +} + +interface FixtureOptions { + /** Raw value to declare at `capabilities.events`; omit for no declaration. */ + capability?: unknown; + descriptors?: object[]; + /** Answer `events/list` with this JSON-RPC error instead of a result. */ + listError?: { code: number; message: string }; + /** + * Poll responses, consumed in order; the last one repeats once exhausted. + * A `{ error }` entry makes that poll answer with a JSON-RPC error. + */ + pollResponses?: Array< + Record | { error: { code: number; message: string } } + >; + /** Overrides keyed by the polled event name, taking priority over the queue. */ + pollByName?: Record< + string, + Record | { error: { code: number; message: string } } + >; + /** Error code for a poll naming an event type the fixture does not serve. */ + unknownNameCode?: number; + /** Error code for a poll whose arguments violate `inputSchema`. */ + invalidArgsCode?: number; +} + +function startFixture(opts: FixtureOptions): Promise<{ + url: string; + server: Server; + polls: Array>; +}> { + const polls: Array> = []; + const queue = [...(opts.pollResponses ?? [pollResult()])]; + const descriptors = opts.descriptors ?? [descriptor()]; + const names = new Set( + descriptors + .map((d) => (d as { name?: unknown }).name) + .filter((n): n is string => typeof n === 'string') + ); + + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const params = (body.params ?? {}) as Record; + + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (code: number, message: string, data?: unknown) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ jsonrpc: '2.0', id, error: { code, message, data } }) + ); + }; + + if (method === 'server/discover') { + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: 'capability' in opts ? { events: opts.capability } : {}, + serverInfo: { name: 'events-negative', version: '1.0.0' } + }); + return; + } + + if (method === 'events/list') { + if (opts.listError) { + fail(opts.listError.code, opts.listError.message); + return; + } + send({ events: descriptors }); + return; + } + + if (method === 'events/poll') { + polls.push(params); + const name = params.name; + + if (typeof name !== 'string') { + fail(-32602, 'InvalidParams: `name` is required'); + return; + } + if (!names.has(name)) { + fail(opts.unknownNameCode ?? -32011, 'NotFound', { kind: 'event' }); + return; + } + + const byName = opts.pollByName?.[name]; + const chosen = + byName ?? + (queue.length > 1 ? queue.shift()! : (queue[0] ?? pollResult())); + + // Argument validation against the fixture's own declared schema, so the + // invalid-arguments probe has something real to violate. + const args = (params.arguments ?? {}) as Record; + const decl = descriptors.find( + (d) => (d as { name?: unknown }).name === name + ) as { inputSchema?: { properties?: Record } }; + for (const [key, value] of Object.entries(args)) { + const declared = decl?.inputSchema?.properties?.[key]; + if (declared?.type === 'string' && typeof value !== 'string') { + fail(opts.invalidArgsCode ?? -32602, 'InvalidParams'); + return; + } + } + + if ('error' in chosen) { + const e = (chosen as { error: { code: number; message: string } }) + .error; + fail(e.code, e.message); + return; + } + send(chosen as Record); + return; + } + + fail(-32601, `Method not found: ${method}`); + }); + + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, () => { + const addr = server.address() as AddressInfo; + resolve({ url: `http://localhost:${addr.port}/mcp`, server, polls }); + }); + }); +} + +async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} + +type Scenario = EventsDiscoveryScenario | EventsPollScenario; + +async function checksFor(scenario: Scenario, opts: FixtureOptions) { + const { url, server } = await startFixture(opts); + try { + const checks = await scenario.run(testContext(url, DRAFT_PROTOCOL_VERSION)); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await new Promise((r) => server.close(() => r())); + } +} + +const discovery = () => new EventsDiscoveryScenario(); +const poll = () => new EventsPollScenario(); + +/** The baseline every negative case is compared against. */ +const CONFORMANT: FixtureOptions = { + capability: { listChanged: true }, + descriptors: [descriptor()], + pollResponses: [pollResult()] +}; + +describe('events capability declaration', () => { + test('an object declaration passes; a boolean one fails rather than skipping', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-9999-capability-events-object')?.status).toBe('SUCCESS'); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + capability: true + }); + const check = broken.get('sep-9999-capability-events-object'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('a boolean'); + }); + + test('a server that declares nothing and serves nothing skips the suite', async () => { + const checks = await checksFor(discovery(), { + listError: { code: -32601, message: 'Method not found' } + }); + for (const check of checks.values()) { + expect(check.status).toBe('SKIPPED'); + } + }); + + // The case mcpkit is actually in: events/list answers, but nothing is + // declared, so a client that reads capabilities first never calls it. A + // plain SKIP here would report that as a clean run. + test('serving events/list while declaring nothing fails rather than skipping', async () => { + const checks = await checksFor(discovery(), { + descriptors: [descriptor()] + }); + const cap = checks.get('sep-9999-capability-events-object'); + expect(cap?.status).toBe('FAILURE'); + expect(cap?.errorMessage).toContain('declares no `capabilities.events`'); + expect(cap?.errorMessage).toContain('unreachable'); + + // And the rest of the scenario is still graded, not abandoned. + expect(checks.get('sep-9999-list-implemented')?.status).toBe('SUCCESS'); + expect(checks.get('sep-9999-descriptor-name')?.status).toBe('SUCCESS'); + }); + + test('the poll scenario likewise grades an undeclared-but-serving server', async () => { + const checks = await checksFor(poll(), { descriptors: [descriptor()] }); + expect(checks.get('sep-9999-poll-implemented')?.status).toBe('SUCCESS'); + + const skipped = await checksFor(poll(), { + listError: { code: -32601, message: 'Method not found' } + }); + expect(skipped.get('sep-9999-poll-implemented')?.status).toBe('SKIPPED'); + }); + + test('a non-boolean listChanged fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + capability: { listChanged: 'yes' } + }); + const check = checks.get('sep-9999-capability-list-changed-flag'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('a string'); + }); +}); + +describe('events/list descriptors', () => { + test('a descriptor missing name fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ name: undefined })] + }); + expect(checks.get('sep-9999-descriptor-name')?.status).toBe('FAILURE'); + }); + + test('an empty delivery array fails; a valid subset passes', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-9999-descriptor-delivery-subset')?.status).toBe( + 'SUCCESS' + ); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ delivery: [] })] + }); + const check = broken.get('sep-9999-descriptor-delivery-subset'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('non-empty'); + }); + + test('a delivery mode outside poll/push/webhook fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ delivery: ['poll', 'carrier-pigeon'] })] + }); + const check = checks.get('sep-9999-descriptor-delivery-subset'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('carrier-pigeon'); + }); + + test('a missing payloadSchema fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ payloadSchema: undefined })] + }); + expect(checks.get('sep-9999-descriptor-payload-schema')?.status).toBe( + 'FAILURE' + ); + }); + + test('an empty catalog reports descriptor checks untestable, never green', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [] + }); + const check = checks.get('sep-9999-descriptor-name'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + expect(check?.details?.untestable).toBe(true); + }); + + test('an unimplemented events/list fails and names the capability mismatch', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + listError: { code: -32601, message: 'Method not found' } + }); + const check = checks.get('sep-9999-list-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('declares `capabilities.events`'); + }); +}); + +describe('events error codes', () => { + test('an unknown event name answering -32011 passes, -32014 fails', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-9999-error-not-found')?.status).toBe('SUCCESS'); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -32014 + }); + const check = broken.get('sep-9999-error-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32014 Unsupported` is for'); + }); + + test('a code outside the server range fails the range check', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -1 + }); + const check = checks.get('sep-9999-error-server-range'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('outside'); + }); +}); + +describe('events/poll response shape', () => { + test('nextPollSeconds is caught and named as the pre-rename field', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-poll-next-poll-ms')?.status).toBe('SUCCESS'); + + // Gap G31: mcpkit still ships this. The check has to name the rename, or a + // reader of the red run cannot tell it from a server that omits the field. + const legacy = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ nextPollMs: undefined, nextPollSeconds: 30 }) + ] + }); + const check = legacy.get('sep-9999-poll-next-poll-ms'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('nextPollSeconds'); + expect(check?.errorMessage).toContain('197c32b4'); + }); + + test('a non-array events field fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ events: null })] + }); + expect(checks.get('sep-9999-poll-events-array')?.status).toBe('FAILURE'); + }); + + test('a numeric cursor fails the opaque-string check', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ cursor: 42 })] + }); + expect(checks.get('sep-9999-poll-response-cursor')?.status).toBe('FAILURE'); + expect(checks.get('sep-9999-cursor-opaque')?.status).toBe('FAILURE'); + }); + + test('replaying history for a null cursor fails start-from-now', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ events: [occurrence()] })] + }); + const check = checks.get('sep-9999-cursor-null-starts-from-now'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('start from now'); + }); +}); + +describe('events/poll cursor lifecycle', () => { + test('a quiet poll that drops the cursor fails advancement', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-cursor-advances-when-quiet')?.status).toBe( + 'SUCCESS' + ); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult(), pollResult({ cursor: 7 })] + }); + expect(broken.get('sep-9999-cursor-advances-when-quiet')?.status).toBe( + 'FAILURE' + ); + }); + + test('a type that flips between a cursor and null warns on consistency', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult(), pollResult({ cursor: null })] + }); + const check = checks.get('sep-9999-cursor-consistency'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('branch once'); + }); + + test('rejecting a poll that omits cursor fails absent-means-null', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + // Poll order: 1 bootstraps, 2 carries the cursor forward, 3 is the one + // that omits `cursor` entirely. Only the third may fail here. + pollResponses: [ + pollResult(), + pollResult(), + { error: { code: -32602, message: 'InvalidParams: cursor required' } } + ] + }); + const check = checks.get('sep-9999-cursor-absent-equals-null'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('absent cursor means'); + }); + + test('truncated true with no fresh cursor fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult(), + pollResult(), + pollResult(), + pollResult({ truncated: true, cursor: null }) + ] + }); + const check = checks.get('sep-9999-truncated-returns-fresh-cursor'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('no fresh position'); + }); +}); + +describe('events/poll error contract', () => { + test('a poll omitting name must not return a result', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-poll-one-subscription-per-request')?.status).toBe( + 'SUCCESS' + ); + }); + + test('wrong-typed arguments answering -32011 fails the invalid-params check', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-poll-invalid-arguments')?.status).toBe('SUCCESS'); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + invalidArgsCode: -32011 + }); + const check = broken.get('sep-9999-poll-invalid-arguments'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected -32602'); + }); + + test('polling a push-only type must answer -32014, not a result', async () => { + const pushOnly = descriptor({ name: 'push.only', delivery: ['push'] }); + const broken = await checksFor(poll(), { + ...CONFORMANT, + descriptors: [descriptor(), pushOnly], + pollByName: { 'push.only': pollResult() } + }); + const check = broken.get('sep-9999-poll-mode-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('does not advertise'); + + const ok = await checksFor(poll(), { + ...CONFORMANT, + descriptors: [descriptor(), pushOnly], + pollByName: { + 'push.only': { error: { code: -32014, message: 'Unsupported' } } + } + }); + expect(ok.get('sep-9999-poll-mode-unsupported')?.status).toBe('SUCCESS'); + }); + + test('every type offering poll reports the unsupported-mode check untestable', async () => { + const checks = await checksFor(poll(), CONFORMANT); + const check = checks.get('sep-9999-poll-mode-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + }); +}); + +describe('EventOccurrence shape', () => { + test('a quiet server reports the occurrence checks untestable, never green', async () => { + const checks = await checksFor(poll(), CONFORMANT); + const check = checks.get('sep-9999-occurrence-event-id'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + }); + + test('a non-ISO timestamp fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence({ timestamp: 'last tuesday' })], + cursor: null + }) + ] + }); + expect(checks.get('sep-9999-occurrence-timestamp')?.status).toBe('FAILURE'); + }); + + test('a repeated eventId within one batch warns', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence(), occurrence()], + cursor: null + }) + ] + }); + const check = checks.get('sep-9999-occurrence-event-id-from-upstream'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('dedup'); + }); + + test('a missing data object fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence({ data: undefined })], + cursor: null + }) + ] + }); + expect(checks.get('sep-9999-occurrence-data')?.status).toBe('FAILURE'); + }); +}); diff --git a/src/scenarios/server/events/poll.ts b/src/scenarios/server/events/poll.ts new file mode 100644 index 00000000..74a8845a --- /dev/null +++ b/src/scenarios/server/events/poll.ts @@ -0,0 +1,1208 @@ +/** + * MCP Events — poll delivery, the `EventOccurrence` shape, and cursor + * lifecycle. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec excerpt lives next to its check ID in + * src/seps/sep-9999.yaml. 9999 is a placeholder SEP number; see that file's + * header. + * + * Poll is the mode a conformance harness can exercise completely. It is + * request/response, it holds no server-side state, and every poll response + * carries the cursor, so cursor advancement and `truncated` are observable + * without waiting for anything to happen upstream. Push needs a live stream + * and webhook needs a reachable callback; both are separate scenarios. + * + * The scenario drives a real quiet-period poll loop: two polls with the cursor + * from the first fed into the second. That is what makes + * `sep-9999-cursor-advances-when-quiet` gradeable against a server with no + * traffic, which is the state a conformance fixture is usually in. + * + * Nothing is hardcoded to a fixture's event names. The scenario picks the + * first descriptor advertising `poll` and works from there; when none does, + * every check reports the unmet prerequisite rather than passing vacuously. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_EXTENSION_ID, + EVENTS_POLL_METHOD, + EVENTS_SPEC_REF, + EVENTS_NOT_FOUND, + EVENTS_UNSUPPORTED, + JSONRPC_INVALID_PARAMS, + JSONRPC_METHOD_NOT_FOUND, + type EventDescriptor, + type EventOccurrence, + type EventsPollResult, + declaredEventsCapability, + eventsCheck, + eventsListAll, + eventsPoll, + occurrences, + deliveryModes, + descriptorName, + describeValue, + isObject, + isValidCursor, + isIso8601, + minimalArguments, + firstSupporting, + unknownEventName +} from './helpers'; + +const POLL_IDS = [ + 'sep-9999-poll-implemented', + 'sep-9999-poll-one-subscription-per-request', + 'sep-9999-poll-bootstraps-subscription', + 'sep-9999-poll-events-array', + 'sep-9999-poll-response-cursor', + 'sep-9999-poll-next-poll-ms', + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + 'sep-9999-poll-has-more', + 'sep-9999-poll-max-events-cap', + 'sep-9999-poll-stateless-request', + 'sep-9999-poll-errors-are-jsonrpc', + 'sep-9999-poll-invalid-arguments', + 'sep-9999-poll-mode-unsupported', + 'sep-9999-removal-poll-not-found' +] as const; + +const OCCURRENCE_IDS = [ + 'sep-9999-occurrence-event-id', + 'sep-9999-occurrence-name', + 'sep-9999-occurrence-timestamp', + 'sep-9999-occurrence-data', + 'sep-9999-occurrence-cursor-optional', + 'sep-9999-occurrence-meta-ungoverned', + 'sep-9999-occurrence-event-id-from-upstream' +] as const; + +const CURSOR_IDS = [ + 'sep-9999-cursor-opaque', + 'sep-9999-cursor-null-starts-from-now', + 'sep-9999-cursor-absent-equals-null', + 'sep-9999-cursor-consistency', + 'sep-9999-cursor-advances-when-quiet', + 'sep-9999-max-age-ms-floor', + 'sep-9999-max-age-ms-sets-truncated', + 'sep-9999-max-age-ms-ignored-when-cursor-null', + 'sep-9999-replay-ceiling-sets-truncated', + 'sep-9999-truncated-returns-fresh-cursor', + 'sep-9999-truncated-poll-never-an-error', + 'sep-9999-truncated-false-when-no-replay' +] as const; + +const ALL_IDS = [...POLL_IDS, ...OCCURRENCE_IDS, ...CURSOR_IDS]; + +/** Milliseconds of replay to request when probing the `maxAgeMs` floor. */ +const MAX_AGE_PROBE_MS = 300_000; + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); +} + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +export class EventsPollScenario implements ClientScenario { + name = 'events-poll'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: \`events/poll\` delivery, the \`EventOccurrence\` shape, and cursor lifecycle. + +**Methods**: \`events/poll\`, plus \`events/list\` to discover a poll-capable event type + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-poll-implemented\` / \`sep-9999-poll-bootstraps-subscription\` — a poll with \`cursor: null\` succeeds with no prior subscribe step +- \`sep-9999-poll-events-array\` / \`sep-9999-poll-response-cursor\` / \`sep-9999-poll-next-poll-ms\` / \`sep-9999-poll-has-more\` — the four response fields +- \`sep-9999-poll-max-events-cap\` — \`maxEvents\` caps the batch and sets \`hasMore\` when more remain +- \`sep-9999-poll-stateless-request\` — two identical polls are answerable without server-side memory of the first +- \`sep-9999-poll-errors-are-jsonrpc\` / \`sep-9999-removal-poll-not-found\` / \`sep-9999-poll-invalid-arguments\` / \`sep-9999-poll-mode-unsupported\` — the error contract +- \`sep-9999-occurrence-*\` — \`eventId\`, \`name\`, \`timestamp\`, \`data\` required; \`cursor\` and \`_meta\` optional +- \`sep-9999-cursor-*\` — opaqueness, \`null\` means start-from-now, absent means \`null\`, consistency, and advancement during quiet periods +- \`sep-9999-max-age-ms-*\` / \`sep-9999-truncated-*\` — bounding replay and signalling a gap + +**Discovery is dynamic**: a server that neither declares the capability nor implements \`events/list\` SKIPs everything; no poll-capable event type reports the poll checks as untestable. Checks that need a delivered event (the \`EventOccurrence\` shape) are untestable against a quiet server rather than passing vacuously.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + return await this.checks(conn); + } finally { + await conn.close(); + } + } + + private async checks(conn: Connection): Promise { + const { declared } = await declaredEventsCapability(conn); + const listed = await eventsListAll(conn); + + if ('error' in listed) { + // Undeclared and unimplemented is the one case that legitimately skips: + // the server simply does not do events. Every other shape is graded, + // including the undeclared-but-serving case events-discovery fails on. + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no poll-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const descriptors = listed.descriptors; + const target = firstSupporting(descriptors, 'poll'); + const name = target ? descriptorName(target) : undefined; + + if (!target || !name) { + return untestableAll( + ALL_IDS, + descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no poll-capable event type could be exercised.' + : 'No event type advertises `poll` delivery, so `events/poll` could not be exercised. Poll is optional per event type.' + ); + } + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties, so the harness cannot construct arguments it is confident the server will accept.` + ); + } + + const checks: ConformanceCheck[] = []; + + // --- The bootstrap poll ---------------------------------------------- + const first = await eventsPoll(conn, { + name, + arguments: args, + cursor: null + }); + if ('error' in first) { + const err = first.error; + checks.push( + eventsCheck( + 'sep-9999-poll-implemented', + '`events/poll` is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.', + 'FAILURE', + { + errorMessage: `Event type \`${name}\` advertises \`poll\` delivery but \`${EVENTS_POLL_METHOD}\` failed: ${err.code} ${err.message}`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-9999-poll-implemented'), + `The bootstrap \`${EVENTS_POLL_METHOD}\` for \`${name}\` failed with ${err.code} ${err.message}.` + ) + ); + return checks; + } + + const r1 = first.result; + checks.push( + eventsCheck( + 'sep-9999-poll-implemented', + '`events/poll` is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.', + 'SUCCESS' + ) + ); + checks.push( + eventsCheck( + 'sep-9999-poll-bootstraps-subscription', + 'No separate subscribe step needed — the first poll with a null cursor bootstraps the subscription.', + 'SUCCESS', + { details: { name } } + ) + ); + + checks.push(...this.responseShapeChecks(r1)); + checks.push(...this.cursorNullChecks(r1)); + + // --- Quiet-period advancement ---------------------------------------- + checks.push(...(await this.quietAdvanceChecks(conn, name, args, r1))); + + // --- maxEvents / hasMore --------------------------------------------- + checks.push(...(await this.maxEventsChecks(conn, name, args))); + + // --- maxAgeMs and truncated ------------------------------------------ + checks.push(...(await this.replayChecks(conn, name, args, r1))); + + // --- EventOccurrence shape ------------------------------------------- + checks.push(...this.occurrenceChecks(r1, target)); + + // --- Error contract --------------------------------------------------- + checks.push(...(await this.errorChecks(conn, descriptors, name, args))); + + return checks; + } + + /** The four response fields, graded off the bootstrap poll. */ + private responseShapeChecks(r: EventsPollResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + + out.push( + Array.isArray(r.events) + ? eventsCheck( + 'sep-9999-poll-events-array', + 'The poll response carries an `events` array. An empty array means nothing happened.', + 'SUCCESS', + { details: { count: occurrences(r).length } } + ) + : eventsCheck( + 'sep-9999-poll-events-array', + 'The poll response carries an `events` array. An empty array means nothing happened.', + 'FAILURE', + { + errorMessage: `\`events\` is ${describeValue(r.events)}, expected an array (empty when nothing happened).`, + details: { events: r.events } + } + ) + ); + + out.push( + isValidCursor(r.cursor) + ? eventsCheck( + 'sep-9999-poll-response-cursor', + 'The poll response carries `cursor` at the response level, the subscription position after this batch.', + 'SUCCESS', + { details: { cursor: r.cursor ?? null } } + ) + : eventsCheck( + 'sep-9999-poll-response-cursor', + 'The poll response carries `cursor` at the response level, the subscription position after this batch.', + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(r.cursor)}, expected a string, null, or absent.`, + details: { cursor: r.cursor } + } + ) + ); + + // nextPollMs is the field the 197c32b4 rename introduced. A server still + // emitting nextPollSeconds is the single most likely failure here, so name + // it rather than reporting a generic absence. + const legacy = 'nextPollSeconds' in r; + if (typeof r.nextPollMs === 'number' && Number.isFinite(r.nextPollMs)) { + out.push( + eventsCheck( + 'sep-9999-poll-next-poll-ms', + '`nextPollMs` allows the server to dynamically adjust polling frequency.', + 'SUCCESS', + { details: { nextPollMs: r.nextPollMs } } + ) + ); + } else { + out.push( + eventsCheck( + 'sep-9999-poll-next-poll-ms', + '`nextPollMs` allows the server to dynamically adjust polling frequency.', + 'WARNING', + { + errorMessage: legacy + ? 'Response carries `nextPollSeconds`, the pre-rename field name. Spec commit `197c32b4` (2026-05-10) renamed the duration fields to `nextPollMs`.' + : `\`nextPollMs\` is ${describeValue(r.nextPollMs)}, expected a number of milliseconds.`, + details: { + nextPollMs: r.nextPollMs, + nextPollSeconds: r.nextPollSeconds + } + } + ) + ); + } + + out.push( + r.hasMore === undefined || typeof r.hasMore === 'boolean' + ? eventsCheck( + 'sep-9999-poll-has-more', + '`hasMore` indicates whether additional events are available beyond the returned batch.', + 'SUCCESS', + { details: { hasMore: r.hasMore ?? false } } + ) + : eventsCheck( + 'sep-9999-poll-has-more', + '`hasMore` indicates whether additional events are available beyond the returned batch.', + 'FAILURE', + { + errorMessage: `\`hasMore\` is ${describeValue(r.hasMore)}, expected a boolean.`, + details: { hasMore: r.hasMore } + } + ) + ); + + // `nextPollMs` is ignored when `hasMore` is true, which is only observable + // on a response that actually sets it. + out.push( + r.hasMore === true + ? eventsCheck( + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + '`nextPollMs` is ignored when `hasMore` is `true`.', + 'SUCCESS', + { details: { hasMore: true, nextPollMs: r.nextPollMs } } + ) + : untestableCheck( + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + '`nextPollMs` is ignored when `hasMore` is `true`.', + 'Server has no backlog, so no response set `hasMore: true` and the interaction between the two fields could not be observed.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** What a `cursor: null` bootstrap poll establishes on its own. */ + private cursorNullChecks(r: EventsPollResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const events = occurrences(r); + + out.push( + events.length === 0 + ? eventsCheck( + 'sep-9999-cursor-null-starts-from-now', + 'Passing `cursor: null` means "start from now." No historical events are replayed.', + 'SUCCESS', + { details: { returned: 0 } } + ) + : eventsCheck( + 'sep-9999-cursor-null-starts-from-now', + 'Passing `cursor: null` means "start from now." No historical events are replayed.', + 'FAILURE', + { + errorMessage: `A poll with \`cursor: null\` returned ${events.length} event(s); "start from now" replays nothing.`, + details: { count: events.length } + } + ) + ); + + out.push( + isValidCursor(r.cursor) + ? eventsCheck( + 'sep-9999-cursor-opaque', + 'Cursors are opaque strings managed by the server, representing a position in the event stream.', + 'SUCCESS', + { + details: { + cursorType: + r.cursor === null || r.cursor === undefined + ? 'null' + : 'string' + } + } + ) + : eventsCheck( + 'sep-9999-cursor-opaque', + 'Cursors are opaque strings managed by the server, representing a position in the event stream.', + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(r.cursor)}; a cursor is an opaque string (or null when the type has no replay).`, + details: { cursor: r.cursor } + } + ) + ); + + return out; + } + + /** + * Poll twice with the cursor from the first response and check the server + * answers the second without being told anything it was not told the first + * time. + * + * This grades three rows at once: the cursor advances (or stays put + * legitimately) during a quiet period, the request is self-contained, and an + * omitted `cursor` field is accepted as `null`. + */ + private async quietAdvanceChecks( + conn: Connection, + name: string, + args: Record, + first: EventsPollResult + ): Promise { + const out: ConformanceCheck[] = []; + const cursor = first.cursor; + const noReplay = cursor === null || cursor === undefined; + + const second = await eventsPoll(conn, { + name, + arguments: args, + ...(noReplay ? {} : { cursor }) + }); + + if ('error' in second) { + const reason = `A follow-up \`${EVENTS_POLL_METHOD}\` carrying the cursor from the first response failed: ${second.error.code} ${second.error.message}`; + out.push( + eventsCheck( + 'sep-9999-poll-stateless-request', + 'Each poll request is self-contained: the server does not need to remember previous poll requests to answer them.', + 'FAILURE', + { errorMessage: reason, details: { code: second.error.code } } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-cursor-advances-when-quiet', + 'sep-9999-cursor-consistency' + ], + reason + ) + ); + out.push( + ...untestableAll( + ['sep-9999-cursor-absent-equals-null'], + reason, + 'FAILURE' + ) + ); + return out; + } + + const r2 = second.result; + + out.push( + eventsCheck( + 'sep-9999-poll-stateless-request', + 'Each poll request is self-contained: the server does not need to remember previous poll requests to answer them.', + 'SUCCESS', + { details: { secondPollAccepted: true } } + ) + ); + + // Cursor advancement during a quiet period. A server with no traffic may + // legitimately return the same cursor — the requirement is that the + // response carries one at all, so the client's persisted position does not + // go stale. An event type with no replay carries null both times, which is + // equally conformant. + out.push( + isValidCursor(r2.cursor) + ? eventsCheck( + 'sep-9999-cursor-advances-when-quiet', + "Every poll response carries `cursor`, including when `events: []`, so the client's persisted cursor advances during quiet periods.", + 'SUCCESS', + { + details: { + first: cursor ?? null, + second: r2.cursor ?? null, + advanced: (r2.cursor ?? null) !== (cursor ?? null) + } + } + ) + : eventsCheck( + 'sep-9999-cursor-advances-when-quiet', + "Every poll response carries `cursor`, including when `events: []`, so the client's persisted cursor advances during quiet periods.", + 'FAILURE', + { + errorMessage: `Follow-up poll returned \`cursor\` as ${describeValue(r2.cursor)}; a quiet poll must still carry a position.`, + details: { cursor: r2.cursor } + } + ) + ); + + // Consistency: a type that returns a cursor once returns one always. + const firstNull = cursor === null || cursor === undefined; + const secondNull = r2.cursor === null || r2.cursor === undefined; + out.push( + firstNull === secondNull + ? eventsCheck( + 'sep-9999-cursor-consistency', + 'An event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.', + 'SUCCESS', + { details: { replaySupported: !firstNull } } + ) + : eventsCheck( + 'sep-9999-cursor-consistency', + 'An event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.', + 'WARNING', + { + errorMessage: `Event type \`${name}\` returned ${firstNull ? 'null' : 'a cursor'} then ${secondNull ? 'null' : 'a cursor'}; clients branch once at subscribe time on this.`, + details: { first: cursor ?? null, second: r2.cursor ?? null } + } + ) + ); + + // Absent means null: the second poll omitted `cursor` entirely when the + // type has no replay, and the server answered anyway. + out.push( + noReplay + ? eventsCheck( + 'sep-9999-cursor-absent-equals-null', + 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`; a receiver MUST NOT fail because it is missing.', + 'SUCCESS', + { details: { omittedCursorAccepted: true } } + ) + : await this.absentCursorCheck(conn, name, args) + ); + + return out; + } + + /** Probe "absent means null" directly by omitting the field. */ + private async absentCursorCheck( + conn: Connection, + name: string, + args: Record + ): Promise { + const id = 'sep-9999-cursor-absent-equals-null'; + const description = + 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`; a receiver MUST NOT fail because it is missing.'; + const probe = await eventsPoll(conn, { name, arguments: args }); + if ('error' in probe) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll omitting \`cursor\` was rejected with ${probe.error.code} ${probe.error.message}; an absent cursor means "start from now", not a malformed request.`, + details: { code: probe.error.code, message: probe.error.message } + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { returned: occurrences(probe.result).length } + }); + } + + /** `maxEvents` caps the batch; `hasMore` reports whether more remain. */ + private async maxEventsChecks( + conn: Connection, + name: string, + args: Record + ): Promise { + const id = 'sep-9999-poll-max-events-cap'; + const description = + '`maxEvents` is an optional cap on the number of events returned. If more are available, the server returns a partial batch with an intermediate cursor and sets `hasMore: true`.'; + + const probe = await eventsPoll(conn, { + name, + arguments: args, + cursor: null, + maxEvents: 1 + }); + + if ('error' in probe) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll carrying \`maxEvents: 1\` was rejected with ${probe.error.code} ${probe.error.message}; \`maxEvents\` is an optional request field, not an error.`, + details: { code: probe.error.code, message: probe.error.message } + }) + ]; + } + + const returned = occurrences(probe.result).length; + if (returned > 1) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`maxEvents: 1\` returned ${returned} events.`, + details: { returned } + }) + ]; + } + + return [ + eventsCheck(id, description, 'SUCCESS', { + details: { returned, hasMore: probe.result.hasMore ?? false } + }) + ]; + } + + /** + * `maxAgeMs` and `truncated`. + * + * Against a quiet server the floor never advances past the cursor, so the + * two `truncated`-setting rows cannot be driven to true. What is gradeable + * everywhere: the field is accepted, it is ignored when the cursor is null, + * `truncated` is a boolean rather than an error, and when it is true the + * response still carries a fresh cursor. + */ + private async replayChecks( + conn: Connection, + name: string, + args: Record, + bootstrap: EventsPollResult + ): Promise { + const out: ConformanceCheck[] = []; + const noReplay = + bootstrap.cursor === null || bootstrap.cursor === undefined; + + const withMaxAge = await eventsPoll(conn, { + name, + arguments: args, + cursor: bootstrap.cursor ?? null, + maxAgeMs: MAX_AGE_PROBE_MS + }); + + if ('error' in withMaxAge) { + const reason = `A poll carrying \`maxAgeMs\` was rejected with ${withMaxAge.error.code} ${withMaxAge.error.message}.`; + out.push( + eventsCheck( + 'sep-9999-max-age-ms-floor', + 'All three modes accept an optional `maxAgeMs` alongside `cursor`; the server begins replay from whichever is later, the cursor or `now − maxAgeMs`.', + 'FAILURE', + { errorMessage: reason, details: { code: withMaxAge.error.code } } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-max-age-ms-sets-truncated', + 'sep-9999-replay-ceiling-sets-truncated', + 'sep-9999-truncated-returns-fresh-cursor', + 'sep-9999-truncated-poll-never-an-error', + 'sep-9999-truncated-false-when-no-replay', + 'sep-9999-max-age-ms-ignored-when-cursor-null' + ], + reason, + 'WARNING' + ) + ); + return out; + } + + const r = withMaxAge.result; + out.push( + eventsCheck( + 'sep-9999-max-age-ms-floor', + 'All three modes accept an optional `maxAgeMs` alongside `cursor`; the server begins replay from whichever is later, the cursor or `now − maxAgeMs`.', + 'SUCCESS', + { details: { maxAgeMs: MAX_AGE_PROBE_MS } } + ) + ); + + // truncated is a response field, never an error. A server that rejected + // the maxAgeMs poll outright already failed above. + const truncated = r.truncated; + out.push( + truncated === undefined || typeof truncated === 'boolean' + ? eventsCheck( + 'sep-9999-truncated-poll-never-an-error', + 'For poll, `truncated` appears in the result body. Never a JSON-RPC error.', + 'SUCCESS', + { details: { truncated: truncated ?? false } } + ) + : eventsCheck( + 'sep-9999-truncated-poll-never-an-error', + 'For poll, `truncated` appears in the result body. Never a JSON-RPC error.', + 'FAILURE', + { + errorMessage: `\`truncated\` is ${describeValue(truncated)}, expected a boolean.`, + details: { truncated } + } + ) + ); + + // When truncated is true the response must still carry a servable cursor. + out.push( + truncated === true + ? isValidCursor(r.cursor) && r.cursor !== null && r.cursor !== undefined + ? eventsCheck( + 'sep-9999-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'SUCCESS', + { details: { cursor: r.cursor } } + ) + : eventsCheck( + 'sep-9999-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'FAILURE', + { + errorMessage: `\`truncated: true\` was returned with \`cursor\` as ${describeValue(r.cursor)}; the client has no fresh position to persist.`, + details: { cursor: r.cursor } + } + ) + : untestableCheck( + 'sep-9999-truncated-returns-fresh-cursor', + 'sep-9999-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'No probe produced `truncated: true`, so the fresh-cursor obligation could not be observed. Driving it requires a stale cursor the harness cannot mint against an opaque cursor space.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + for (const id of [ + 'sep-9999-max-age-ms-sets-truncated', + 'sep-9999-replay-ceiling-sets-truncated' + ]) { + out.push( + untestableCheck( + id, + id, + id, + "Requires a cursor older than the `maxAgeMs` floor or the server's replay ceiling. Cursors are opaque, so the harness cannot mint a stale one, and a quiet fixture has no history to fall out of.", + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + out.push( + noReplay + ? truncated === true + ? eventsCheck( + 'sep-9999-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'WARNING', + { + errorMessage: `Event type \`${name}\` returns \`cursor: null\` (no replay) but set \`truncated: true\`; there is no position to have advanced past.`, + details: { truncated } + } + ) + : eventsCheck( + 'sep-9999-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'SUCCESS', + { details: { truncated: truncated ?? false } } + ) + : eventsCheck( + 'sep-9999-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'SKIPPED', + { + errorMessage: `Event type \`${name}\` supports replay, so this rule does not apply to it.` + } + ) + ); + + // maxAgeMs is ignored when cursor is null. Observable as "the server does + // not replay history in response to it". + const nullCursorMaxAge = await eventsPoll(conn, { + name, + arguments: args, + cursor: null, + maxAgeMs: MAX_AGE_PROBE_MS + }); + const id = 'sep-9999-max-age-ms-ignored-when-cursor-null'; + const description = + '`maxAgeMs` is ignored when `cursor` is `null` (null already means "now").'; + if ('error' in nullCursorMaxAge) { + out.push( + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`cursor: null\` and \`maxAgeMs\` was rejected with ${nullCursorMaxAge.error.code} ${nullCursorMaxAge.error.message}.`, + details: { code: nullCursorMaxAge.error.code } + }) + ); + } else { + const replayed = occurrences(nullCursorMaxAge.result).length; + out.push( + replayed === 0 + ? eventsCheck(id, description, 'SUCCESS', { details: { replayed } }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`cursor: null\` and \`maxAgeMs: ${MAX_AGE_PROBE_MS}\` replayed ${replayed} event(s); \`maxAgeMs\` is ignored when the cursor is null.`, + details: { replayed } + }) + ); + } + + return out; + } + + /** + * The `EventOccurrence` field contract. + * + * Only gradeable against a response that actually carried an event. A quiet + * fixture reports the whole group as untestable rather than passing an empty + * array through seven shape checks, which would read as green. + */ + private occurrenceChecks( + r: EventsPollResult, + descriptor: EventDescriptor + ): ConformanceCheck[] { + const events = occurrences(r); + if (events.length === 0) { + return untestableAll( + OCCURRENCE_IDS, + 'No poll returned an event, so the `EventOccurrence` shape could not be validated. The fixture needs a diagnostic event type that emits on demand.' + ); + } + + const out: ConformanceCheck[] = []; + const label = (e: EventOccurrence, i: number) => + typeof e.eventId === 'string' ? `\`${e.eventId}\`` : `events[${i}]`; + + const field = ( + id: string, + description: string, + severity: 'FAILURE' | 'WARNING', + predicate: (e: EventOccurrence) => string | undefined + ) => { + for (const [i, e] of events.entries()) { + const problem = predicate(e); + if (problem) { + out.push( + eventsCheck(id, description, severity, { + errorMessage: `${label(e, i)}: ${problem}`, + details: { occurrence: e } + }) + ); + return; + } + } + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { occurrencesChecked: events.length } + }) + ); + }; + + field( + 'sep-9999-occurrence-event-id', + '`eventId` (string) is required on every `EventOccurrence`: a stable identifier for deduplication.', + 'FAILURE', + (e) => + typeof e.eventId === 'string' && e.eventId.length > 0 + ? undefined + : `\`eventId\` is ${describeValue(e.eventId)}, expected a non-empty string.` + ); + + field( + 'sep-9999-occurrence-name', + '`name` (string) is required on every `EventOccurrence`: the event type name.', + 'FAILURE', + (e) => + typeof e.name === 'string' && e.name === descriptorName(descriptor) + ? undefined + : `\`name\` is ${JSON.stringify(e.name)}, expected \`${descriptorName(descriptor)}\`.` + ); + + field( + 'sep-9999-occurrence-timestamp', + '`timestamp` (string, ISO 8601) is required on every `EventOccurrence`: when the event occurred.', + 'FAILURE', + (e) => + isIso8601(e.timestamp) + ? undefined + : `\`timestamp\` is ${JSON.stringify(e.timestamp)}, expected an ISO 8601 instant.` + ); + + field( + 'sep-9999-occurrence-data', + "`data` (object) is required on every `EventOccurrence`: payload conforming to the event type's `payloadSchema`.", + 'FAILURE', + (e) => + isObject(e.data) + ? undefined + : `\`data\` is ${describeValue(e.data)}, expected an object.` + ); + + field( + 'sep-9999-occurrence-cursor-optional', + '`cursor` on an `EventOccurrence` is optional; poll carries the cursor at the response level.', + 'FAILURE', + (e) => + isValidCursor(e.cursor) + ? undefined + : `\`cursor\` is ${describeValue(e.cursor)}, expected a string, null, or absent.` + ); + + field( + 'sep-9999-occurrence-meta-ungoverned', + '`_meta` is reserved for protocol/extension metadata and is not governed by `payloadSchema`.', + 'WARNING', + (e) => + e._meta === undefined || isObject(e._meta) + ? undefined + : `\`_meta\` is ${describeValue(e._meta)}, expected an object when present.` + ); + + // Whether an eventId came from upstream is not observable from one run; + // what is observable is that ids are distinct within a batch, which a + // per-delivery counter would violate. + const ids = events + .map((e) => e.eventId) + .filter((v): v is string => typeof v === 'string'); + out.push( + new Set(ids).size === ids.length + ? eventsCheck( + 'sep-9999-occurrence-event-id-from-upstream', + "The server SHOULD use the upstream's stable event identifier as `eventId` so the same upstream event carries the same id across delivery paths.", + 'SUCCESS', + { details: { distinct: new Set(ids).size, total: ids.length } } + ) + : eventsCheck( + 'sep-9999-occurrence-event-id-from-upstream', + "The server SHOULD use the upstream's stable event identifier as `eventId` so the same upstream event carries the same id across delivery paths.", + 'WARNING', + { + errorMessage: + 'A single batch repeated an `eventId`, so the value cannot be an upstream-stable identifier and client-side dedup would drop distinct events.', + details: { ids } + } + ) + ); + + return out; + } + + /** The poll error contract: unknown name, bad arguments, unsupported mode. */ + private async errorChecks( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + args: Record + ): Promise { + const out: ConformanceCheck[] = []; + + // Unknown name. + const unknown = unknownEventName(); + const probe = await eventsPoll(conn, { + name: unknown, + arguments: {}, + cursor: null + }); + const notFoundDesc = + 'A poll against a name the server does not serve returns `-32011 NotFound`.'; + const jsonRpcDesc = + 'Errors are returned as a standard JSON-RPC error response for the request; there is no partial-success model.'; + + if ('error' in probe) { + out.push( + eventsCheck( + 'sep-9999-poll-errors-are-jsonrpc', + jsonRpcDesc, + 'SUCCESS', + { + details: { code: probe.error.code } + } + ) + ); + out.push( + probe.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-9999-removal-poll-not-found', + notFoundDesc, + 'SUCCESS', + { + details: { code: probe.error.code, probedName: unknown } + } + ) + : eventsCheck( + 'sep-9999-removal-poll-not-found', + notFoundDesc, + 'FAILURE', + { + errorMessage: `Unknown event name answered ${probe.error.code}, expected ${EVENTS_NOT_FOUND} NotFound.`, + details: { + code: probe.error.code, + message: probe.error.message + } + } + ) + ); + } else { + const msg = `A poll for unknown event name \`${unknown}\` returned a result instead of an error.`; + out.push( + eventsCheck( + 'sep-9999-poll-errors-are-jsonrpc', + jsonRpcDesc, + 'FAILURE', + { + errorMessage: msg, + details: { result: probe.result } + } + ) + ); + out.push( + eventsCheck( + 'sep-9999-removal-poll-not-found', + notFoundDesc, + 'FAILURE', + { + errorMessage: msg + } + ) + ); + } + + // One subscription per request: `name` identifies it, so a poll without + // one is not a request the server can answer. + const noName = await eventsPoll(conn, { arguments: {}, cursor: null }); + const oneSubDesc = + 'Each `events/poll` request carries one subscription, identified by `name`.'; + out.push( + 'error' in noName + ? noName.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck( + 'sep-9999-poll-one-subscription-per-request', + oneSubDesc, + 'SUCCESS', + { details: { code: noName.error.code } } + ) + : eventsCheck( + 'sep-9999-poll-one-subscription-per-request', + oneSubDesc, + 'WARNING', + { + errorMessage: `A poll omitting \`name\` answered ${noName.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + code: noName.error.code, + message: noName.error.message + } + } + ) + : eventsCheck( + 'sep-9999-poll-one-subscription-per-request', + oneSubDesc, + 'FAILURE', + { + errorMessage: + 'A poll omitting `name` returned a result; each request carries exactly one subscription and `name` is what identifies it.', + details: { result: noName.result } + } + ) + ); + + // Invalid arguments. Only probeable when the schema constrains something. + out.push(await this.invalidArgumentsCheck(conn, descriptors, name, args)); + + // A delivery mode the event type does not offer. + out.push(await this.unsupportedModeCheck(conn, descriptors)); + + return out; + } + + /** + * Send arguments the descriptor's `inputSchema` cannot accept. + * + * Only constructible when the schema declares a typed property: a fully open + * schema has no invalid value to send, and inventing one would grade the + * server on a rule the schema never stated. + */ + private async invalidArgumentsCheck( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + _args: Record + ): Promise { + const id = 'sep-9999-poll-invalid-arguments'; + const description = + "A poll whose `arguments` do not match the event's `inputSchema` returns `-32602 InvalidParams`."; + + const target = descriptors.find((d) => descriptorName(d) === name); + const schema = target?.inputSchema; + const props = + isObject(schema) && isObject(schema.properties) + ? schema.properties + : undefined; + const typed = props + ? Object.entries(props).find( + ([, v]) => + isObject(v) && + typeof v.type === 'string' && + ['string', 'boolean', 'integer', 'number'].includes(v.type) + ) + : undefined; + + if (!typed) { + return untestableCheck( + id, + id, + description, + `Event type \`${name}\` declares no typed \`inputSchema\` property, so no argument value can be known-invalid against it.`, + [EVENTS_SPEC_REF], + 'FAILURE' + ); + } + + const [prop, spec] = typed; + const propType = (spec as Record).type as string; + // A value of the wrong JSON type for the declared one. + const wrongValue = propType === 'string' ? 12345 : 'not-a-valid-value'; + + const probe = await eventsPoll(conn, { + name, + arguments: { [prop]: wrongValue }, + cursor: null + }); + + if (!('error' in probe)) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll sending \`${prop}: ${JSON.stringify(wrongValue)}\` against a declared \`${propType}\` returned a result instead of ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { property: prop, declaredType: propType, sent: wrongValue } + }); + } + + return probe.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck(id, description, 'SUCCESS', { + details: { property: prop, declaredType: propType } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Arguments violating \`inputSchema\` answered ${probe.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + property: prop, + declaredType: propType, + code: probe.error.code, + message: probe.error.message + } + }); + } + + /** Poll an event type whose `delivery` omits `poll`. */ + private async unsupportedModeCheck( + conn: Connection, + descriptors: EventDescriptor[] + ): Promise { + const id = 'sep-9999-poll-mode-unsupported'; + const description = + 'A poll against an event type whose `delivery` does not list `poll` returns `-32014 Unsupported`.'; + + const nonPoll = descriptors.find( + (d) => + descriptorName(d) !== undefined && !deliveryModes(d).includes('poll') + ); + if (!nonPoll) { + return untestableCheck( + id, + id, + description, + 'Every event type the server offers advertises `poll` delivery, so there is no event type to probe the unsupported-mode path with.', + [EVENTS_SPEC_REF], + 'FAILURE' + ); + } + + const name = descriptorName(nonPoll)!; + const probe = await eventsPoll(conn, { + name, + arguments: minimalArguments(nonPoll) ?? {}, + cursor: null + }); + + if (!('error' in probe)) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Event type \`${name}\` does not advertise \`poll\` delivery (\`delivery\` is ${JSON.stringify(nonPoll.delivery)}) but \`${EVENTS_POLL_METHOD}\` returned a result.`, + details: { name, delivery: deliveryModes(nonPoll) } + }); + } + + return probe.error.code === EVENTS_UNSUPPORTED + ? eventsCheck(id, description, 'SUCCESS', { + details: { name, code: probe.error.code, data: probe.error.data } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Polling \`${name}\`, which does not offer poll delivery, answered ${probe.error.code}, expected ${EVENTS_UNSUPPORTED} Unsupported.`, + details: { + name, + delivery: deliveryModes(nonPoll), + code: probe.error.code, + message: probe.error.message + } + }); + } +} + +/** Exported for the negative tests, which assert the full emitted set. */ +export const EVENTS_POLL_CHECK_IDS = ALL_IDS; diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml new file mode 100644 index 00000000..15497836 --- /dev/null +++ b/src/seps/sep-9999.yaml @@ -0,0 +1,537 @@ +# spec_source: modelcontextprotocol/experimental-ext-triggers-events@main docs/design-sketch-proposal.md +# extracted: 2026-09-15 +# +# ############################################################################ +# # 9999 IS A PLACEHOLDER. MCP Events has no SEP number yet. # +# # Renaming it is a release blocker before this file is upstreamed. # +# ############################################################################ +# +# SEP numbers are PR numbers in modelcontextprotocol/modelcontextprotocol, +# and no events PR has been opened there — the work lives in the separate +# experimental-ext-triggers-events repository. Both ends enforce a number: +# src/traceability/index.ts:174 filters the directory on `/^sep-\d+\.yaml$/` +# and line 41 filters emitted check IDs on `/^sep-\d+-/`, so a file named +# `events.yaml` is silently dropped from the manifest rather than rejected; +# src/new-sep/index.ts:163 rejects any argument that is not a positive +# integer from the other side. There is no honest name that works. +# +# 9999 is far from the live range (~2663), so a collision is implausible. +# The rename is mechanical and wide: this filename, the `sep:` field below, +# every `sep-9999-` id here, and the same ids in +# src/scenarios/server/events/. +# +# This must not merge under the placeholder. plan.modelcontextprotocol.io +# reads the manifest built from `main`, so a draft PR is invisible to it, but +# a merge would publish SEP 9999 as though it were real and a later rename +# would then have to retract it. The rename is therefore a merge blocker, not +# a follow-up. +# +# The reservation question is open with the WG in #triggers-events-wg. Peter +# is the person to ask and is away the week of 2026-09-14 for AGNTCon, so the +# answer is not expected before the scenarios land. +# +# provenance: extracted against the merged design sketch on `main`, which +# landed 2026-09-08. Before that merge the document was a moving PR head and +# not worth scoring against. Two spec commits land inside this extraction: +# `197c32b4` (2026-05-10) renamed the duration fields to `nextPollMs` and +# `maxAgeMs`, and `28ec35e9` (2026-09-04) added event-type removal and +# breaking-change termination. +# +# The sketch is a design document rather than a spec diff, so there is no +# `docs/specification/draft/*.mdx` to point `new-sep` at — `specPathToUrl` +# (src/new-sep/index.ts:19) hard-requires that prefix, so `--spec-path` does +# not help either. The file was scaffolded with `--spec-url`, which +# short-circuits the GitHub lookup. There are no section anchors to cite, so +# `spec_url` names the document itself and rows carry no per-row `url`. +# Expect one more pass once the text becomes a spec PR and gains anchors. +# +# coverage: 131 declared checks plus 30 excluded rows. +# +# A keyword sweep of the source finds 144 RFC 2119 keyword occurrences across +# 119 sentences: 34 MUST, 9 MUST NOT, 3 REQUIRED, 56 SHOULD, 7 SHOULD NOT +# (109 normative), plus 34 MAY and 1 OPTIONAL. Counting MUST NOT inside MUST +# and SHOULD NOT inside SHOULD, as the handoff table does, the same sweep +# reads as 43 / 3 / 63; the two counts agree. +# +# Only 46 of the declared rows quote a sentence carrying a keyword. The other +# 85 are shape requirements the document states declaratively — the +# `EventOccurrence` field table, the error-code table, the `events/list` +# descriptor fields, the response payloads. They are normative and directly +# testable, they are simply not written with a keyword, which is why the row +# count exceeds the sentence count in the sweep. One keyword sentence also +# frequently carries several distinct obligations (the Standard Webhooks +# signature sentence, the subscription-key sentence), which pushes the same +# way. +# +# severity rule, since the usual "follow the keyword" mapping is undefined +# for a shape row: MUST / MUST NOT / REQUIRED -> FAILURE (30 rows), SHOULD / +# SHOULD NOT -> WARNING (16 rows). For a shape row, a field the document +# marks Required in a schema table, or a wire fact a client cannot work +# without (an error code, a method existing at all), is FAILURE; everything +# else drawn from an example payload is WARNING. +# +# Pure MAY and OPTIONAL sentences get no row at all, per the handoff +# disposition table. The five that initially got one are demoted to +# `excluded:` at the foot of this file rather than deleted, so the sweep +# stays auditable against the document. +# +# excluded rather than declared: client-SDK and host obligations (poll floor, +# payload sanitization, policy evaluation), receiver obligations the harness +# implements rather than grades, server-SDK implementation guidance from the +# two "SDK Guidance" sections, and the WAF/deployment profile advisories. +# None is a server wire obligation, so declaring them would inflate the +# denominator with rows no server-side scenario could ever emit. +# +# deliberately not declared here: pagination semantics for `events/list`. The +# sketch defers them ("same semantics as tools/list etc."), so the obligation +# is the base protocol's and belongs to the core pagination scenarios. +# `sep-9999-list-pagination` declares only the part this document adds, which +# is that `events/list` participates in that scheme at all. +# +# An `events/poll` against an unknown name is stated twice, once under Error +# Codes and once as the poll consequence of an event type having been +# removed. Only `sep-9999-removal-poll-not-found` is declared for it; a +# second row would double-count one probe. +# +# `## Key Design Decisions`, `## Open Questions` and `## What Is NOT in v1` +# are rationale rather than requirements and produce no rows. The design +# table restates the webhook secret, TTL and TLS rules already declared above +# it. +# +# backing_scenarios: server ClientScenarios under src/scenarios/server/events/ +# emit the check IDs below (a row is "tested" once a scenario emits its ID; +# see src/traceability/). Phase 1 ships two of the five and emits 45 of the +# 131 rows. The remaining 86 report as untested until the later scenarios +# land, which is the manifest working as intended rather than a gap here: +# discovery.ts (events-discovery), 12 rows — the two capability rows, the +# two sep-9999-list-* rows, the six sep-9999-descriptor-* rows, and +# sep-9999-error-not-found plus sep-9999-error-server-range, both graded +# off one unknown-name probe. +# poll.ts (events-poll), 33 rows — the sep-9999-poll-* rows, the +# sep-9999-occurrence-* rows, the sep-9999-cursor-* / +# sep-9999-max-age-* / sep-9999-truncated-* rows reachable through poll, +# and sep-9999-removal-poll-not-found, which is the poll leg of the +# removal rules. +# +# The other four sep-9999-removal-* rows and the remaining sep-9999-error-* +# codes need a server that can be made to remove an event type mid-run, or a +# delivery mode this phase does not drive, so they wait on the push and +# webhook scenarios. +# Still to come, and reported untested in the manifest until they land: +# push.ts (events-push) — the sep-9999-stream-* rows. +# webhook.ts (events-webhook) — the sep-9999-subscribe-*, sep-9999-ttl-* +# and sep-9999-unsubscribe-* rows. +# webhook-delivery.ts (events-webhook-delivery) — the sep-9999-delivery-*, +# sep-9999-verification-*, sep-9999-ssrf-* and sep-9999-envelope-* rows. +# These need a callback URL the server under test can reach over https, +# which localhost cannot satisfy: the SSRF rules declared below require a +# conformant server to refuse it. +# +# measured against mcpkit, 2026-09-15, examples/events/kitchen-sink at main: +# events-discovery 8/11, events-poll 18/28. Six divergences, each a real gap +# rather than something the suite should accommodate. Do not soften these +# rows to make mcpkit pass; catching them is the point. +# +# sep-9999-capability-events-object — the server answers `events/list` but +# declares no `capabilities.events`, so a client that reads capabilities +# before calling never finds the surface. Not previously tracked. This is +# also why the scenario asks before it skips: a plain "undeclared optional +# capability" SKIP reports this server as a clean run. +# sep-9999-poll-next-poll-ms — `nextPollSeconds` persists (events.go:504) +# after `197c32b4` renamed it, and `maxAge` is still in seconds +# (events.go:544). mcpkit's own wire_shape_test.go:44 asserts the +# pre-rename name. Gap G31. +# sep-9999-descriptor-delivery-subset — `events.topology` advertises an +# empty `delivery`, where the document requires a non-empty subset of +# poll/push/webhook. That source is the substitute for the missing +# `notifications/events/list_changed`. Gap G34. +# sep-9999-descriptor-input-schema — no descriptor carries `inputSchema`, +# which also makes `sep-9999-poll-invalid-arguments` untestable, since +# there is no declared constraint left to violate. Not previously tracked. +# sep-9999-poll-events-array — `events` is omitted rather than returned as +# an empty array when nothing happened. Not previously tracked. +# sep-9999-poll-mode-unsupported — `events/poll` answers for an event type +# whose `delivery` does not list `poll`, where the document requires +# `-32014 Unsupported`. Not previously tracked. +# +# The removal and termination rules (gap G30) are declared here but not yet +# exercised: `RemoveSource` (registry.go:182) fires topology only and +# `UnsupportedData` (errors.go:90) has no `reason` field, but driving that +# path needs a server that can drop an event type mid-run, which waits on +# the push scenario. +sep: 9999 +spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md +requirements: + # === Capability Declaration === + - check: sep-9999-capability-events-object + text: 'Servers advertise event support in their capabilities: `{"capabilities": {"events": {"listChanged": true}}}`.' + - check: sep-9999-capability-list-changed-flag + text: 'The `listChanged` flag under `capabilities.events` advertises that the server sends `notifications/events/list_changed`.' + + # === Listing Available Events === + - check: sep-9999-list-implemented + text: '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.' + - check: sep-9999-list-pagination + text: '`nextCursor` is "present when more pages are available; same semantics as tools/list etc."' + - check: sep-9999-descriptor-name + text: 'Each event descriptor carries a `name` identifying the event type.' + - check: sep-9999-descriptor-description + text: 'Each event descriptor carries a `description` of when the event fires.' + - check: sep-9999-descriptor-delivery-subset + text: '`delivery` lists the delivery modes this event type supports — any non-empty subset of `"poll"`, `"push"`, `"webhook"`. No mode is mandatory.' + - check: sep-9999-descriptor-input-schema + text: '`inputSchema` is a JSON Schema describing valid subscription arguments — these may include filters (which narrow the event stream), transforms (which modify payloads), or other server-defined configuration.' + - check: sep-9999-descriptor-payload-schema + text: '`payloadSchema` describes the shape of `data` in delivered events.' + - check: sep-9999-descriptor-meta + text: '`_meta` on an event descriptor is optional; same semantics as on Tool/Resource/Prompt.' + - check: sep-9999-schema-evolution-additive + text: "Servers SHOULD evolve an event type's `inputSchema` and `payloadSchema` additively for the lifetime of its `name`: new optional fields MAY be added; existing fields SHOULD NOT be removed, renamed, or retyped; enums SHOULD NOT be narrowed; and `inputSchema` SHOULD NOT be tightened such that previously accepted `arguments` become invalid." + - check: sep-9999-breaking-change-new-name + text: 'A breaking change SHOULD instead be published under a new event name, served alongside the old one for a migration period, after which the old name is removed.' + + # === Dynamic Event Types === + - check: sep-9999-list-changed-notification + text: 'If the set of available event types, or the descriptor of any of them (`description`, `delivery`, `inputSchema`, `payloadSchema`), changes at runtime, the server sends a `notifications/events/list_changed` notification.' + + # === Event Type Removal and Breaking Changes (spec commit 28ec35e9) === + - check: sep-9999-removal-terminates-subscriptions + text: "The server SHOULD end them using each mode's termination signal: `notifications/events/terminated` on push streams and a `terminated` envelope to webhook subscriptions." + - check: sep-9999-removal-error-not-found + text: 'The `error` is `-32011 NotFound` with `data: {"kind": "event"}` when the type was removed.' + - check: sep-9999-removal-error-schema-changed + text: 'The `error` is `-32014 Unsupported` with `data: {"feature": "payloadSchema" | "inputSchema", "reason": "schema_changed"}` when it was changed in place — not `-32012 Forbidden`, since the principal''s access is unchanged.' + - check: sep-9999-removal-additive-no-terminate + text: 'Purely additive changes MUST NOT terminate subscriptions.' + - check: sep-9999-removal-poll-not-found + text: 'Poll holds no server-side subscription to terminate: a poll against a removed name already returns `-32011 NotFound`.' + + # === Error Codes === + - check: sep-9999-error-invalid-params + text: "`-32602 InvalidParams` — request is statically invalid: arguments don't match the event's inputSchema, the callback `delivery.url` is malformed or non-`https`, or `delivery.secret` is not a valid `whsec_` value." + - check: sep-9999-error-not-found + text: '`-32011 NotFound` — a referenced entity does not exist: an unknown event name, or no subscription matching the key on `events/unsubscribe`.' + - check: sep-9999-error-forbidden + text: '`-32012 Forbidden` — the authenticated principal is not permitted for this event/arguments combination, or its access was revoked.' + - check: sep-9999-error-resource-exhausted + text: '`-32013 ResourceExhausted` — a server-imposed limit or quota was reached. `data.limit` names it (e.g. `"subscriptions"`).' + - check: sep-9999-error-unsupported + text: '`-32014 Unsupported` — the request is well-formed but a requested capability or option is not supported here, e.g. a delivery mode the event type does not offer. `data` identifies it (e.g. `{"feature": "deliveryMode", "value": "push"}`).' + - check: sep-9999-error-callback-endpoint-error + text: '`-32015 CallbackEndpointError` — a client-supplied callback endpoint failed verification or could not be reached (webhook mode only). `data.reason` is one of the `lastError` categories.' + - check: sep-9999-error-server-range + text: 'These are general-purpose codes carried in the JSON-RPC implementation-defined server range `[-32000, -32099]`.' + + # === Poll-Based Delivery === + - check: sep-9999-poll-implemented + text: 'Poll (`events/poll`) is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.' + - check: sep-9999-poll-one-subscription-per-request + text: 'Each `events/poll` request carries one subscription. A client with multiple subscriptions runs one poll loop per subscription.' + - check: sep-9999-poll-bootstraps-subscription + text: 'No separate subscribe step needed — the first poll with a null cursor bootstraps the subscription. Server holds no protocol-required state.' + - check: sep-9999-poll-events-array + text: 'The poll response carries an `events` array of `EventOccurrence` entries. Empty `events` array means nothing happened — this is the common case and should be cheap.' + - check: sep-9999-poll-response-cursor + text: 'The poll response carries `cursor` at the response level, the subscription position after this batch.' + - check: sep-9999-poll-next-poll-ms + text: '`nextPollMs` allows the server to dynamically adjust polling frequency (e.g., back off when rate-limited upstream, speed up when activity is detected).' + - check: sep-9999-poll-next-poll-ms-ignored-when-has-more + text: '`nextPollMs` is ignored when `hasMore` is `true`.' + - check: sep-9999-poll-has-more + text: '`hasMore` indicates whether additional events are available beyond the returned batch. When `true`, the client should poll again immediately with the updated cursor. When `false`, the client waits `nextPollMs` before the next poll.' + - check: sep-9999-poll-max-events-cap + text: '`maxEvents` is an optional cap on the number of events returned. If more events are available than the limit, the server returns a partial batch with an intermediate cursor and sets `hasMore: true`. If omitted, the server uses its own default limit.' + - check: sep-9999-poll-stateless-request + text: 'Each poll request is self-contained: the client provides the event name, arguments, and cursor. The server does not need to "remember" previous poll requests to answer them.' + - check: sep-9999-poll-errors-are-jsonrpc + text: 'Errors (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`) are returned as a standard JSON-RPC error response for the request — there is no partial-success model since each request carries one subscription.' + - check: sep-9999-poll-invalid-arguments + text: "A poll whose `arguments` do not match the event's `inputSchema` returns `-32602 InvalidParams`." + - check: sep-9999-poll-mode-unsupported + text: 'A poll against an event type whose `delivery` does not list `"poll"` returns `-32014 Unsupported` identifying the unsupported delivery mode.' + + # === EventOccurrence schema === + - check: sep-9999-occurrence-event-id + text: '`eventId` (string) is required on every `EventOccurrence`: a stable identifier for deduplication.' + - check: sep-9999-occurrence-name + text: '`name` (string) is required on every `EventOccurrence`: the event type name.' + - check: sep-9999-occurrence-timestamp + text: '`timestamp` (string, ISO 8601) is required on every `EventOccurrence`: when the event occurred.' + - check: sep-9999-occurrence-data + text: "`data` (object) is required on every `EventOccurrence`: payload conforming to the event type's `payloadSchema`." + - check: sep-9999-occurrence-cursor-optional + text: '`cursor` (string | null) is not required: subscription position after this event (push/webhook only; poll carries cursor at the response level).' + - check: sep-9999-occurrence-meta-ungoverned + text: '`_meta` is reserved for protocol/extension metadata, consistent with `_meta` on other MCP types. Not governed by `payloadSchema`.' + - check: sep-9999-occurrence-event-id-from-upstream + text: '`eventId` is server-assigned: when the upstream source provides a stable event identifier, the server SHOULD use that value as `eventId` so that the same upstream event surfaced via multiple paths carries the same `eventId` and dedup works.' + + # === Cursor Lifecycle === + - check: sep-9999-cursor-opaque + text: 'Cursors are opaque strings managed by the server. They represent a position in the event stream. `cursor` is opaque to the client.' + - check: sep-9999-cursor-null-starts-from-now + text: 'Passing `cursor: null` in any mode means "start from now." The server returns a fresh cursor representing the current position (or `null` if the event type does not support replay). No historical events are replayed.' + - check: sep-9999-cursor-absent-equals-null + text: 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`: a sender MAY omit the field instead of writing `null`, and a receiver MUST NOT fail because it is missing.' + - check: sep-9999-cursor-consistency + text: 'Servers SHOULD be consistent: an event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`, so clients can branch once at subscribe time rather than per delivery.' + - check: sep-9999-cursor-advances-when-quiet + text: "The client's persisted cursor must advance during quiet periods. Poll covers this inherently (every response carries `cursor`, including when `events: []`)." + - check: sep-9999-max-age-ms-floor + text: 'All three modes accept an optional `maxAgeMs` (integer milliseconds) alongside `cursor`. When present, the server begins replay from whichever is later: the supplied `cursor`, or `now − maxAgeMs`.' + - check: sep-9999-max-age-ms-sets-truncated + text: 'If the floor advances past the cursor, the server SHOULD set `truncated: true` on the first response (poll result / `notifications/events/active` / webhook subscribe response) so the client knows older events were skipped.' + - check: sep-9999-max-age-ms-ignored-when-cursor-null + text: '`maxAgeMs` is ignored when `cursor` is `null` (null already means "now") and for event types that do not support replay.' + - check: sep-9999-replay-ceiling-sets-truncated + text: 'Servers MAY also apply their own replay ceiling independent of `maxAgeMs` and MUST signal it via `truncated: true` when they do.' + - check: sep-9999-truncated-returns-fresh-cursor + text: 'In all cases the server resets to a position it can serve from, returns that position as the fresh `cursor` alongside `truncated: true`, and continues — the client never reconnects or re-subscribes in response.' + - check: sep-9999-truncated-poll-never-an-error + text: 'For poll, `truncated` appears in the result body: `{events:[], cursor:, truncated:true, hasMore, nextPollMs}`. Never a JSON-RPC error.' + - check: sep-9999-truncated-false-when-no-replay + text: 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.' + + # === Push-Based Delivery === + - check: sep-9999-stream-implemented + text: 'Push delivery uses a long-lived `events/stream` request — one per subscription. The request is a standard JSON-RPC request with an `id`, which enables cancellation via `notifications/cancelled` and is echoed in every notification for routing.' + - check: sep-9999-stream-error-before-open + text: 'If the subscription is invalid (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`), the server responds immediately with a JSON-RPC error and no stream is opened.' + - check: sep-9999-stream-active-confirmation + text: 'Otherwise the server confirms the subscription with `notifications/events/active {cursor, truncated, _meta.subscriptionId}` and delivers events as notifications.' + - check: sep-9999-stream-subscription-id-meta + text: 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]` so a client with multiple concurrent streams can route notifications to the correct stream.' + - check: sep-9999-stream-event-notification + text: 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.' + - check: sep-9999-stream-error-is-recoverable + text: '`notifications/events/error` reports a recoverable failure (e.g., a single upstream fetch failed); the subscription remains active and the server retries and resumes.' + - check: sep-9999-stream-terminated-ends-subscription + text: 'Only `notifications/events/terminated` ends the subscription.' + - check: sep-9999-stream-gap-resends-active + text: "A gap (e.g., the cursor fell outside the upstream's retention window) is not an error — the server sends a fresh `notifications/events/active {cursor:, truncated:true, _meta.subscriptionId}` and continues delivering." + - check: sep-9999-stream-heartbeat-required + text: 'The server MUST send periodic keepalive messages on the push stream so the client can distinguish "nothing to send" from "connection is dead."' + - check: sep-9999-stream-heartbeat-carries-cursor + text: "The heartbeat is `notifications/events/heartbeat` — `cursor` is the position the server has checked up to, so the client's persisted cursor advances even when no events match; it is `null` for event types that do not support replay." + - check: sep-9999-stream-heartbeat-interval + text: 'The server SHOULD send a heartbeat at least every 30 seconds.' + - check: sep-9999-stream-heartbeat-not-sse-comment + text: 'On Streamable HTTP this is sent as an SSE `data:` frame; the SSE comment form (`: keepalive`) is not used since it cannot carry cursor state.' + - check: sep-9999-stream-final-result-shape + text: 'The `StreamEventsResult` is an empty typed result (`{"_meta": {}}`). It carries no information — it satisfies JSON-RPC''s requirement that every request gets a response.' + - check: sep-9999-stream-final-result-timing + text: 'It is sent whenever the server can write a final frame: on Streamable HTTP only when the server initiates the close; on stdio the server MAY send it, and clients MUST NOT depend on receiving it.' + - check: sep-9999-stream-cancel-stops-delivery + text: 'In both cases, the server MUST stop delivering events and release any associated resources.' + - check: sep-9999-stream-exempt-from-concurrency-cap + text: 'Server SDKs MUST exempt `events/stream` from any general request-concurrency cap, since each push subscription is a long-lived request that never completes until cancelled.' + - check: sep-9999-stream-carries-only-event-notifications + text: "The `events/stream` response carries only `notifications/events/*` messages. This stream carries only this subscription's event notifications; it is not a general server-to-client channel." + + # === Webhook: subscribing === + - check: sep-9999-subscribe-webhook-only + text: '`events/subscribe` is ONLY used for webhook delivery. Poll and push do not need it.' + - check: sep-9999-subscribe-secret-required + text: '`delivery.secret` is REQUIRED. The client supplies the HMAC signing secret; the server never generates one.' + - check: sep-9999-subscribe-secret-format + text: 'The value MUST be a Standard Webhooks symmetric secret: the literal prefix `whsec_` followed by base64 of 24–64 random bytes.' + - check: sep-9999-subscribe-secret-rejected + text: 'Servers MUST reject a `delivery.secret` that is not `whsec_` followed by base64 decoding to 24–64 bytes (per Standard Webhooks) with `InvalidParams`.' + - check: sep-9999-subscribe-url-https-required + text: 'Callback URLs MUST use `https://`.' + - check: sep-9999-subscribe-url-non-https-rejected + text: 'Servers MUST reject `events/subscribe` with a non-`https` `delivery.url` (`-32602 InvalidParams`).' + - check: sep-9999-subscribe-auth-required + text: '`events/subscribe` and `events/unsubscribe` MUST be called with an authenticated principal; servers MUST reject calls without an authorized principal with `-32012 Forbidden`.' + - check: sep-9999-subscribe-key-composition + text: "The subscription key is `(principal, delivery.url, name, arguments)`, where `principal` is the server's canonical identifier for the authenticated subject. `arguments` is compared by canonical-JSON equality. There is no client-generated `id`." + - check: sep-9999-subscribe-key-immutable + text: "All four components are immutable for the subscription's lifetime: a subscribe call with a different value for any of them addresses a different subscription." + - check: sep-9999-subscribe-idempotent-upsert + text: '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields. If a subscription with the same scoped key exists, the server resets the TTL and updates mutable fields in place.' + - check: sep-9999-subscribe-id-derived + text: 'The server computes a deterministic `id` over the key (e.g., a truncated SHA-256 of the canonical key serialization) and returns it in the subscribe response. It is stable across refreshes and server restarts.' + - check: sep-9999-subscribe-id-not-an-input + text: "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method." + - check: sep-9999-subscribe-refresh-replaces-secret + text: 'On an idempotent subscribe against an existing key, `delivery.secret` is replaced. To rotate, the client supplies a new value on refresh.' + - check: sep-9999-subscribe-refresh-reactivates + text: 'On an idempotent subscribe against an existing key, `active` is set to `true`. If delivery had been suspended due to repeated failures, the server resumes retrying pending events.' + - check: sep-9999-subscribe-response-cursor + text: "The subscribe response carries `cursor`, a safe-to-persist watermark that advances the client's cursor even if no events arrive before next refresh." + - check: sep-9999-subscribe-response-truncated + text: 'The subscribe response carries `truncated`, true if delivery started later than the supplied cursor (retention window, `maxAgeMs` floor, or server-side ceiling).' + - check: sep-9999-subscribe-cross-tenant-isolation + text: 'Because the key includes `principal` and `delivery.url`, two distinct tenants subscribing to the same `(name, arguments)` get distinct subscriptions.' + + # === Webhook: TTL negotiation === + - check: sep-9999-ttl-refresh-before-lte-suggestion + text: '`refreshBefore` (response) is the grant. It SHOULD be less than or equal to the suggestion.' + - check: sep-9999-ttl-no-rejection-path + text: 'Clamping in either direction is self-announcing — the client reads the granted `refreshBefore` and schedules its refresh loop from that, so a clamped grant is not an error and there is no rejection path for TTL values.' + - check: sep-9999-ttl-null-only-when-requested + text: 'A server MUST NOT return `null` unless the client suggested `ttlMs: null` — no expiry exceeds every finite suggestion.' + - check: sep-9999-ttl-omitted-means-default + text: 'Omitting `ttlMs` means "server default." An explicit `ttlMs: null` requests a subscription with no expiry.' + - check: sep-9999-ttl-long-grant-retained + text: 'A server granting long or no-expiry TTLs MUST retain subscriptions for the lifetime it granted, including across restarts.' + - check: sep-9999-ttl-no-expiry-persisted + text: 'The server MUST persist no-expiry subscriptions across restarts, because a client that never refreshes will never detect (or repair) a silently dropped one.' + - check: sep-9999-ttl-no-expiry-gc-terminated + text: 'The server MAY drop a no-expiry subscription after sustained delivery failure (server-defined window), and SHOULD attempt a `terminated` envelope when it does.' + + # === Webhook: unsubscribing === + - check: sep-9999-unsubscribe-by-key + text: '`events/unsubscribe` is eager cleanup; the server looks the subscription up by the same compound key used for idempotent upsert on subscribe.' + - check: sep-9999-unsubscribe-unknown-not-found + text: 'No subscription matching the key on `events/unsubscribe` returns `-32011 NotFound`.' + + # === Webhook: delivery status === + - check: sep-9999-delivery-status-last-error-category + text: '`lastError` MUST be a server-generated category string — one of `connection_refused`, `timeout`, `tls_error`, `http_4xx`, `http_5xx`, `challenge_failed` — and MUST NOT include raw response bodies.' + + # === Webhook: delivery mechanics === + - check: sep-9999-delivery-post-json + text: 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.' + - check: sep-9999-delivery-standard-webhooks-headers + text: 'Every delivery MUST include `webhook-id` (the `eventId` for event deliveries; `msg__` for control envelopes), `webhook-timestamp` (Unix seconds), and `webhook-signature`.' + - check: sep-9999-delivery-subscription-id-header + text: 'In addition to the Standard Webhooks headers, deliveries MUST include `X-MCP-Subscription-Id` (the subscription `id`) so the receiver can select the correct secret without parsing the body. This is the only MCP-specific header.' + - check: sep-9999-delivery-signature-formula + text: 'The signature is `HMAC-SHA256(secret, webhook-id + "." + webhook-timestamp + "." + body)` encoded as base64 with a `v1,` prefix, where `body` is the raw HTTP request body bytes exactly as received and `secret` is the base64-decoded bytes of the value after the `whsec_` prefix.' + - check: sep-9999-delivery-retry-regenerates-signature + text: "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window." + - check: sep-9999-delivery-dual-sign-on-rotation + text: 'The server SHOULD dual-sign deliveries with both the old and new secrets for a short grace window so in-flight deliveries verify under either.' + - check: sep-9999-delivery-body-size + text: 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with Payload Minimality.' + - check: sep-9999-delivery-413-non-retryable + text: 'Receivers and intermediaries MAY reject larger bodies with `413 Payload Too Large`; servers MUST treat `413` as a non-retryable failure for that event.' + - check: sep-9999-delivery-410-non-retryable + text: 'A receiver that intentionally rejects a delivery and does not want it retried responds `410 Gone`; the server MUST treat it as non-retryable.' + - check: sep-9999-delivery-retries-bounded + text: 'Retries are bounded: servers SHOULD cap both the attempt count and the elapsed retry window (for example, 3–5 attempts spread over no more than 10–15 minutes).' + + # === Webhook: SSRF and endpoint verification === + - check: sep-9999-ssrf-validate-callback-url + text: 'The server MUST validate callback URLs.' + - check: sep-9999-ssrf-reject-non-routable + text: 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA IPv4 and IPv6 Special-Purpose Address Registries unless explicitly configured to allow them.' + - check: sep-9999-ssrf-validate-at-delivery-time + text: 'To prevent DNS rebinding, this validation MUST be performed at delivery time, not only at subscribe time: the server resolves the hostname, checks the resolved IP against the blocklist, and connects directly to that validated IP.' + - check: sep-9999-ssrf-no-redirects + text: 'Webhook delivery requests MUST NOT follow HTTP redirects, since a redirect can target an internal address that bypasses the blocklist.' + - check: sep-9999-verification-required-before-delivery + text: "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed, by one of: a verification handshake, a server-configured allowlist, prior out-of-band verification, or a receiver-published well-known document." + - check: sep-9999-verification-challenge-echo + text: 'Before activating, the server POSTs a `verification` control envelope carrying a single-use, short-lived `challenge` nonce (signed and headed like any delivery), and the endpoint proves intent by echoing the nonce in a `2xx` body (`{"challenge":""}`), which the server compares in constant time.' + - check: sep-9999-verification-failure-error + text: 'A reachable endpoint that fails to echo yields `-32015 CallbackEndpointError` with `data.reason: "challenge_failed"`; an unreachable one yields the same code with the relevant connection-failure category.' + - check: sep-9999-verification-cached-per-principal-url + text: "Verification is cached per `(principal, url)`: a successful handshake, allowlist hit, or well-known match covers that principal's subscriptions to that URL across refreshes and `arguments`, so varying `arguments` cannot multiply verification POSTs at a victim and one principal's verification never waives the challenge for another." + - check: sep-9999-verification-persisted-for-no-expiry + text: 'A server that persists no-expiry subscriptions across restarts MUST persist their verification status alongside them.' + - check: sep-9999-verification-uses-ssrf-hardened-path + text: 'The verification POST MUST use the same SSRF-hardened path as deliveries (delivery-time IP validation, no redirects).' + - check: sep-9999-verification-no-raw-endpoint-responses + text: 'Failures surface only via the `lastError` category `challenge_failed`, never raw endpoint responses.' + - check: sep-9999-server-identity-key-discovery + text: 'The verifying public key MUST be discovered from an origin the client already authenticates, and never from the challenge or delivery body, which the attacker controls.' + + # === Webhook: control envelopes === + - check: sep-9999-envelope-type-discriminator + text: 'A body with a top-level `type` field is a control envelope; a body without one is an `EventOccurrence`.' + - check: sep-9999-envelope-signed-like-deliveries + text: 'Control envelopes are signed and headed exactly like event deliveries (Standard Webhooks headers + `X-MCP-Subscription-Id`); the body carries a `type` discriminator instead of `eventId`/`data`.' + - check: sep-9999-envelope-webhook-id-format + text: '`webhook-id` for control envelopes is a per-message identifier of the form `msg__` so receivers can dedup retries.' + - check: sep-9999-envelope-gap + text: 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes. The client persists `cursor` and treats it as `truncated: true`.' + - check: sep-9999-envelope-terminated + text: 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended (e.g., authorization revoked). The subscription no longer exists server-side.' + + # === Authorization === + - check: sep-9999-authz-subscribe-time + text: 'When the caller is authenticated, the server MUST verify the principal has permission to subscribe to the requested event type with the given arguments.' + - check: sep-9999-authz-delivery-time-reverify + text: 'The server SHOULD periodically re-verify permissions.' + + # === Payload handling (server side) === + - check: sep-9999-payload-minimality + text: 'Servers SHOULD keep event payloads minimal — enough to identify and triage the event, not the full content.' + + # ========================================================================== + # Excluded rows. Each carries an RFC 2119 keyword in the source but is not a + # server wire obligation, so no server-side scenario can emit a check for it. + # Kept here so the keyword sweep stays auditable against the document. + # ========================================================================== + + # --- Client / host obligations --- + - text: 'The client SHOULD re-call `events/list` to refresh its event type registry.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'The client SHOULD poll again immediately (ignoring `nextPollMs`) to drain the backlog.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Clients SHOULD apply a configurable floor (default 1000 ms) to guard against a misbehaving server inducing a tight loop.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Clients SHOULD NOT apply their default request timeout to `events/stream`; a client that has received neither an event nor a heartbeat for more than twice the heartbeat interval SHOULD treat the stream as dead and reconnect with its cursor.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Unless granted no expiry, the client MUST re-call `events/subscribe` with the same subscription key before `refreshBefore` to keep the subscription alive.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Even with no expiry, clients SHOULD still re-call `events/subscribe` (and `events/list`) occasionally.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'A client receiving `cursor: null` MUST NOT attempt to persist or replay from it; on reconnect/resubscribe it sends `cursor: null` (start from now).' + excluded: client-side obligation; needs a client-conformance scenario set + - text: '`truncated: true` always implies a possible gap; clients SHOULD treat it as such and persist the fresh cursor.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: "Client SDKs SHOULD generate the secret on the application's behalf rather than expose an interface that encourages hand-picked values, from a CSPRNG by default." + excluded: client-SDK guidance; not observable from the server under test + - text: 'Event payloads MUST be treated with the same caution as tool results. Clients SHOULD sanitize or sandbox event payloads before presenting them to an LLM.' + excluded: host obligation; not observable from the server under test + - text: 'Clients SHOULD support policy evaluation between event receipt and action execution, and SHOULD respect server-declared governance metadata.' + excluded: host obligation (Enterprise Governance); not observable from the server under test + - text: 'The client SDK SHOULD remove the subscription and notify the application when a termination signal arrives.' + excluded: client-SDK guidance; not observable from the server under test + + # --- Receiver (callback endpoint) obligations --- + - text: 'The receiver MUST verify the signature before processing, SHOULD reject deliveries where `webhook-timestamp` is more than 5 minutes old, and SHOULD deduplicate on `webhook-id`.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'Receivers MUST compute the HMAC over the raw body, never over a re-serialized JSON object.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'The endpoint MUST make `cursor` and `eventId` available to the consuming client, and MUST forward control envelopes by the same channel it forwards events.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'The endpoint SHOULD NOT return `2xx` until the event has been durably persisted or forwarded, and SHOULD respond quickly.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'A receiver that gets a delivery for an `id` it has not yet been told to route SHOULD return a retryable status (`503` or `425 Too Early`).' + excluded: receiver obligation; the harness implements this rather than grading it + + # --- Server-SDK implementation guidance (not wire-observable) --- + - text: "The SDK SHOULD enable poll by default so it's available unless the author opts out." + excluded: server-SDK guidance; indistinguishable on the wire from an author opting in + - text: 'SDKs SHOULD refuse a no-expiry cap unless the author has wired up durable storage.' + excluded: server-SDK guidance; a construction-time policy with no wire signal + - text: 'SDKs SHOULD provide an in-memory ring buffer that retains a bounded window of emitted events per event type.' + excluded: server-SDK guidance; an internal structure with no wire signal + - text: 'Servers SHOULD enforce the subscription limit (`ResourceExhausted`) before invoking `on_subscribe`, so a rejected subscription never provisions upstream resources.' + excluded: server-SDK ordering guidance; the harness cannot observe hook invocation order + - text: "The lease window is SDK-configurable and SHOULD default to a small multiple of the server's typical `nextPollMs`. Server authors SHOULD write `on_subscribe` to be idempotent." + excluded: server-SDK guidance; an internal lease table with no wire signal + - text: 'Authors SHOULD prefer a `check()` function that queries the upstream over emit-only for upstreams offering a durable cursor.' + excluded: server-authoring guidance; a design recommendation with no wire signal + - text: 'On HTTP/1.1 each stream is a TCP connection, so SDKs SHOULD prefer HTTP/2 when many subscriptions are active.' + excluded: transport guidance; not a protocol conformance obligation + + # --- Deployment profile advisories --- + - text: 'A WAF MAY drop requests missing any required header as a cheap pre-filter; WAF rules SHOULD NOT rely on UA matching as a security control. Servers SHOULD document their egress ranges out-of-band.' + excluded: deployment advisory addressed to operators, not to an implementation + + # --- Pure MAY / OPTIONAL (handoff disposition: no row) --- + - text: 'A server MAY return `cursor: null` in any delivery (poll result, push `notifications/events/event` and `notifications/events/active`, webhook payload) when the event type does not support replay.' + excluded: pure MAY; kept as context for sep-9999-cursor-consistency, which carries the testable half + - text: 'The one sanctioned exception is a server-side floor: a server MAY clamp an impractically short suggestion up to its minimum TTL to protect itself from refresh storms.' + excluded: pure MAY; the testable half is sep-9999-ttl-refresh-before-lte-suggestion, which this sentence excepts + - text: 'The header MAY contain multiple space-delimited signatures (`v1, v1,`) during secret rotation; the receiver accepts if any verifies.' + excluded: pure MAY, and the obligation it creates falls on the receiver + - text: 'After repeated failures (server-defined threshold), the server MAY suspend delivery (`deliveryStatus.active: false`).' + excluded: pure MAY; suspension is observable only through the OPTIONAL deliveryStatus object + - text: '`deliveryStatus` is OPTIONAL — servers MAY omit it entirely. The `events/subscribe` response MAY include a `deliveryStatus` object when refreshing an existing subscription.' + excluded: OPTIONAL by its own wording; a server omitting deliveryStatus entirely is conformant diff --git a/src/types.ts b/src/types.ts index ebe75a27..970f97e4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,7 +103,13 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/auth/dpop', 'io.modelcontextprotocol/auth/wif', 'io.modelcontextprotocol/tasks', - 'io.modelcontextprotocol/skills' + 'io.modelcontextprotocol/skills', + // MCP Events declares its capability top-level as `capabilities.events`, + // not inside `capabilities.extensions`. This id is a suite-selection key + // only — it keeps the Events scenarios off the `--spec-version` timeline + // (see `matchesSpecVersion`), and is never a path into the capability + // object. See src/scenarios/server/events/helpers.ts. + 'io.modelcontextprotocol/events' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number];