diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9821850..02edce0 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -62,8 +62,8 @@ jobs: with: tool: cargo-binstall - - name: Install cf-integration 0.2.1 binary - run: cargo binstall cf-integration@0.2.1 --no-confirm + - name: Install cf-integration 0.3.1 binary + run: cargo binstall cf-integration@0.3.1 --no-confirm - name: Prepare the exact data-plane image env: diff --git a/Makefile b/Makefile index 240dc62..24a8251 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,8 @@ conformance: ## Run strict modern MCP conformance against the committed data-pla "$(CF_INTEGRATION)" conformance run \ --client-era modern \ --server-era modern \ - --lane external-data-plane \ + --lane external \ + --standalone \ --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ --output-dir "$(CF_INTEGRATION_DIR)/reports" @@ -63,7 +64,8 @@ conformance-bless: ## Run strict modern conformance and atomically update its ba "$(CF_INTEGRATION)" conformance run \ --client-era modern \ --server-era modern \ - --lane external-data-plane \ + --lane external \ + --standalone \ --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ --output-dir "$(CF_INTEGRATION_DIR)/reports" \ --bless diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index d3cc32b..87f68ed 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -76,11 +76,15 @@ Comment exactly `/conformance` on a pull request to run the **Conformance** Actions workflow. Only repository owners, members, and collaborators can start it. The workflow acknowledges the command, tests the pull request merge commit, and reports the final result back to the pull request. It runs the modern client -and modern server eras through the external dataplane. Selecting that lane also -runs the fixture-direct server leg and the scoped external-dataplane client leg: +and modern server eras through the external dataplane in standalone mode. This +starts Redis, the dataplane, nginx, and the official fixture without the control +plane. The harness discovers the fixture's tools, resources, templates, and +prompts and publishes their routes and actual tool schemas through the +dataplane serializer. Selecting that lane also runs the fixture-direct server +leg and the scoped external-dataplane client leg: ```bash -cargo binstall cf-integration@0.1.0 --no-confirm +cargo binstall cf-integration@0.3.1 --no-confirm make conformance ``` @@ -106,3 +110,14 @@ use `fixture-direct.yml` and `external-data-plane.yml`; scoped client findings use `client/external-data-plane.yml`. Operational failures are always failures and cannot be blessed. Runtime checkouts, logs, results, and reports remain beneath `.integration/`. + +A strict baseline match means the observed findings have not changed; it does +not mean complete protocol coverage. Inspect the report's individual checks +before blessing changes. In particular, the external dataplane deliberately +rejects catalog list methods owned by the control plane. Scenarios that need +`tools/list` for setup cannot exercise the subsequent schema or parameter-header +checks through this lane; `NotObserved` findings record missing coverage, not +successful validation. The pinned fixture also lacks an `x-mcp-header`-annotated +tool for the custom-header server scenario. Parameter-header validation remains +covered by the active in-repo tests described above. Keep these setup gaps +distinct from observed protocol failures when interpreting conformance results. diff --git a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml index 2cb8db3..84efe29 100644 --- a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml @@ -1 +1,141 @@ -findings: [] +findings: +- scenario: caching + check: sep-2549-cache-scope-valid + name: CacheScopeValid + status: FAILURE +- scenario: caching + check: sep-2549-prompts-list-caching-hints + name: PromptsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-list-caching-hints + name: ResourcesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-templates-list-caching-hints + name: ResourcesTemplatesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-tools-list-caching-hints + name: ToolsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-ttl-non-negative + name: TtlNonNegative + status: FAILURE +- scenario: completion-complete + check: completion-complete + name: CompletionComplete + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-custom-setup + name: HttpCustomHeaderServerValidationSetup + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-decode-base64 + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-invalid-param-chars + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-param-mismatch + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-validate-param-match + name: NotObserved + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-accepts-whitespace-header-value + name: ServerAcceptsWhitespaceHeaderValue + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-standard-setup + name: HttpHeaderValidationSetup + status: FAILURE +- scenario: input-required-result-basic-elicitation + check: sep-2322-elicitation-incomplete + name: InputRequiredResultElicitationIncomplete + status: FAILURE +- scenario: input-required-result-basic-list-roots + check: sep-2322-list-roots-incomplete + name: InputRequiredResultListRootsIncomplete + status: FAILURE +- scenario: input-required-result-basic-sampling + check: sep-2322-sampling-incomplete + name: InputRequiredResultSamplingIncomplete + status: FAILURE +- scenario: input-required-result-capability-check + check: sep-2322-respect-client-capabilities + name: RespectClientCapabilities + status: FAILURE +- scenario: input-required-result-missing-input-response + check: sep-2322-missing-response-rerequests + name: InputRequiredResultMissingResponseRerequests + status: WARNING +- scenario: input-required-result-multi-round + check: sep-2322-multi-round-r1 + name: InputRequiredResultMultiRoundR1 + status: FAILURE +- scenario: input-required-result-multiple-input-requests + check: sep-2322-multiple-inputs-incomplete + name: InputRequiredResultMultipleInputsIncomplete + status: FAILURE +- scenario: input-required-result-non-tool-request + check: sep-2322-non-tool-incomplete + name: InputRequiredResultNonToolIncomplete + status: FAILURE +- scenario: input-required-result-request-state + check: sep-2322-request-state-incomplete + name: InputRequiredResultRequestStateIncomplete + status: FAILURE +- scenario: input-required-result-result-type + check: sep-2322-result-type-included + name: ResultTypeIncluded + status: FAILURE +- scenario: input-required-result-tampered-state + check: sep-2322-reject-tampered-state + name: RejectTamperedState + status: FAILURE +- scenario: json-schema-2020-12 + check: json-schema-2020-12-error + name: JsonSchema2020_12Error + status: FAILURE +- scenario: prompts-list + check: prompts-list + name: PromptsList + status: FAILURE +- scenario: resources-list + check: resources-list + name: ResourcesList + status: FAILURE +- scenario: resources-templates-read + check: resources-templates-read + name: ResourcesTemplateRead + status: FAILURE +- scenario: sep-2164-resource-not-found + check: sep-2164-data-uri + name: ResourcesNotFoundDataUri + status: WARNING +- scenario: server-stateless + check: sep-2575-discover-capabilities-match-handlers + name: DiscoverCapabilitiesMatchHandlers + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-no-independent-requests-on-stream + name: HttpServerNoIndependentRequestsOnStream + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-declares-prompts-in-discover + name: ServerDeclaresPromptsInDiscover + status: FAILURE +- scenario: tools-call-with-progress + check: tools-call-with-progress + name: ToolsCallWithProgress + status: FAILURE +- scenario: tools-list + check: tools-list + name: ToolsList + status: FAILURE