diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e733400b..0af5f333 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,8 @@ jobs: run: sbt -v test - name: Client conformance tests (2025-11-25) run: sbt -v "clientConformance/conformance client --requirements 2025-11-25" + - name: Client conformance tests (2026-07-28) + run: sbt -v "clientConformance/conformance client --requirements 2026-07-28" - name: Server conformance tests (2025-11-25) run: sbt -v "serverConformance/conformance server --requirements 2025-11-25" - name: Server conformance tests (2026-07-28) diff --git a/build.sbt b/build.sbt index fdd2a057..e2d71c3e 100644 --- a/build.sbt +++ b/build.sbt @@ -251,7 +251,7 @@ lazy val serverConformance = (project in file("server-conformance")) val baseline = specVersion .map(v => rootDir / s"conformance-baseline-$v.yml") .filter(_.exists) - .getOrElse(rootDir / "conformance-baseline.yml") + .getOrElse(rootDir / "conformance-baseline-2025-11-25.yml") .getAbsolutePath val log = streams.value.log @@ -327,8 +327,23 @@ lazy val clientConformance = (project in file("client-conformance")) val baseDir = baseDirectory.value val rootDir = (LocalRootProject / baseDirectory).value val wrapper = (baseDir / "bin" / "chimp-conformance-client").getAbsolutePath + val specVersion = args + .sliding(2) + .collectFirst { + case Seq("--requirements", v) => v + case Seq("--spec-version", v) => v + } + .orElse(args.collectFirst { + case a if a.startsWith("--requirements=") => a.stripPrefix("--requirements=") + case a if a.startsWith("--spec-version=") => a.stripPrefix("--spec-version=") + }) + val baseline = specVersion + .map(v => rootDir / s"conformance-baseline-$v.yml") + .filter(_.exists) + .getOrElse(rootDir / "conformance-baseline-2025-11-25.yml") + .getAbsolutePath val cmd = List("npx", s"@modelcontextprotocol/conformance@$conformanceHarnessV") ++ args ++ - List("--command", wrapper, "--expected-failures", (rootDir / "conformance-baseline.yml").getAbsolutePath) + List("--command", wrapper, "--expected-failures", baseline) val rc = Process(cmd, rootDir, "CHIMP_CLIENT_CONFORMANCE_JAR" -> jar.getAbsolutePath).! if (rc != 0) sys.error(s"conformance harness exited with code $rc") } diff --git a/client-conformance/README.md b/client-conformance/README.md index 4e0250eb..525d1a3f 100644 --- a/client-conformance/README.md +++ b/client-conformance/README.md @@ -22,17 +22,18 @@ The sbt task pins the harness to the version set in `conformanceHarnessV` in [`b ## How to run -Run the full requirement set: +Run a full requirement set (CI runs both revisions): ```bash sbt 'clientConformance/conformance client --requirements 2025-11-25' +sbt 'clientConformance/conformance client --requirements 2026-07-28' ``` -Run one scenario during development: +Run one scenario during development (pass the spec version): ```bash sbt 'clientConformance/conformance client --scenario initialize --spec-version 2025-11-25' -sbt 'clientConformance/conformance client --scenario tools_call --spec-version 2025-11-25' +sbt 'clientConformance/conformance client --scenario tools_call --spec-version 2026-07-28' ``` The sbt task downloads `@modelcontextprotocol/conformance` with npm; `npx` must be available on the PATH. @@ -44,8 +45,10 @@ with `McpClient`, return exit code 0 on success. Once it passes, remove the entr ## The baseline file -[`conformance-baseline.yml`](../conformance-baseline.yml) lists scenarios that are known to fail today. The harness -uses it like this: +Each spec revision keeps its own baseline, selected from the run's `--requirements` (or `--spec-version`): +[`conformance-baseline-2025-11-25.yml`](../conformance-baseline-2025-11-25.yml) and +[`conformance-baseline-2026-07-28.yml`](../conformance-baseline-2026-07-28.yml). Each file lists the scenarios (in its +`server:` and `client:` sections) known to fail today. The harness uses the chosen file like this: | Scenario result | In baseline? | Exit code | Meaning | |-----------------|--------------|-----------|---------------------------------------| diff --git a/conformance-baseline.yml b/conformance-baseline-2025-11-25.yml similarity index 100% rename from conformance-baseline.yml rename to conformance-baseline-2025-11-25.yml diff --git a/conformance-baseline-2026-07-28.yml b/conformance-baseline-2026-07-28.yml index d6bdc93a..5c0d8b66 100644 --- a/conformance-baseline-2026-07-28.yml +++ b/conformance-baseline-2026-07-28.yml @@ -1,6 +1,6 @@ -# Expected server-conformance failures for spec version 2026-07-28. +# Expected conformance failures for spec version 2026-07-28 (server and client). # These need protocol features that land in later tickets; remove each entry once its -# scenario passes. See server-conformance/README.md for how the baseline is used. +# scenario passes. See server-conformance/README.md and client-conformance/README.md. server: # multi-round-trip requests (tasks) — T5, https://github.com/softwaremill/chimp/issues/243 - tasks-capability-negotiation @@ -31,3 +31,43 @@ server: - sep-2164-resource-not-found - input-required-result-missing-input-response - input-required-result-ignore-extra-params +client: + # OAuth client flows — chimp's client has no full OAuth support yet + - auth/authorization-server-migration + - auth/basic-cimd + - auth/client-credentials-basic + - auth/client-credentials-jwt + - auth/dpop + - auth/dpop-nonce + - auth/enterprise-managed-authorization + - auth/iss-normalized + - auth/iss-not-advertised + - auth/iss-supported + - auth/iss-supported-missing + - auth/iss-unexpected + - auth/iss-wrong-issuer + - auth/metadata-default + - auth/metadata-issuer-mismatch + - auth/metadata-var1 + - auth/metadata-var2 + - auth/metadata-var3 + - auth/offline-access-not-supported + - auth/offline-access-scope + - auth/pre-registration + - auth/scope-from-scopes-supported + - auth/scope-from-www-authenticate + - auth/scope-omitted-when-undefined + - auth/scope-retry-limit + - auth/scope-step-up + - auth/token-endpoint-auth-basic + - auth/token-endpoint-auth-none + - auth/token-endpoint-auth-post + - auth/wif-jwt-bearer + # modern (2026-07-28) client era: stateless lifecycle, request _meta, HTTP headers — T10-T12, #248/#249/#250 + - tools_call + - http-custom-headers + - http-invalid-tool-headers + - json-schema-2020-12-preservation + - json-schema-ref-no-deref + - request-metadata + - sep-2322-client-request-state diff --git a/server-conformance/README.md b/server-conformance/README.md index b0ed7282..c44ea505 100644 --- a/server-conformance/README.md +++ b/server-conformance/README.md @@ -17,17 +17,18 @@ The sbt task pins the harness to the version set in `conformanceHarnessV` in [`b ## How to run -Run the full requirement set: +Run a full requirement set (CI runs both revisions): ```bash sbt 'serverConformance/conformance server --requirements 2025-11-25' +sbt 'serverConformance/conformance server --requirements 2026-07-28' ``` -Run one scenario during development: +Run one scenario during development (pass the spec version): ```bash sbt 'serverConformance/conformance server --scenario ping --spec-version 2025-11-25' -sbt 'serverConformance/conformance server --scenario server-initialize --spec-version 2025-11-25' +sbt 'serverConformance/conformance server --scenario server-discover --spec-version 2026-07-28' ``` The sbt task downloads `@modelcontextprotocol/conformance` with npm; `npx` must be available on the PATH. @@ -41,10 +42,10 @@ and what result shape it wants. Remove the corresponding entry from the baseline ## The baseline file -[`conformance-baseline.yml`](../conformance-baseline.yml) lists scenarios that are known to fail today. When a -`conformance-baseline-.yml` exists for the run's spec version (e.g. -[`conformance-baseline-2026-07-28.yml`](../conformance-baseline-2026-07-28.yml) for `--requirements 2026-07-28`), the sbt -task uses it instead — each spec revision keeps its own baseline. The harness uses the chosen file like this: +Each spec revision keeps its own baseline, selected from the run's `--requirements` (or `--spec-version`): +[`conformance-baseline-2025-11-25.yml`](../conformance-baseline-2025-11-25.yml) and +[`conformance-baseline-2026-07-28.yml`](../conformance-baseline-2026-07-28.yml). Each file lists the scenarios (in its +`server:` and `client:` sections) known to fail today. The harness uses the chosen file like this: | Scenario result | In baseline? | Exit code | Meaning | |-----------------|--------------|-----------|---------------------------------------|