feat: conformance for two revisions (T3)

.github/workflows/ci.yml

@@ -48,6 +48,8 @@ jobs:
         run: sbt -v test
       - name: Client conformance tests (2025-11-25)
         run: sbt -v "clientConformance/conformance client --requirements 2025-11-25"
+      - name: Client conformance tests (2026-07-28)
+        run: sbt -v "clientConformance/conformance client --requirements 2026-07-28"
       - name: Server conformance tests (2025-11-25)
         run: sbt -v "serverConformance/conformance server --requirements 2025-11-25"
       - name: Server conformance tests (2026-07-28)

build.sbt

@@ -251,7 +251,7 @@ lazy val serverConformance = (project in file("server-conformance"))
       val baseline = specVersion
         .map(v => rootDir / s"conformance-baseline-$v.yml")
         .filter(_.exists)
-        .getOrElse(rootDir / "conformance-baseline.yml")
+        .getOrElse(rootDir / "conformance-baseline-2025-11-25.yml")
         .getAbsolutePath
       val log = streams.value.log
 
@@ -327,8 +327,23 @@ lazy val clientConformance = (project in file("client-conformance"))
       val baseDir = baseDirectory.value
       val rootDir = (LocalRootProject / baseDirectory).value
       val wrapper = (baseDir / "bin" / "chimp-conformance-client").getAbsolutePath
+      val specVersion = args
+        .sliding(2)
+        .collectFirst {
+          case Seq("--requirements", v) => v
+          case Seq("--spec-version", v) => v
+        }
+        .orElse(args.collectFirst {
+          case a if a.startsWith("--requirements=") => a.stripPrefix("--requirements=")
+          case a if a.startsWith("--spec-version=") => a.stripPrefix("--spec-version=")
+        })
+      val baseline = specVersion
+        .map(v => rootDir / s"conformance-baseline-$v.yml")
+        .filter(_.exists)
+        .getOrElse(rootDir / "conformance-baseline-2025-11-25.yml")
+        .getAbsolutePath
       val cmd = List("npx", s"@modelcontextprotocol/conformance@$conformanceHarnessV") ++ args ++
-        List("--command", wrapper, "--expected-failures", (rootDir / "conformance-baseline.yml").getAbsolutePath)
+        List("--command", wrapper, "--expected-failures", baseline)
       val rc = Process(cmd, rootDir, "CHIMP_CLIENT_CONFORMANCE_JAR" -> jar.getAbsolutePath).!
       if (rc != 0) sys.error(s"conformance harness exited with code $rc")
     }

client-conformance/README.md

@@ -22,17 +22,18 @@ The sbt task pins the harness to the version set in `conformanceHarnessV` in [`b
 
 ## How to run
 
-Run the full requirement set:
+Run a full requirement set (CI runs both revisions):
 
 ```bash
 sbt 'clientConformance/conformance client --requirements 2025-11-25'
+sbt 'clientConformance/conformance client --requirements 2026-07-28'
 ```
 
-Run one scenario during development:
+Run one scenario during development (pass the spec version):
 
 ```bash
 sbt 'clientConformance/conformance client --scenario initialize --spec-version 2025-11-25'
-sbt 'clientConformance/conformance client --scenario tools_call --spec-version 2025-11-25'
+sbt 'clientConformance/conformance client --scenario tools_call --spec-version 2026-07-28'
 ```
 
 The sbt task downloads `@modelcontextprotocol/conformance` with npm; `npx` must be available on the PATH.
@@ -44,8 +45,10 @@ with `McpClient`, return exit code 0 on success. Once it passes, remove the entr
 
 ## The baseline file
 
-[`conformance-baseline.yml`](../conformance-baseline.yml) lists scenarios that are known to fail today. The harness
-uses it like this:
+Each spec revision keeps its own baseline, selected from the run's `--requirements` (or `--spec-version`):
+[`conformance-baseline-2025-11-25.yml`](../conformance-baseline-2025-11-25.yml) and
+[`conformance-baseline-2026-07-28.yml`](../conformance-baseline-2026-07-28.yml). Each file lists the scenarios (in its
+`server:` and `client:` sections) known to fail today. The harness uses the chosen file like this:
 
 | Scenario result | In baseline? | Exit code | Meaning                               |
 |-----------------|--------------|-----------|---------------------------------------|

conformance-baseline-2026-07-28.yml

@@ -1,6 +1,6 @@
-# Expected server-conformance failures for spec version 2026-07-28.
+# Expected conformance failures for spec version 2026-07-28 (server and client).
 # These need protocol features that land in later tickets; remove each entry once its
-# scenario passes. See server-conformance/README.md for how the baseline is used.
+# scenario passes. See server-conformance/README.md and client-conformance/README.md.
 server:
   # multi-round-trip requests (tasks) — T5, https://github.com/softwaremill/chimp/issues/243
   - tasks-capability-negotiation
@@ -31,3 +31,43 @@ server:
   - sep-2164-resource-not-found
   - input-required-result-missing-input-response
   - input-required-result-ignore-extra-params
+client:
+  # OAuth client flows — chimp's client has no full OAuth support yet
+  - auth/authorization-server-migration
+  - auth/basic-cimd
+  - auth/client-credentials-basic
+  - auth/client-credentials-jwt
+  - auth/dpop
+  - auth/dpop-nonce
+  - auth/enterprise-managed-authorization
+  - auth/iss-normalized
+  - auth/iss-not-advertised
+  - auth/iss-supported
+  - auth/iss-supported-missing
+  - auth/iss-unexpected
+  - auth/iss-wrong-issuer
+  - auth/metadata-default
+  - auth/metadata-issuer-mismatch
+  - auth/metadata-var1
+  - auth/metadata-var2
+  - auth/metadata-var3
+  - auth/offline-access-not-supported
+  - auth/offline-access-scope
+  - auth/pre-registration
+  - auth/scope-from-scopes-supported
+  - auth/scope-from-www-authenticate
+  - auth/scope-omitted-when-undefined
+  - auth/scope-retry-limit
+  - auth/scope-step-up
+  - auth/token-endpoint-auth-basic
+  - auth/token-endpoint-auth-none
+  - auth/token-endpoint-auth-post
+  - auth/wif-jwt-bearer
+  # modern (2026-07-28) client era: stateless lifecycle, request _meta, HTTP headers — T10-T12, #248/#249/#250
+  - tools_call
+  - http-custom-headers
+  - http-invalid-tool-headers
+  - json-schema-2020-12-preservation
+  - json-schema-ref-no-deref
+  - request-metadata
+  - sep-2322-client-request-state

server-conformance/README.md

@@ -17,17 +17,18 @@ The sbt task pins the harness to the version set in `conformanceHarnessV` in [`b
 
 ## How to run
 
-Run the full requirement set:
+Run a full requirement set (CI runs both revisions):
 
 ```bash
 sbt 'serverConformance/conformance server --requirements 2025-11-25'
+sbt 'serverConformance/conformance server --requirements 2026-07-28'
 ```
 
-Run one scenario during development:
+Run one scenario during development (pass the spec version):
 
 ```bash
 sbt 'serverConformance/conformance server --scenario ping --spec-version 2025-11-25'
-sbt 'serverConformance/conformance server --scenario server-initialize --spec-version 2025-11-25'
+sbt 'serverConformance/conformance server --scenario server-discover --spec-version 2026-07-28'
 ```
 
 The sbt task downloads `@modelcontextprotocol/conformance` with npm; `npx` must be available on the PATH.
@@ -41,10 +42,10 @@ and what result shape it wants. Remove the corresponding entry from the baseline
 
 ## The baseline file
 
-[`conformance-baseline.yml`](../conformance-baseline.yml) lists scenarios that are known to fail today. When a
-`conformance-baseline-<version>.yml` exists for the run's spec version (e.g.
-[`conformance-baseline-2026-07-28.yml`](../conformance-baseline-2026-07-28.yml) for `--requirements 2026-07-28`), the sbt
-task uses it instead — each spec revision keeps its own baseline. The harness uses the chosen file like this:
+Each spec revision keeps its own baseline, selected from the run's `--requirements` (or `--spec-version`):
+[`conformance-baseline-2025-11-25.yml`](../conformance-baseline-2025-11-25.yml) and
+[`conformance-baseline-2026-07-28.yml`](../conformance-baseline-2026-07-28.yml). Each file lists the scenarios (in its
+`server:` and `client:` sections) known to fail today. The harness uses the chosen file like this:
 
 | Scenario result | In baseline? | Exit code | Meaning                               |
 |-----------------|--------------|-----------|---------------------------------------|