feat(provider-tck): emit a machine-readable conformance report

.github/scripts/tck-report-summary.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Renders the conformance reports in $1 as a Markdown table, one row per suite.
+#
+# A scenario's outcome is the most severe status among its steps, the same rule the TCK's report
+# plugin documents, so a capability-gated skip counts as skipped rather than passed.
+set -euo pipefail
+
+dir="${1:?usage: tck-report-summary.sh <report dir>}"
+shopt -s nullglob
+envelopes=("$dir"/*.json)
+
+echo "## Provider conformance"
+echo
+if [ ${#envelopes[@]} -eq 0 ]; then
+  echo "No conformance reports were written."
+  exit 0
+fi
+
+echo "| Provider | Configuration | Backend | Passed | Failed | Skipped | Declared | Known deviations |"
+echo "|---|---|---|--:|--:|--:|---|--:|"
+for envelope in "${envelopes[@]}"; do
+  stream="${envelope%.json}.ndjson"
+  counts=$(jq -rs '
+    def rank: {"FAILED":5,"AMBIGUOUS":4,"UNDEFINED":3,"PENDING":2,"SKIPPED":1,"PASSED":0}[.] // 0;
+    [ .[] | .testStepFinished // empty ]
+    | group_by(.testCaseStartedId)
+    | map([ .[].testStepResult.status ] | max_by(rank))
+    | [ (map(select(. == "PASSED")) | length),
+        (map(select(rank >= 2)) | length),
+        (map(select(. == "SKIPPED")) | length) ]
+    | @tsv' "$stream")
+  read -r passed failed skipped <<<"$counts"
+  jq -r --arg p "$passed" --arg f "$failed" --arg s "$skipped" '
+    "| \(.provider.name) | \(.provider.configuration) | \(.backend.backendType) | \($p) | \($f) | \($s) | "
+    + ((.declaration.declared // []) | join(" "))
+    + " | \((.knownDeviations // []) | length) |"' "$envelope"
+done

.github/workflows/ci.yml

@@ -47,6 +47,20 @@ jobs:
         # !codequality: a module's own default-active codequality profile (flagd's runs vmlens
         # *CTest suites) is not switched off by a profile in the root pom; main enforces it anyway
         run: mvn --batch-mode --activate-profiles 'tck,!codequality' clean verify
+        env:
+          TCK_REPORT_DIR: ${{ github.workspace }}/tck-reports
+
+      - name: Summarise the conformance reports
+        if: ${{ !cancelled() }}
+        run: bash .github/scripts/tck-report-summary.sh tck-reports >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload the conformance reports
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: conformance-reports
+          path: tck-reports/
+          if-no-files-found: ignore
 
   main:
     strategy:

tools/tck/README.md

@@ -231,6 +231,56 @@ provider's pom (or pass `-Dtck.testFailureIgnore=true`):
 </properties>
 ```
 
+## The canonical set cannot be reduced
+
+`CanonicalScenarioGuard` is a JUnit test the suite selects. It fails the build when the run is set up
+to execute less than the canonical set: a feature file added to or shadowing one in `gherkin/`,
+`cucumber.filter.tags` or `cucumber.filter.name`, or selectors or glue overridden in
+`junit-platform.properties`. It checks the plan and the filter configuration before the first
+scenario, so it needs no backend. Extension scenarios are ignored.
+
+Narrow a run with `capabilities()`, whose skips are reported with a reason. To filter anyway while
+debugging, set `-Dtck.partial=true` (or `TCK_PARTIAL`); the guard then reports itself as skipped
+rather than passed.
+
+## Conformance reports
+
+Set `TCK_REPORT_DIR` (or `-Dtck.report.dir=...`) and each suite writes an envelope conforming to the
+[report schema][report-schema] and its results as a [Cucumber Messages][messages] stream. Unset means
+no report. Several suites in one JVM each write their own pair.
+
+```console
+$ TCK_REPORT_DIR=./reports mvn -Ptck -pl providers/flagd test
+$ ls reports/
+flagd-in-process.json  flagd-in-process.ndjson  flagd-rpc.json  flagd-rpc.ndjson
+```
+
+The `.ndjson` is written by Cucumber's own `MessageFormatter`, the class behind the built-in
+`message:` plugin; a plugin of its own exists only so the path can follow the run. A scenario's
+outcome is the most severe result among its steps, hooks included, which is what makes a
+capability-gated skip read as `SKIPPED`. A scenario is identified by `pickle.astNodeIds`, whose
+second entry names the `Examples` row of an outline.
+
+The envelope carries what the stream cannot:
+
+- **`provider`**: the provider's own metadata name, and the suite name as its `configuration`
+  (`ProviderTckHarness.configuration()` overrides the derived one).
+- **`sdk`**: the SDK version on the classpath at runtime.
+- **`tck`**: the implementation and `specRevision`, the open-feature/spec commit the packaged assets
+  came from, baked in from this module's POM.
+- **`backend`**: the control's description and `backendType`, which the control states through
+  `BackendControl.backendType()`: `http`, `in-process`, or `custom` for one that states nothing.
+  Always present, never inferred.
+- **`declaration`**: the declared capabilities. A skip for a capability this SDK cannot express
+  carries a reason worded unlike an undeclared one, so the two are distinguishable in the results.
+
+`knownDeviations` says whether a withheld capability is a limitation or a bug; `results.digest`
+covers the `.ndjson`. Every scenario appears exactly once, which `ConformanceReportPluginTest`
+asserts over a real Cucumber run.
+
+[report-schema]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/report/conformance-report.schema.json
+[messages]: https://github.com/cucumber/messages
+
 ## Extensions
 
 Provider-specific scenarios (e.g. flagd's `fractional`) run inside the same suite:

tools/tck/pom.xml

@@ -18,6 +18,12 @@
         <!-- skip moving the spec submodule to the pinned commit; digest test still guards assets -->
         <tck.spec.checkout.skip>false</tck.spec.checkout.skip>
 
+        <!--
+            The open-feature/spec commit the packaged assets came from, recorded in every
+            conformance report. Must equal `git -C spec rev-parse HEAD`; ConformanceReportPluginTest
+            fails the build when it does not.
+        -->
+        <tck.spec.revision>10b1a610ed0afe6bf0e38b441d7362f5a76cb47d</tck.spec.revision>
         <assertj.version>3.27.7</assertj.version>
         <awaitility.version>4.3.0</awaitility.version>
         <jackson-databind.version>2.22.3</jackson-databind.version>
@@ -66,6 +72,39 @@
             <artifactId>cucumber-junit-platform-engine</artifactId>
         </dependency>
 
+        <!--
+            Cucumber core and the message types, already transitive through
+            cucumber-junit-platform-engine: ConformanceReportPlugin compiles against them and writes
+            the results stream with Cucumber's own MessageFormatter.
+        -->
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-core</artifactId>
+            <!-- version managed by cucumber-bom in parent -->
+        </dependency>
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>messages</artifactId>
+            <!-- version managed by cucumber-bom in parent -->
+        </dependency>
+
+        <!-- Gherkin: CanonicalScenarios parses the canonical set with the runner's own parser -->
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>gherkin</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+        <!--
+            JUnit Jupiter API: CanonicalScenarioGuard is a test the suite selects, because a
+            TestExecutionListener's exception is logged rather than failing the run.
+        -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
         <!--
             The container TckObjectFactory builds, not io.cucumber:cucumber-picocontainer. That
             artifact also registers Cucumber's own PicoFactory through META-INF/services, which
@@ -149,6 +188,18 @@
     </dependencies>
 
     <build>
+        <!-- only the generated build-info properties are filtered; the shared assets are copied verbatim -->
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>false</filtering>
+            </resource>
+            <resource>
+                <directory>src/main/resources-filtered</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+
         <plugins>
             <!-- phase 1: move spec submodule working tree to the pinned commit -->
             <plugin>