ci: run Docker-gated generation tests on relevant PRs and nightly

.github/workflows/generate-qualification.yml

@@ -0,0 +1,181 @@
+name: Docker generate qualification
+
+on:
+  # No paths filter. A required check must report on every pull request and
+  # every queue entry, so `changes` decides whether the shards run and
+  # `docker-generate` always reports one status for the whole job.
+  pull_request:
+    branches: [main]
+  merge_group:
+  schedule:
+    - cron: '23 5 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docker-generate-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  # The one row this workflow proves. The shards run its packages under its
+  # build tag (both derived from pkg/conformance/matrix.json), its gates
+  # cannot skip, and the summary verifies its receipts across every shard.
+  CODEFLY_CONFORMANCE_REQUIRED: linux-amd64-docker-generate
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      relevant: ${{ steps.scope.outputs.relevant }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Decide whether anything but documentation changed
+        id: scope
+        env:
+          BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+        run: |
+          set -euo pipefail
+          # The nightly and a manual run have no base: they always qualify.
+          if [[ -z "$BASE" ]]; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          git fetch --no-tags --depth=1 origin "$BASE"
+          ./scripts/generate-qualification-scope.sh "$BASE" | tee -a "$GITHUB_OUTPUT"
+
+  generate:
+    needs: changes
+    if: needs.changes.outputs.relevant == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        # Round-robin over discovered tests. Which shard draws a
+        # receipt-bearing test moves as tests are added, so receipts are
+        # verified across all shards in docker-generate, never per shard.
+        # Separate runners also isolate the Docker daemon; each runner pulls
+        # the published image only once.
+        shard: [0, 1, 2]
+    env:
+      GOWORK: 'off'
+      GOFLAGS: -mod=mod
+      CODEFLY_GENERATE_QUALIFY: '1'
+      CODEFLY_CONFORMANCE_RECEIPTS: ${{ github.workspace }}/conformance-receipts
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: '24'
+      - name: Restore Go build cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: cli-go-build-generate-${{ runner.os }}-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            cli-go-build-generate-${{ runner.os }}-
+      - name: Pull the published companion once and provision its buf
+        run: |
+          set -euo pipefail
+          docker info
+          image=$(go run ./scripts/generate-companion-image)
+          docker pull "$image"
+          docker image inspect "$image" --format '{{json .RepoDigests}}'
+          # The SDK fixture needs host buf. Reuse the static binary shipped
+          # in the image under test, without another version or download.
+          container=$(docker create "$image" /bin/true)
+          trap 'docker rm "$container"' EXIT
+          mkdir -p "$RUNNER_TEMP/generate-bin"
+          docker cp "$container:/usr/local/bin/buf" "$RUNNER_TEMP/generate-bin/buf"
+          "$RUNNER_TEMP/generate-bin/buf" --version
+          echo "$RUNNER_TEMP/generate-bin" >> "$GITHUB_PATH"
+      - name: Run tagged and qualification tests in isolated processes
+        run: ./scripts/test-generate-isolated.sh "${{ matrix.shard }}" "${{ strategy.job-total }}"
+      - name: Retain test output, timings and conformance receipts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: docker-generate-evidence-${{ matrix.shard }}
+          path: |
+            generate-evidence/
+            conformance-receipts/
+          if-no-files-found: error
+
+  # The one status to require: it fails unless the shards passed and every
+  # shard's receipts together prove the row, and it passes without them only
+  # when `changes` found nothing but documentation.
+  docker-generate:
+    needs: [changes, generate]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    env:
+      CODEFLY_CONFORMANCE_RECEIPTS: ${{ github.workspace }}/conformance-receipts
+    steps:
+      - name: Read the shards' outcome
+        id: outcome
+        env:
+          CHANGES: ${{ needs.changes.result }}
+          RELEVANT: ${{ needs.changes.outputs.relevant }}
+          SHARDS: ${{ needs.generate.result }}
+        run: |
+          set -euo pipefail
+          if [[ "$CHANGES" != success ]]; then
+            echo "the change classification ended '$CHANGES', so nothing decided whether the row must run" >&2
+            exit 1
+          fi
+          case "$RELEVANT" in
+            false)
+              if [[ "$SHARDS" != skipped ]]; then
+                echo "only documentation changed, yet the shards ended '$SHARDS'" >&2
+                exit 1
+              fi
+              echo "only documentation changed; nothing that generation compiles or reads moved"
+              echo "verify=false" >> "$GITHUB_OUTPUT"
+              ;;
+            true)
+              if [[ "$SHARDS" != success ]]; then
+                echo "the generate shards ended '$SHARDS'" >&2
+                exit 1
+              fi
+              echo "verify=true" >> "$GITHUB_OUTPUT"
+              ;;
+            *)
+              echo "the change classification reported relevant='$RELEVANT', want true or false" >&2
+              exit 1
+              ;;
+          esac
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: steps.outcome.outputs.verify == 'true'
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        if: steps.outcome.outputs.verify == 'true'
+        with:
+          pattern: docker-generate-evidence-*
+          path: shards
+      - name: Verify conformance receipts across every shard
+        if: steps.outcome.outputs.verify == 'true'
+        run: |
+          set -euo pipefail
+          mkdir -p "$CODEFLY_CONFORMANCE_RECEIPTS"
+          shopt -s nullglob
+          receipts=(shards/*/conformance-receipts/*.json)
+          shopt -u nullglob
+          # Receipt names carry the package and test, and shards run disjoint
+          # tests, so no copy overwrites another shard's receipt.
+          if (( ${#receipts[@]} > 0 )); then
+            cp "${receipts[@]}" "$CODEFLY_CONFORMANCE_RECEIPTS/"
+          fi
+          ./scripts/verify-conformance-receipts.sh

cmd/generate/client_test.go

@@ -967,8 +967,10 @@ func assertPythonImports(t *testing.T, pythonDir, dottedModule string) {
 		t.Fatalf("CompanionImage: %v", err)
 	}
 	cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
-		"-v", pythonDir+":/out", "-w", "/out", image.FullName(),
-		"/venv/bin/python", "-c", "import "+dottedModule)
+		// This is a read-only probe. Root-owned __pycache__ directories on a
+		// writable bind mount prevent the Linux runner from cleaning t.TempDir.
+		"-v", pythonDir+":/out:ro", "-w", "/out", image.FullName(),
+		"/venv/bin/python", "-B", "-c", "import "+dottedModule)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatalf("import %s failed: %v\n%s", dottedModule, err, out)
 	}

docs/supported-matrix.md

@@ -67,7 +67,7 @@ Keep the required rows and receipt verifier enabled; absence is still a failure.
 
 | Status | Meaning |
 |---|---|
-| `qualified` | A CI job drives this row on every change and lists it in `CODEFLY_CONFORMANCE_REQUIRED`, so its tests cannot skip themselves when the backend is missing. Every qualified row must declare a gate call site — without one nothing could hold it to the claim. |
+| `qualified` | A CI job drives this row on every change and lists it in `CODEFLY_CONFORMANCE_REQUIRED`, so its tests cannot skip themselves when the backend is missing. The job may pass over a change that touches only documentation, which cannot alter the row's outcome. Every qualified row must declare a gate call site — without one nothing could hold it to the claim. |
 | `not-yet-qualified` | Real tests and a real gate exist, but nothing proves the row outside a developer machine. **This is not a support claim.** Each row lists what blocks it. |
 | `unsupported` | Not shipped and not tested. |
 
@@ -78,13 +78,36 @@ Keep the required rows and receipt verifier enabled; absence is still a failure.
 | `linux-amd64-source` | qualified | linux/amd64 | none | — | `go.yml` — `coverage`, `race`, `lint` |
 | `linux-amd64-native-npm` | qualified | linux/amd64 | native | `npm` | `go.yml` — `coverage`, `race` |
 | `linux-amd64-native-control` | qualified | linux/amd64 | native protocol peers | `go` | `go.yml` — `control-integration` |
-| `linux-amd64-docker-generate` | not-yet-qualified | linux/amd64 | docker | `docker`, `buf` | `CODEFLY_GENERATE_QUALIFY=1`, developer machine only |
+| `linux-amd64-docker-generate` | qualified | linux/amd64 | docker | `docker`, `buf` | `generate-qualification.yml` — `generate`, reported as `docker-generate` |
 | `linux-amd64-docker-mcp-run` | not-yet-qualified | linux/amd64 | docker | `docker` | `CODEFLY_MCP_RUN_QUALIFY=1`, developer machine only |
 | `linux-amd64-network-mcp-agent` | not-yet-qualified | linux/amd64 | native | — | `CODEFLY_MCP_AGENT_QUALIFY=1`, developer machine only |
 | `linux-amd64-k3d-deploy` | not-yet-qualified | linux/amd64 | k3d | `docker`, `k3d`, `kubectl`, `ssh-keygen`, `git` | `CODEFLY_GITOPS_K3D_QUALIFY=1`, developer machine only |
 | `darwin-arm64-native-user-service` | not-yet-qualified | darwin/arm64 | native | `launchctl` | `CODEFLY_SERVICE_INTEGRATION=1`, developer machine only |
 | `windows-amd64-native-run` | unsupported | windows/amd64 | native | — | none |
 
+The Docker generate job runs every gate package of the row and every package
+holding a file built under its `build_tag` (`proto_companion_required`), with
+that tag and `CODEFLY_GENERATE_QUALIFY=1`. Both come from the matrix and the
+tagged sources through
+[`scripts/conformance-packages`](../scripts/conformance-packages), never from
+a list in the script, and a test in `pkg/conformance` fails if the script's
+loop drifts from them. It pulls the published proto image resolved by Core
+once per runner, and extracts that image's `buf` for the SDK fixture. Each
+package is compiled once per shard; three runners split the discovered tests
+round-robin, and every top-level test runs in a fresh process with its own
+Codefly home, so package globals cannot leak between tests. The shards retain
+per-test output, timings and conformance receipts.
+
+It runs on every pull request, every merge-queue entry and nightly. One job,
+`docker-generate`, always reports: it fails unless every shard passed and the
+receipts of all shards together prove the row. A change touching only
+documentation (`docs/` or Markdown outside `testdata/`) cannot alter what
+generation compiles or reads, so there the shards are skipped and
+`docker-generate` passes without them; every other change runs them.
+[`scripts/generate-qualification-scope.sh`](../scripts/generate-qualification-scope.sh)
+decides, with rename detection off: a file moved into `docs/` or renamed to
+`.md` counts by its old path too.
+
 ### Qualification gaps
 
 The matrix carries the blocker for every `not-yet-qualified` row. The two that
@@ -149,7 +172,8 @@ the test step's exit code.
    test actually uses.
 3. Add the row to this table.
 4. When a CI job provisions the backend, add the job to the row's `ci` list,
-   name the row in that gate's `conformance_rows` in `go.yml`, drop the
+   name the row in that gate's `CODEFLY_CONFORMANCE_REQUIRED` (or its
+   `conformance_rows` matrix value), drop the
    blockers, and flip the status to `qualified`. The tests in `pkg/conformance`
    fail if any of those four are missing — the `ci` entry is resolved against
    the workflow's real jobs and matrix gates, not matched as a substring.

pkg/conformance/matrix.go

@@ -36,7 +36,10 @@ var embeddedMatrix []byte
 type Status string
 
 const (
-	// StatusQualified means CI drives the row end to end on every change.
+	// StatusQualified means CI drives the row end to end on every change and
+	// requires it in CODEFLY_CONFORMANCE_REQUIRED, so its tests cannot skip.
+	// The job may pass over a change that touches only documentation, which
+	// cannot alter the row's outcome (docs/supported-matrix.md).
 	StatusQualified Status = "qualified"
 	// StatusNotYetQualified means the row has a real gate and real tests, but
 	// nothing proves it outside a developer machine. It is not supported.

pkg/conformance/matrix.json

@@ -57,7 +57,7 @@
     },
     {
       "id": "linux-amd64-docker-generate",
-      "status": "not-yet-qualified",
+      "status": "qualified",
       "summary": "Contract and client generation through the real proto companion image, including the solution SDK facade compiled from a buf-built descriptor set.",
       "os": "linux",
       "arch": "amd64",
@@ -67,13 +67,14 @@
         "buf"
       ],
       "gate_env": "CODEFLY_GENERATE_QUALIFY",
-      "blockers": [
-        "No CI job provisions the proto companion image, so no run has ever proved this row."
-      ],
       "gates": [
         "cmd/generate",
         "cmd/sync"
-      ]
+      ],
+      "ci": [
+        "generate-qualification.yml#generate"
+      ],
+      "build_tag": "proto_companion_required"
     },
     {
       "id": "linux-amd64-docker-mcp-run",

pkg/conformance/matrix_test.go

@@ -204,9 +204,10 @@ func TestEmbeddedMatrixIsQualifiedOnlyWhereCIProvesIt(t *testing.T) {
 		}
 	}
 	want := map[string]bool{
-		"linux-amd64-source":         true,
-		"linux-amd64-native-npm":     true,
-		"linux-amd64-native-control": true,
+		"linux-amd64-source":          true,
+		"linux-amd64-native-npm":      true,
+		"linux-amd64-native-control":  true,
+		"linux-amd64-docker-generate": true,
 	}
 	if len(qualified) != len(want) {
 		t.Fatalf("qualified rows = %v, want %v", qualified, want)