diff --git a/.github/workflows/generate-qualification.yml b/.github/workflows/generate-qualification.yml new file mode 100644 index 000000000..5feae773a --- /dev/null +++ b/.github/workflows/generate-qualification.yml @@ -0,0 +1,181 @@ +name: Docker generate qualification + +on: + # No paths filter. A required check must report on every pull request and + # every queue entry, so `changes` decides whether the shards run and + # `docker-generate` always reports one status for the whole job. + pull_request: + branches: [main] + merge_group: + schedule: + - cron: '23 5 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docker-generate-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # The one row this workflow proves. The shards run its packages under its + # build tag (both derived from pkg/conformance/matrix.json), its gates + # cannot skip, and the summary verifies its receipts across every shard. + CODEFLY_CONFORMANCE_REQUIRED: linux-amd64-docker-generate + +jobs: + changes: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + relevant: ${{ steps.scope.outputs.relevant }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Decide whether anything but documentation changed + id: scope + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + run: | + set -euo pipefail + # The nightly and a manual run have no base: they always qualify. + if [[ -z "$BASE" ]]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + git fetch --no-tags --depth=1 origin "$BASE" + ./scripts/generate-qualification-scope.sh "$BASE" | tee -a "$GITHUB_OUTPUT" + + generate: + needs: changes + if: needs.changes.outputs.relevant == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Round-robin over discovered tests. Which shard draws a + # receipt-bearing test moves as tests are added, so receipts are + # verified across all shards in docker-generate, never per shard. + # Separate runners also isolate the Docker daemon; each runner pulls + # the published image only once. + shard: [0, 1, 2] + env: + GOWORK: 'off' + GOFLAGS: -mod=mod + CODEFLY_GENERATE_QUALIFY: '1' + CODEFLY_CONFORMANCE_RECEIPTS: ${{ github.workspace }}/conformance-receipts + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '24' + - name: Restore Go build cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/go-build + key: cli-go-build-generate-${{ runner.os }}-${{ hashFiles('go.sum') }} + restore-keys: | + cli-go-build-generate-${{ runner.os }}- + - name: Pull the published companion once and provision its buf + run: | + set -euo pipefail + docker info + image=$(go run ./scripts/generate-companion-image) + docker pull "$image" + docker image inspect "$image" --format '{{json .RepoDigests}}' + # The SDK fixture needs host buf. Reuse the static binary shipped + # in the image under test, without another version or download. + container=$(docker create "$image" /bin/true) + trap 'docker rm "$container"' EXIT + mkdir -p "$RUNNER_TEMP/generate-bin" + docker cp "$container:/usr/local/bin/buf" "$RUNNER_TEMP/generate-bin/buf" + "$RUNNER_TEMP/generate-bin/buf" --version + echo "$RUNNER_TEMP/generate-bin" >> "$GITHUB_PATH" + - name: Run tagged and qualification tests in isolated processes + run: ./scripts/test-generate-isolated.sh "${{ matrix.shard }}" "${{ strategy.job-total }}" + - name: Retain test output, timings and conformance receipts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: docker-generate-evidence-${{ matrix.shard }} + path: | + generate-evidence/ + conformance-receipts/ + if-no-files-found: error + + # The one status to require: it fails unless the shards passed and every + # shard's receipts together prove the row, and it passes without them only + # when `changes` found nothing but documentation. + docker-generate: + needs: [changes, generate] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + CODEFLY_CONFORMANCE_RECEIPTS: ${{ github.workspace }}/conformance-receipts + steps: + - name: Read the shards' outcome + id: outcome + env: + CHANGES: ${{ needs.changes.result }} + RELEVANT: ${{ needs.changes.outputs.relevant }} + SHARDS: ${{ needs.generate.result }} + run: | + set -euo pipefail + if [[ "$CHANGES" != success ]]; then + echo "the change classification ended '$CHANGES', so nothing decided whether the row must run" >&2 + exit 1 + fi + case "$RELEVANT" in + false) + if [[ "$SHARDS" != skipped ]]; then + echo "only documentation changed, yet the shards ended '$SHARDS'" >&2 + exit 1 + fi + echo "only documentation changed; nothing that generation compiles or reads moved" + echo "verify=false" >> "$GITHUB_OUTPUT" + ;; + true) + if [[ "$SHARDS" != success ]]; then + echo "the generate shards ended '$SHARDS'" >&2 + exit 1 + fi + echo "verify=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "the change classification reported relevant='$RELEVANT', want true or false" >&2 + exit 1 + ;; + esac + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: steps.outcome.outputs.verify == 'true' + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: steps.outcome.outputs.verify == 'true' + with: + pattern: docker-generate-evidence-* + path: shards + - name: Verify conformance receipts across every shard + if: steps.outcome.outputs.verify == 'true' + run: | + set -euo pipefail + mkdir -p "$CODEFLY_CONFORMANCE_RECEIPTS" + shopt -s nullglob + receipts=(shards/*/conformance-receipts/*.json) + shopt -u nullglob + # Receipt names carry the package and test, and shards run disjoint + # tests, so no copy overwrites another shard's receipt. + if (( ${#receipts[@]} > 0 )); then + cp "${receipts[@]}" "$CODEFLY_CONFORMANCE_RECEIPTS/" + fi + ./scripts/verify-conformance-receipts.sh diff --git a/cmd/generate/client_test.go b/cmd/generate/client_test.go index 0001d0dc1..96ca4b541 100644 --- a/cmd/generate/client_test.go +++ b/cmd/generate/client_test.go @@ -967,8 +967,10 @@ func assertPythonImports(t *testing.T, pythonDir, dottedModule string) { t.Fatalf("CompanionImage: %v", err) } cmd := exec.CommandContext(ctx, "docker", "run", "--rm", - "-v", pythonDir+":/out", "-w", "/out", image.FullName(), - "/venv/bin/python", "-c", "import "+dottedModule) + // This is a read-only probe. Root-owned __pycache__ directories on a + // writable bind mount prevent the Linux runner from cleaning t.TempDir. + "-v", pythonDir+":/out:ro", "-w", "/out", image.FullName(), + "/venv/bin/python", "-B", "-c", "import "+dottedModule) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("import %s failed: %v\n%s", dottedModule, err, out) } diff --git a/docs/supported-matrix.md b/docs/supported-matrix.md index 44e8a3434..6f81c7c75 100644 --- a/docs/supported-matrix.md +++ b/docs/supported-matrix.md @@ -67,7 +67,7 @@ Keep the required rows and receipt verifier enabled; absence is still a failure. | Status | Meaning | |---|---| -| `qualified` | A CI job drives this row on every change and lists it in `CODEFLY_CONFORMANCE_REQUIRED`, so its tests cannot skip themselves when the backend is missing. Every qualified row must declare a gate call site — without one nothing could hold it to the claim. | +| `qualified` | A CI job drives this row on every change and lists it in `CODEFLY_CONFORMANCE_REQUIRED`, so its tests cannot skip themselves when the backend is missing. The job may pass over a change that touches only documentation, which cannot alter the row's outcome. Every qualified row must declare a gate call site — without one nothing could hold it to the claim. | | `not-yet-qualified` | Real tests and a real gate exist, but nothing proves the row outside a developer machine. **This is not a support claim.** Each row lists what blocks it. | | `unsupported` | Not shipped and not tested. | @@ -78,13 +78,36 @@ Keep the required rows and receipt verifier enabled; absence is still a failure. | `linux-amd64-source` | qualified | linux/amd64 | none | — | `go.yml` — `coverage`, `race`, `lint` | | `linux-amd64-native-npm` | qualified | linux/amd64 | native | `npm` | `go.yml` — `coverage`, `race` | | `linux-amd64-native-control` | qualified | linux/amd64 | native protocol peers | `go` | `go.yml` — `control-integration` | -| `linux-amd64-docker-generate` | not-yet-qualified | linux/amd64 | docker | `docker`, `buf` | `CODEFLY_GENERATE_QUALIFY=1`, developer machine only | +| `linux-amd64-docker-generate` | qualified | linux/amd64 | docker | `docker`, `buf` | `generate-qualification.yml` — `generate`, reported as `docker-generate` | | `linux-amd64-docker-mcp-run` | not-yet-qualified | linux/amd64 | docker | `docker` | `CODEFLY_MCP_RUN_QUALIFY=1`, developer machine only | | `linux-amd64-network-mcp-agent` | not-yet-qualified | linux/amd64 | native | — | `CODEFLY_MCP_AGENT_QUALIFY=1`, developer machine only | | `linux-amd64-k3d-deploy` | not-yet-qualified | linux/amd64 | k3d | `docker`, `k3d`, `kubectl`, `ssh-keygen`, `git` | `CODEFLY_GITOPS_K3D_QUALIFY=1`, developer machine only | | `darwin-arm64-native-user-service` | not-yet-qualified | darwin/arm64 | native | `launchctl` | `CODEFLY_SERVICE_INTEGRATION=1`, developer machine only | | `windows-amd64-native-run` | unsupported | windows/amd64 | native | — | none | +The Docker generate job runs every gate package of the row and every package +holding a file built under its `build_tag` (`proto_companion_required`), with +that tag and `CODEFLY_GENERATE_QUALIFY=1`. Both come from the matrix and the +tagged sources through +[`scripts/conformance-packages`](../scripts/conformance-packages), never from +a list in the script, and a test in `pkg/conformance` fails if the script's +loop drifts from them. It pulls the published proto image resolved by Core +once per runner, and extracts that image's `buf` for the SDK fixture. Each +package is compiled once per shard; three runners split the discovered tests +round-robin, and every top-level test runs in a fresh process with its own +Codefly home, so package globals cannot leak between tests. The shards retain +per-test output, timings and conformance receipts. + +It runs on every pull request, every merge-queue entry and nightly. One job, +`docker-generate`, always reports: it fails unless every shard passed and the +receipts of all shards together prove the row. A change touching only +documentation (`docs/` or Markdown outside `testdata/`) cannot alter what +generation compiles or reads, so there the shards are skipped and +`docker-generate` passes without them; every other change runs them. +[`scripts/generate-qualification-scope.sh`](../scripts/generate-qualification-scope.sh) +decides, with rename detection off: a file moved into `docs/` or renamed to +`.md` counts by its old path too. + ### Qualification gaps The matrix carries the blocker for every `not-yet-qualified` row. The two that @@ -149,7 +172,8 @@ the test step's exit code. test actually uses. 3. Add the row to this table. 4. When a CI job provisions the backend, add the job to the row's `ci` list, - name the row in that gate's `conformance_rows` in `go.yml`, drop the + name the row in that gate's `CODEFLY_CONFORMANCE_REQUIRED` (or its + `conformance_rows` matrix value), drop the blockers, and flip the status to `qualified`. The tests in `pkg/conformance` fail if any of those four are missing — the `ci` entry is resolved against the workflow's real jobs and matrix gates, not matched as a substring. diff --git a/pkg/conformance/matrix.go b/pkg/conformance/matrix.go index 7bc22b30c..8c72c35f8 100644 --- a/pkg/conformance/matrix.go +++ b/pkg/conformance/matrix.go @@ -36,7 +36,10 @@ var embeddedMatrix []byte type Status string const ( - // StatusQualified means CI drives the row end to end on every change. + // StatusQualified means CI drives the row end to end on every change and + // requires it in CODEFLY_CONFORMANCE_REQUIRED, so its tests cannot skip. + // The job may pass over a change that touches only documentation, which + // cannot alter the row's outcome (docs/supported-matrix.md). StatusQualified Status = "qualified" // StatusNotYetQualified means the row has a real gate and real tests, but // nothing proves it outside a developer machine. It is not supported. diff --git a/pkg/conformance/matrix.json b/pkg/conformance/matrix.json index 5dfa1818d..e17544d63 100644 --- a/pkg/conformance/matrix.json +++ b/pkg/conformance/matrix.json @@ -57,7 +57,7 @@ }, { "id": "linux-amd64-docker-generate", - "status": "not-yet-qualified", + "status": "qualified", "summary": "Contract and client generation through the real proto companion image, including the solution SDK facade compiled from a buf-built descriptor set.", "os": "linux", "arch": "amd64", @@ -67,13 +67,14 @@ "buf" ], "gate_env": "CODEFLY_GENERATE_QUALIFY", - "blockers": [ - "No CI job provisions the proto companion image, so no run has ever proved this row." - ], "gates": [ "cmd/generate", "cmd/sync" - ] + ], + "ci": [ + "generate-qualification.yml#generate" + ], + "build_tag": "proto_companion_required" }, { "id": "linux-amd64-docker-mcp-run", diff --git a/pkg/conformance/matrix_test.go b/pkg/conformance/matrix_test.go index 248868ed7..96da76e00 100644 --- a/pkg/conformance/matrix_test.go +++ b/pkg/conformance/matrix_test.go @@ -204,9 +204,10 @@ func TestEmbeddedMatrixIsQualifiedOnlyWhereCIProvesIt(t *testing.T) { } } want := map[string]bool{ - "linux-amd64-source": true, - "linux-amd64-native-npm": true, - "linux-amd64-native-control": true, + "linux-amd64-source": true, + "linux-amd64-native-npm": true, + "linux-amd64-native-control": true, + "linux-amd64-docker-generate": true, } if len(qualified) != len(want) { t.Fatalf("qualified rows = %v, want %v", qualified, want) diff --git a/pkg/conformance/packages.go b/pkg/conformance/packages.go new file mode 100644 index 000000000..84aafa9be --- /dev/null +++ b/pkg/conformance/packages.go @@ -0,0 +1,115 @@ +package conformance + +import ( + "bufio" + "fmt" + "go/build/constraint" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// Packages lists, relative to the repository root, every package a job must +// compile to prove the row: its gate packages, plus every package holding a +// file whose build constraint names the row's build tag. Deriving the list is +// what stops a new gate or tagged test from being silently left out of the job +// that claims the row. +func (r *Row) Packages(root string) ([]string, error) { + set := map[string]bool{} + for _, gate := range r.Gates { + set[gate] = true + } + if r.BuildTag != "" { + tagged, err := TaggedPackages(root, r.BuildTag) + if err != nil { + return nil, err + } + for _, pkg := range tagged { + set[pkg] = true + } + } + if len(set) == 0 { + return nil, fmt.Errorf("conformance row %s has no gate package and no build-tagged file: nothing would run", r.ID) + } + packages := make([]string, 0, len(set)) + for pkg := range set { + packages = append(packages, pkg) + } + sort.Strings(packages) + return packages, nil +} + +// TaggedPackages lists, relative to root, every package with a Go file whose +// //go:build constraint names tag. It skips the directories the go tool +// ignores (testdata, and names starting with "." or "_") and node_modules. +func TaggedPackages(root, tag string) ([]string, error) { + set := map[string]bool{} + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + name := entry.Name() + if path != root && (name == "testdata" || name == "node_modules" || + strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_")) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(entry.Name(), ".go") { + return nil + } + names, err := constraintNamesTag(path, tag) + if err != nil || !names { + return err + } + pkg, err := filepath.Rel(root, filepath.Dir(path)) + if err != nil { + return err + } + set[filepath.ToSlash(pkg)] = true + return nil + }) + if err != nil { + return nil, fmt.Errorf("find packages tagged %s: %w", tag, err) + } + packages := make([]string, 0, len(set)) + for pkg := range set { + packages = append(packages, pkg) + } + sort.Strings(packages) + return packages, nil +} + +// constraintNamesTag reports whether the file's //go:build line, which can +// only appear in the comment block before the package clause, names tag. +func constraintNamesTag(path, tag string) (bool, error) { + file, err := os.Open(path) //nolint:gosec // G304: walking the repository's own sources + if err != nil { + return false, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" && !strings.HasPrefix(line, "//") { + break + } + if !constraint.IsGoBuild(line) { + continue + } + expr, err := constraint.Parse(line) + if err != nil { + return false, fmt.Errorf("%s: %w", path, err) + } + named := false + expr.Eval(func(atom string) bool { + named = named || atom == tag + return true + }) + return named, nil + } + return false, scanner.Err() +} diff --git a/pkg/conformance/packages_test.go b/pkg/conformance/packages_test.go new file mode 100644 index 000000000..ab6cf67f2 --- /dev/null +++ b/pkg/conformance/packages_test.go @@ -0,0 +1,232 @@ +package conformance + +import ( + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func writeSource(t *testing.T, root, path, body string) { + t.Helper() + full := filepath.Join(root, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestPackagesUnionGatesAndTaggedSources(t *testing.T) { + root := t.TempDir() + writeSource(t, root, "cmd/tagged/a_test.go", "// Copyright.\n\n//go:build linux && companion\n\npackage tagged\n") + writeSource(t, root, "cmd/negated/b.go", "//go:build !companion\n\npackage negated\n") + writeSource(t, root, "cmd/other/c.go", "//go:build integration\n\npackage other\n") + writeSource(t, root, "cmd/late/d.go", "package late\n\n//go:build companion\n") + writeSource(t, root, "cmd/fixture/testdata/e.go", "//go:build companion\n\npackage fixture\n") + writeSource(t, root, "web/node_modules/f/f.go", "//go:build companion\n\npackage f\n") + + row := Row{ID: "row", Gates: []string{"cmd/gated", "cmd/tagged"}, BuildTag: "companion"} + got, err := row.Packages(root) + if err != nil { + t.Fatal(err) + } + // A negated constraint still names the tag, so its package is compiled + // too; a constraint after the package clause is no constraint at all. + want := []string{"cmd/gated", "cmd/negated", "cmd/tagged"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("Packages = %v, want %v", got, want) + } + + if _, err := (&Row{ID: "empty", BuildTag: "absent"}).Packages(root); err == nil { + t.Fatal("a row with no gate and no tagged file derived packages; nothing would run") + } +} + +// TestGenerateQualificationScriptRunsTheMatrixRow proves the Docker generate +// job, the matrix row it claims and the build-tagged sources agree: the script +// compiles every gate package and every package with a file built under the +// row's tag, under that tag, with no list of its own to drift. The expected +// set is computed here independently of Row.Packages. +func TestGenerateQualificationScriptRunsTheMatrixRow(t *testing.T) { + const workflow, script = "generate-qualification.yml", "scripts/test-generate-isolated.sh" + root := repositoryRoot(t) + var document struct { + Env map[string]string `yaml:"env"` + Jobs map[string]struct { + Steps []struct { + Run string `yaml:"run"` + } `yaml:"steps"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal([]byte(readFile(t, filepath.Join(root, ".github", "workflows", workflow))), &document); err != nil { + t.Fatal(err) + } + id := document.Env[RequiredEnv] + row, ok := Default().Row(id) + if !ok { + t.Fatalf("%s claims row %q in its workflow env, which the matrix does not have", workflow, id) + } + if !slices.Contains(row.CI, workflow+"#generate") { + t.Fatalf("row %s does not name %s#generate in its ci list %v", id, workflow, row.CI) + } + runsScript := false + for _, step := range document.Jobs["generate"].Steps { + runsScript = runsScript || strings.Contains(step.Run, "./"+script) + } + if !runsScript { + t.Fatalf("the generate job of %s never runs %s", workflow, script) + } + if row.BuildTag == "" { + t.Fatalf("row %s declares no build_tag, so its tagged tests could never be compiled", id) + } + + expected := map[string]bool{} + for _, gate := range row.Gates { + expected[gate] = true + } + tagged := regexp.MustCompile(`(?m)^//go:build .*\b` + regexp.QuoteMeta(row.BuildTag) + `\b`) + taggedFiles := 0 + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + switch entry.Name() { + case ".git", "node_modules", "testdata": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(entry.Name(), ".go") || !tagged.MatchString(readFile(t, path)) { + return nil + } + taggedFiles++ + pkg, relErr := filepath.Rel(root, filepath.Dir(path)) + expected[filepath.ToSlash(pkg)] = true + return relErr + }) + if err != nil { + t.Fatal(err) + } + if taggedFiles == 0 { + t.Fatalf("no file carries build tag %s; the row's tag proves nothing", row.BuildTag) + } + want := make([]string, 0, len(expected)) + for pkg := range expected { + want = append(want, pkg) + } + sort.Strings(want) + + cmd := exec.Command("bash", script, "--list") + cmd.Dir = root + cmd.Env = append(os.Environ(), RequiredEnv+"="+id) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s --list: %v\n%s", script, err, out) + } + lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n") + if compile := "go test -tags " + row.BuildTag + " -c"; lines[0] != compile { + t.Errorf("%s compiles with %q, want %q for row %s", script, lines[0], compile, id) + } + if got := lines[1:]; !reflect.DeepEqual(got, want) { + t.Errorf("%s runs packages %v, want the row's gates and tagged packages %v", script, got, want) + } + // A literal equal to today's derived value prints the same list, so the + // script must not spell any of them: a hard-coded tag or package would + // stop following the matrix the day it changes. + body := readFile(t, filepath.Join(root, script)) + for _, literal := range append([]string{row.BuildTag}, want...) { + if strings.Contains(body, literal) { + t.Errorf("%s spells %q itself instead of deriving it from the matrix", script, literal) + } + } + // What --list printed is what compiles: the script builds a test binary + // only through the printed command, never through a second go test line. + for _, line := range strings.Split(body, "\n") { + code := strings.TrimSpace(line) + if strings.HasPrefix(code, "#") || !strings.Contains(code, "go test") { + continue + } + if code != "compile=(go test)" { + t.Errorf("%s runs go test outside the command --list prints: %q", script, code) + } + } + if !strings.Contains(body, `"${compile[@]}" -o`) { + t.Errorf("%s does not compile through the command --list prints", script) + } +} + +// TestDocsOnlyScopeSeesBothSidesOfARename holds the classifier that lets +// docker-generate pass without its shards. Git's rename detection would list +// only a moved file's new path, so moving code or a fixture into docs/ or to a +// .md name must still count as a change the row has to run for. +func TestDocsOnlyScopeSeesBothSidesOfARename(t *testing.T) { + script := filepath.Join(repositoryRoot(t), "scripts", "generate-qualification-scope.sh") + const source = "package generate\n\nfunc Generate() string { return \"a long enough body for rename detection\" }\n" + cases := []struct { + name string + change func(t *testing.T, dir string) + want string + }{ + {"prose only", func(t *testing.T, dir string) { + writeSource(t, dir, "docs/guide.md", "# Guide, revised\n") + writeSource(t, dir, "README.md", "# Readme, revised\n") + }, "relevant=false"}, + {"code moved into docs", func(t *testing.T, dir string) { + gitIn(t, dir, "mv", "cmd/generate/generate.go", "docs/generate.go") + }, "relevant=true"}, + {"test renamed to markdown", func(t *testing.T, dir string) { + gitIn(t, dir, "mv", "cmd/generate/generate_test.go", "cmd/generate/generate_test.md") + }, "relevant=true"}, + {"fixture moved into docs", func(t *testing.T, dir string) { + gitIn(t, dir, "mv", "cmd/sync/testdata/fixture.md", "docs/fixture.md") + }, "relevant=true"}, + {"markdown fixture edited", func(t *testing.T, dir string) { + writeSource(t, dir, "cmd/sync/testdata/fixture.md", "# Fixture, revised\n") + }, "relevant=true"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + gitIn(t, dir, "init", "-q") + writeSource(t, dir, "cmd/generate/generate.go", source) + writeSource(t, dir, "cmd/generate/generate_test.go", strings.Replace(source, "func Generate", "func Test", 1)) + writeSource(t, dir, "cmd/sync/testdata/fixture.md", "# Fixture\n\nA fixture long enough for rename detection.\n") + writeSource(t, dir, "docs/guide.md", "# Guide\n") + writeSource(t, dir, "README.md", "# Readme\n") + gitIn(t, dir, "add", "-A") + gitIn(t, dir, "commit", "-q", "-m", "base") + tc.change(t, dir) + gitIn(t, dir, "add", "-A") + gitIn(t, dir, "commit", "-q", "-m", "change") + + cmd := exec.Command("bash", script, "HEAD~1") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s: %v\n%s", script, err, out) + } + if got := strings.TrimSpace(string(out)); got != tc.want { + t.Fatalf("scope = %s, want %s", got, tc.want) + } + }) + } +} + +func gitIn(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-c", "user.name=test", "-c", "user.email=test@example.invalid", "-c", "commit.gpgsign=false"}, args...)...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} diff --git a/scripts/conformance-packages/main.go b/scripts/conformance-packages/main.go new file mode 100644 index 000000000..3d4c2a61a --- /dev/null +++ b/scripts/conformance-packages/main.go @@ -0,0 +1,38 @@ +// Print what a job must compile to prove one conformance row: the row's build +// tag on the first line (empty when it has none), then one package per line. +// The matrix and the tagged sources are the only inputs, so a qualification +// script never keeps its own package list or tag. +package main + +import ( + "fmt" + "os" + + "github.com/codefly-dev/cli/pkg/conformance" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: conformance-packages ") + os.Exit(2) + } + row, ok := conformance.Default().Row(os.Args[1]) + if !ok { + fmt.Fprintf(os.Stderr, "unknown conformance row %q\n", os.Args[1]) + os.Exit(1) + } + root, err := os.Getwd() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + packages, err := row.Packages(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(row.BuildTag) + for _, pkg := range packages { + fmt.Println(pkg) + } +} diff --git a/scripts/generate-companion-image/main.go b/scripts/generate-companion-image/main.go new file mode 100644 index 000000000..b858086a8 --- /dev/null +++ b/scripts/generate-companion-image/main.go @@ -0,0 +1,20 @@ +// Print the same published image reference that generate uses. CI must not +// duplicate Core's registry or version pin when pre-pulling its companion. +package main + +import ( + "context" + "fmt" + "os" + + "github.com/codefly-dev/core/companions/proto" +) + +func main() { + image, err := proto.CompanionImage(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(image.FullName()) +} diff --git a/scripts/generate-qualification-scope.sh b/scripts/generate-qualification-scope.sh new file mode 100755 index 000000000..5a6e04704 --- /dev/null +++ b/scripts/generate-qualification-scope.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Print relevant=true unless every path changed between and HEAD is +# documentation: docs/, or Markdown outside a testdata directory. +# +# Rename detection is off. With it, a moved file shows only its new path, so +# moving a Go source or a fixture into docs/ (or renaming it to .md) would +# read as a documentation-only change and skip the qualification it removed +# code from. +set -euo pipefail + +base=${1:?usage: generate-qualification-scope.sh } +changed=$(git diff --no-renames --name-only "$base" HEAD) +relevant=false +while IFS= read -r path; do + case "$path" in + '') ;; + # Fixtures can be Markdown; only prose is exempt. + testdata/*|*/testdata/*) relevant=true ;; + docs/*|*.md) ;; + *) relevant=true ;; + esac +done <<<"$changed" +echo "relevant=$relevant" diff --git a/scripts/test-generate-isolated.sh b/scripts/test-generate-isolated.sh new file mode 100755 index 000000000..50465f508 --- /dev/null +++ b/scripts/test-generate-isolated.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Compile once per package, then run every discovered top-level test in its +# own process: Cobra flags and other package globals survive between tests. +# +# The row comes from CODEFLY_CONFORMANCE_REQUIRED, the claim the job already +# makes. Its build tag and packages come from pkg/conformance/matrix.json and +# the tagged sources, never from a list kept here: `--list` prints the compile +# command and the packages. +set -euo pipefail + +row=${CODEFLY_CONFORMANCE_REQUIRED:-} +if [[ -z "$row" || "$row" == *,* ]]; then + echo "CODEFLY_CONFORMANCE_REQUIRED must name exactly one conformance row, got '${row}'" >&2 + exit 1 +fi +# A command substitution under set -e fails the script when the helper does; +# process substitution would hide that failure. +derived=$(go run ./scripts/conformance-packages "$row") +tag=$(head -n 1 <<<"$derived") +packages=() +while IFS= read -r package; do + packages+=("$package") +done < <(tail -n +2 <<<"$derived") +if (( ${#packages[@]} == 0 )); then + echo "row $row derived no packages" >&2 + exit 1 +fi +# The one compile command every package uses. --list prints it, then walks +# the same loop as a run and prints each package it would compile, so a +# repository test holds what actually runs to the matrix. +compile=(go test) +if [[ -n "$tag" ]]; then + compile+=(-tags "$tag") +fi +compile+=(-c) +list=0 +if [[ "${1:-}" == "--list" ]]; then + list=1 + echo "${compile[*]}" +else + shard=${1:-0} + shards=${2:-1} + if [[ ! "$shard" =~ ^[0-9]+$ || ! "$shards" =~ ^[1-9][0-9]*$ ]] || (( shard >= shards )); then + echo "expected a zero-based shard index smaller than the positive shard count" >&2 + exit 1 + fi + index=0 + selected=0 + failed=0 + root=$(pwd) + evidence="${root}/generate-evidence" + mkdir -p "$evidence" + work=$(mktemp -d) + trap 'rm -rf "$work"' EXIT + printf 'package\ttest\tseconds\tstatus\n' > "$evidence/timings.tsv" +fi + +for package in "${packages[@]}"; do + if (( list )); then + echo "$package" + continue + fi + name=${package//\//-} + binary="$work/$name.test" + "${compile[@]}" -o "$binary" "./$package" + # Do not hide discovery failures behind process substitution. A package + # with no tests must fail instead of silently claiming qualification. + "$binary" -test.list '^Test' > "$work/tests" + grep '^Test' "$work/tests" > "$work/selected" + while IFS= read -r test; do + slot=$((index % shards)) + index=$((index + 1)) + if (( slot != shard )); then + continue + fi + selected=$((selected + 1)) + echo "::group::$package/$test" + started=$SECONDS + status=passed + # Use the package directory just as go test does (fixtures are relative). + # Each process also gets its own on-disk Codefly state. + if ! (cd "$root/$package" && CODEFLY_HOME="$work/home-$name-$test" \ + "$binary" -test.run "^${test}$" -test.v -test.count=1 -test.timeout=4m) \ + 2>&1 | tee "$evidence/$name-$test.log"; then + status=failed + failed=1 + fi + printf '%s\t%s\t%s\t%s\n' "$package" "$test" "$((SECONDS - started))" "$status" >> "$evidence/timings.tsv" + echo '::endgroup::' + done < "$work/selected" +done +if (( list )); then + exit 0 +fi +if (( selected == 0 )); then + echo "shard $shard/$shards selected no tests" >&2 + exit 1 +fi +echo "shard $shard/$shards ran $selected of $index discovered tests" +exit "$failed"