diff --git a/.github/scripts/changelog-merge/main_test.go b/.github/scripts/changelog-merge/main_test.go index a6edd15b..767966a4 100644 --- a/.github/scripts/changelog-merge/main_test.go +++ b/.github/scripts/changelog-merge/main_test.go @@ -473,6 +473,30 @@ func TestMergeBudgetOutlivesRequiredGates(t *testing.T) { if executionTimeout([]string{"review"}) >= jobBudget { t.Fatal("review must finish within its CI job budget") } + // Auto Release watches the whole authoritative run, so the watch must + // outlive its slowest job, and the release job must hold the watch plus + // the guarded merge wait inside GitHub's 360-minute hosted job limit. + const queueing = 15 * time.Minute + mutationBudget := time.Duration(workflow.Jobs["mutation"].TimeoutMinutes) * time.Minute + if mutationBudget <= 0 || watchBudget < mutationBudget+queueing { + t.Fatalf("authoritative watch %v must outlive Mutation Testing's %v plus %v queueing", watchBudget, mutationBudget, queueing) + } + release, err := os.ReadFile("../../workflows/auto-release.yml") + if err != nil { + t.Fatal(err) + } + var releaseWorkflow struct { + Jobs map[string]struct { + TimeoutMinutes int `yaml:"timeout-minutes"` + } + } + if err := yaml.Unmarshal(release, &releaseWorkflow); err != nil { + t.Fatal(err) + } + releaseBudget := time.Duration(releaseWorkflow.Jobs["release"].TimeoutMinutes) * time.Minute + if releaseBudget < watchBudget+executionTimeout([]string{"enable"})+queueing || releaseBudget > 360*time.Minute { + t.Fatalf("release job %v must cover watch %v, merge wait %v and %v of steps, within 360m", releaseBudget, watchBudget, executionTimeout([]string{"enable"}), queueing) + } } func TestReadOnlyReviewNeverRequests(t *testing.T) { diff --git a/.github/scripts/changelog-merge/request_test.go b/.github/scripts/changelog-merge/request_test.go index cfae2c0f..e7e3137d 100644 --- a/.github/scripts/changelog-merge/request_test.go +++ b/.github/scripts/changelog-merge/request_test.go @@ -150,7 +150,7 @@ func TestAuthoritativeDispatchHasAnIsolatedConcurrencyLane(t *testing.T) { t.Fatal(err) } for _, required := range []string{ - `timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status`, + `timeout 240m gh run watch "$run_id" --repo "$repo" --exit-status`, `gh api --paginate "repos/${repo}/actions/runs/${run_id}/jobs?per_page=100" |`, `jq -sr '[.[].jobs[] |`, `"$gate_conclusion" != "success"`, diff --git a/.github/scripts/validate-main-ruleset.go b/.github/scripts/validate-main-ruleset.go index 8c97f045..8309bfb2 100644 --- a/.github/scripts/validate-main-ruleset.go +++ b/.github/scripts/validate-main-ruleset.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "slices" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -163,6 +164,7 @@ func validateVersionedWorkflows() { validateReleasePublication(releaseJob, autoReleaseContent) validateExistingRelease(releaseJob) validateChangelogDelivery(releaseJob, authoritativeRunContent) + validateReleaseWaitBudget(ci, releaseJob, authoritativeRunContent) fmt.Fprintln(os.Stdout, "main ruleset and release workflows are consistent") } @@ -363,6 +365,27 @@ func validateChangelogPreparation(job workflowJob, prepare workflowStep) { require(stepIndex(job, prepare.Name) < stepIndex(job, "Build cross-platform binaries"), "candidate creation must precede build and irreversible publication") } +// Auto Release waits for the dispatched changelog CI run and then for the +// guarded merge, all inside one GitHub-hosted job. +const ( + hostedJobLimitMinutes = 360 + changelogMergeWaitMinutes = 90 // changelog-merge executionTimeout for "enable" + authoritativeQueueMinutes = 15 +) + +var authoritativeWatchPattern = regexp.MustCompile(`timeout ([1-9][0-9]*)m gh run watch "\$run_id" --repo "\$repo" --exit-status`) + +func validateReleaseWaitBudget(ci workflow, releaseJob workflowJob, authoritativeRunContent string) { + match := authoritativeWatchPattern.FindStringSubmatch(authoritativeRunContent) + require(match != nil, "authoritative verifier must bound its run watch in minutes") + watch, err := strconv.Atoi(match[1]) + require(err == nil, "authoritative verifier watch must be a whole number of minutes") + mutation := ci.Jobs["mutation"].TimeoutMinutes + require(mutation > 0 && watch >= mutation+authoritativeQueueMinutes, "authoritative run watch must outlive the Mutation Testing timeout plus queueing") + release := releaseJob.TimeoutMinutes + require(release >= watch+changelogMergeWaitMinutes+authoritativeQueueMinutes && release <= hostedJobLimitMinutes, "Auto Release job timeout must cover the run watch and guarded merge within the hosted job limit") +} + func validateChangelogDelivery(releaseJob workflowJob, authoritativeRunContent string) { mergeChangelog := namedStep(releaseJob, "Queue guarded changelog auto-merge and await completion") validatePreparedChangelogDelivery(releaseJob, mergeChangelog) @@ -371,7 +394,7 @@ func validateChangelogDelivery(releaseJob workflowJob, authoritativeRunContent s require(mergeChangelog.Env["RELEASE_SHA"] == "${{ github.event.workflow_run.head_sha }}", "changelog reconciliation must retain the released commit SHA") require(strings.Contains(mergeChangelog.Run, `"repos/${GITHUB_REPOSITORY}/actions/workflows/ci.yml/dispatches"`) && strings.Contains(mergeChangelog.Run, `-f ref="$branch" --jq '.workflow_run_id'`), "changelog pull request must dispatch CI on its exact branch and capture the run ID") require(strings.Contains(mergeChangelog.Run, `bash .github/scripts/verify-authoritative-run.sh \`) && strings.Contains(mergeChangelog.Run, `"$GITHUB_REPOSITORY" "$pr_url" "$head_sha" "$ci_run"`), "changelog merge must invoke the authoritative run verifier with exact identities") - require(strings.Contains(authoritativeRunContent, `timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status`) && strings.Contains(authoritativeRunContent, `"$actual_head" != "$expected_head"`), "authoritative verifier must wait for the dispatched run and verify its head") + require(authoritativeWatchPattern.MatchString(authoritativeRunContent) && strings.Contains(authoritativeRunContent, `"$actual_head" != "$expected_head"`), "authoritative verifier must wait for the dispatched run and verify its head") require( strings.Contains(authoritativeRunContent, `gh api --paginate "repos/${repo}/actions/runs/${run_id}/jobs?per_page=100" |`) && strings.Contains(authoritativeRunContent, `jq -sr '[.[].jobs[] |`) && diff --git a/.github/scripts/validate-main-ruleset_test.go b/.github/scripts/validate-main-ruleset_test.go index 5c222b42..5b040eb8 100644 --- a/.github/scripts/validate-main-ruleset_test.go +++ b/.github/scripts/validate-main-ruleset_test.go @@ -311,6 +311,7 @@ func copyRepositoryInputs(t *testing.T, root string) { ".github/workflows/ci.yml", ".github/workflows/auto-release.yml", ".github/scripts/verify-authoritative-run.sh", + ".github/scripts/prepare-changelog-pr.sh", ".github/quality-tools.env", } { data, err := os.ReadFile("../../" + path) @@ -478,6 +479,54 @@ func TestVersionedWorkflowValidation(t *testing.T) { } } +// Auto Release's watch and job timeout must keep covering the slowest gate. +// Each edit below brings back a budget that fails a real release (#289). +func TestRepositoryValidationRejectsShortReleaseWaits(t *testing.T) { + for name, tc := range map[string]struct { + path, old, new, wantErr string + }{ + "watch shorter than mutation": { + ".github/scripts/verify-authoritative-run.sh", "timeout 240m gh run watch", "timeout 60m gh run watch", + "authoritative run watch must outlive the Mutation Testing timeout", + }, + "mutation longer than watch": { + ".github/workflows/ci.yml", " timeout-minutes: 225\n", " timeout-minutes: 300\n", + "authoritative run watch must outlive the Mutation Testing timeout", + }, + "release job without room for the merge wait": { + ".github/workflows/auto-release.yml", " timeout-minutes: 350\n", " timeout-minutes: 300\n", + "Auto Release job timeout must cover the run watch and guarded merge", + }, + "release job past the hosted limit": { + ".github/workflows/auto-release.yml", " timeout-minutes: 350\n", " timeout-minutes: 400\n", + "within the hosted job limit", + }, + } { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + copyRepositoryInputs(t, root) + data, err := os.ReadFile("../rulesets/main.json") + if err != nil { + t.Fatal(err) + } + writeTestFile(t, root+"/.github/rulesets/main.json", data) + original, err := os.ReadFile(root + "/" + tc.path) + if err != nil { + t.Fatal(err) + } + text := strings.ReplaceAll(string(original), "\r\n", "\n") + if strings.Count(text, tc.old) != 1 { + t.Fatalf("%s must contain %q exactly once", tc.path, tc.old) + } + writeTestFile(t, root+"/"+tc.path, []byte(strings.Replace(text, tc.old, tc.new, 1))) + t.Chdir(root) + if err := validateRepository(); err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("validateRepository() error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + func TestRepositoryValidationFailureReturnsError(t *testing.T) { t.Chdir(t.TempDir()) if err := validateRepository(); err == nil || !strings.Contains(err.Error(), "read .github/rulesets/main.json") { diff --git a/.github/scripts/verify-authoritative-run.sh b/.github/scripts/verify-authoritative-run.sh index 4053d107..11209b40 100644 --- a/.github/scripts/verify-authoritative-run.sh +++ b/.github/scripts/verify-authoritative-run.sh @@ -17,7 +17,8 @@ fail() { exit 1 } -if ! timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status --interval 30; then +# Outlive the slowest gate (Mutation Testing's timeout-minutes) plus queueing. +if ! timeout 240m gh run watch "$run_id" --repo "$repo" --exit-status --interval 30; then fail "stopped before a successful Quality Gate" fi if ! actual_head=$(gh api "repos/${repo}/actions/runs/${run_id}" --jq '.head_sha'); then diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 3b714943..269cbfad 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -15,6 +15,9 @@ jobs: group: auto-release cancel-in-progress: false runs-on: ubuntu-latest + # Covers the authoritative-run watch, the guarded merge wait and the + # release steps, inside GitHub's 360-minute hosted job limit. + timeout-minutes: 350 permissions: actions: write attestations: write diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b038a98..fcb7d9b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -376,10 +376,12 @@ jobs: mutation: name: Mutation Testing runs-on: ubuntu-latest - # Each covered mutant executes real package tests. The full ./src run - # took 74 minutes with 3 workers on Mini's 12 CPUs. Runtime on this - # 4-vCPU runner is unmeasured; retain room for the first hosted result. - timeout-minutes: 300 + # Each covered mutant executes real package tests. 39 successful hosted + # runs from 2026-11-06 to 2026-11-10 took 110 to 175 minutes. Auto Release + # waits for this job inside a 360-minute hosted job, so keep this limit, + # the authoritative-run watch and the release job timeout in step + # (TestMergeBudgetOutlivesRequiredGates). + timeout-minutes: 225 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0