From f6fa199c598f16e181b89b2a907296a3700c81e9 Mon Sep 17 00:00:00 2001 From: HemSoft <8227352+HemSoft@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:35:11 -0500 Subject: [PATCH] fix: let Auto Release wait for the full authoritative CI run Auto Release watched the dispatched changelog CI run for only 60 minutes, while Mutation Testing takes 110 to 175 minutes on hosted runners (39 successful runs, 2026-11-06 to 2026-11-10). Every release since the per-mutant test change failed its first attempt and needed a manual docs-PR merge plus a rerun. - verify-authoritative-run.sh watches for 240 minutes. - Mutation Testing's timeout drops from 300 to 225 minutes, 50 above the slowest measured run, so the watch outlives it. - The release job gets an explicit 350-minute timeout: the watch plus the 90-minute guarded merge wait and the earlier steps, inside GitHub's 360-minute hosted job limit. - validate-main-ruleset and TestMergeBudgetOutlivesRequiredGates tie the three values together, and new validator cases reject each regression. Closes #289 Co-Authored-By: Claude Opus 5.5 --- .github/scripts/changelog-merge/main_test.go | 24 +++++++++ .../scripts/changelog-merge/request_test.go | 2 +- .github/scripts/validate-main-ruleset.go | 25 +++++++++- .github/scripts/validate-main-ruleset_test.go | 49 +++++++++++++++++++ .github/scripts/verify-authoritative-run.sh | 3 +- .github/workflows/auto-release.yml | 3 ++ .github/workflows/ci.yml | 10 ++-- 7 files changed, 109 insertions(+), 7 deletions(-) diff --git a/.github/scripts/changelog-merge/main_test.go b/.github/scripts/changelog-merge/main_test.go index a6edd15b..767966a4 100644 --- a/.github/scripts/changelog-merge/main_test.go +++ b/.github/scripts/changelog-merge/main_test.go @@ -473,6 +473,30 @@ func TestMergeBudgetOutlivesRequiredGates(t *testing.T) { if executionTimeout([]string{"review"}) >= jobBudget { t.Fatal("review must finish within its CI job budget") } + // Auto Release watches the whole authoritative run, so the watch must + // outlive its slowest job, and the release job must hold the watch plus + // the guarded merge wait inside GitHub's 360-minute hosted job limit. + const queueing = 15 * time.Minute + mutationBudget := time.Duration(workflow.Jobs["mutation"].TimeoutMinutes) * time.Minute + if mutationBudget <= 0 || watchBudget < mutationBudget+queueing { + t.Fatalf("authoritative watch %v must outlive Mutation Testing's %v plus %v queueing", watchBudget, mutationBudget, queueing) + } + release, err := os.ReadFile("../../workflows/auto-release.yml") + if err != nil { + t.Fatal(err) + } + var releaseWorkflow struct { + Jobs map[string]struct { + TimeoutMinutes int `yaml:"timeout-minutes"` + } + } + if err := yaml.Unmarshal(release, &releaseWorkflow); err != nil { + t.Fatal(err) + } + releaseBudget := time.Duration(releaseWorkflow.Jobs["release"].TimeoutMinutes) * time.Minute + if releaseBudget < watchBudget+executionTimeout([]string{"enable"})+queueing || releaseBudget > 360*time.Minute { + t.Fatalf("release job %v must cover watch %v, merge wait %v and %v of steps, within 360m", releaseBudget, watchBudget, executionTimeout([]string{"enable"}), queueing) + } } func TestReadOnlyReviewNeverRequests(t *testing.T) { diff --git a/.github/scripts/changelog-merge/request_test.go b/.github/scripts/changelog-merge/request_test.go index cfae2c0f..e7e3137d 100644 --- a/.github/scripts/changelog-merge/request_test.go +++ b/.github/scripts/changelog-merge/request_test.go @@ -150,7 +150,7 @@ func TestAuthoritativeDispatchHasAnIsolatedConcurrencyLane(t *testing.T) { t.Fatal(err) } for _, required := range []string{ - `timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status`, + `timeout 240m gh run watch "$run_id" --repo "$repo" --exit-status`, `gh api --paginate "repos/${repo}/actions/runs/${run_id}/jobs?per_page=100" |`, `jq -sr '[.[].jobs[] |`, `"$gate_conclusion" != "success"`, diff --git a/.github/scripts/validate-main-ruleset.go b/.github/scripts/validate-main-ruleset.go index 8c97f045..8309bfb2 100644 --- a/.github/scripts/validate-main-ruleset.go +++ b/.github/scripts/validate-main-ruleset.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "slices" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -163,6 +164,7 @@ func validateVersionedWorkflows() { validateReleasePublication(releaseJob, autoReleaseContent) validateExistingRelease(releaseJob) validateChangelogDelivery(releaseJob, authoritativeRunContent) + validateReleaseWaitBudget(ci, releaseJob, authoritativeRunContent) fmt.Fprintln(os.Stdout, "main ruleset and release workflows are consistent") } @@ -363,6 +365,27 @@ func validateChangelogPreparation(job workflowJob, prepare workflowStep) { require(stepIndex(job, prepare.Name) < stepIndex(job, "Build cross-platform binaries"), "candidate creation must precede build and irreversible publication") } +// Auto Release waits for the dispatched changelog CI run and then for the +// guarded merge, all inside one GitHub-hosted job. +const ( + hostedJobLimitMinutes = 360 + changelogMergeWaitMinutes = 90 // changelog-merge executionTimeout for "enable" + authoritativeQueueMinutes = 15 +) + +var authoritativeWatchPattern = regexp.MustCompile(`timeout ([1-9][0-9]*)m gh run watch "\$run_id" --repo "\$repo" --exit-status`) + +func validateReleaseWaitBudget(ci workflow, releaseJob workflowJob, authoritativeRunContent string) { + match := authoritativeWatchPattern.FindStringSubmatch(authoritativeRunContent) + require(match != nil, "authoritative verifier must bound its run watch in minutes") + watch, err := strconv.Atoi(match[1]) + require(err == nil, "authoritative verifier watch must be a whole number of minutes") + mutation := ci.Jobs["mutation"].TimeoutMinutes + require(mutation > 0 && watch >= mutation+authoritativeQueueMinutes, "authoritative run watch must outlive the Mutation Testing timeout plus queueing") + release := releaseJob.TimeoutMinutes + require(release >= watch+changelogMergeWaitMinutes+authoritativeQueueMinutes && release <= hostedJobLimitMinutes, "Auto Release job timeout must cover the run watch and guarded merge within the hosted job limit") +} + func validateChangelogDelivery(releaseJob workflowJob, authoritativeRunContent string) { mergeChangelog := namedStep(releaseJob, "Queue guarded changelog auto-merge and await completion") validatePreparedChangelogDelivery(releaseJob, mergeChangelog) @@ -371,7 +394,7 @@ func validateChangelogDelivery(releaseJob workflowJob, authoritativeRunContent s require(mergeChangelog.Env["RELEASE_SHA"] == "${{ github.event.workflow_run.head_sha }}", "changelog reconciliation must retain the released commit SHA") require(strings.Contains(mergeChangelog.Run, `"repos/${GITHUB_REPOSITORY}/actions/workflows/ci.yml/dispatches"`) && strings.Contains(mergeChangelog.Run, `-f ref="$branch" --jq '.workflow_run_id'`), "changelog pull request must dispatch CI on its exact branch and capture the run ID") require(strings.Contains(mergeChangelog.Run, `bash .github/scripts/verify-authoritative-run.sh \`) && strings.Contains(mergeChangelog.Run, `"$GITHUB_REPOSITORY" "$pr_url" "$head_sha" "$ci_run"`), "changelog merge must invoke the authoritative run verifier with exact identities") - require(strings.Contains(authoritativeRunContent, `timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status`) && strings.Contains(authoritativeRunContent, `"$actual_head" != "$expected_head"`), "authoritative verifier must wait for the dispatched run and verify its head") + require(authoritativeWatchPattern.MatchString(authoritativeRunContent) && strings.Contains(authoritativeRunContent, `"$actual_head" != "$expected_head"`), "authoritative verifier must wait for the dispatched run and verify its head") require( strings.Contains(authoritativeRunContent, `gh api --paginate "repos/${repo}/actions/runs/${run_id}/jobs?per_page=100" |`) && strings.Contains(authoritativeRunContent, `jq -sr '[.[].jobs[] |`) && diff --git a/.github/scripts/validate-main-ruleset_test.go b/.github/scripts/validate-main-ruleset_test.go index 5c222b42..5b040eb8 100644 --- a/.github/scripts/validate-main-ruleset_test.go +++ b/.github/scripts/validate-main-ruleset_test.go @@ -311,6 +311,7 @@ func copyRepositoryInputs(t *testing.T, root string) { ".github/workflows/ci.yml", ".github/workflows/auto-release.yml", ".github/scripts/verify-authoritative-run.sh", + ".github/scripts/prepare-changelog-pr.sh", ".github/quality-tools.env", } { data, err := os.ReadFile("../../" + path) @@ -478,6 +479,54 @@ func TestVersionedWorkflowValidation(t *testing.T) { } } +// Auto Release's watch and job timeout must keep covering the slowest gate. +// Each edit below brings back a budget that fails a real release (#289). +func TestRepositoryValidationRejectsShortReleaseWaits(t *testing.T) { + for name, tc := range map[string]struct { + path, old, new, wantErr string + }{ + "watch shorter than mutation": { + ".github/scripts/verify-authoritative-run.sh", "timeout 240m gh run watch", "timeout 60m gh run watch", + "authoritative run watch must outlive the Mutation Testing timeout", + }, + "mutation longer than watch": { + ".github/workflows/ci.yml", " timeout-minutes: 225\n", " timeout-minutes: 300\n", + "authoritative run watch must outlive the Mutation Testing timeout", + }, + "release job without room for the merge wait": { + ".github/workflows/auto-release.yml", " timeout-minutes: 350\n", " timeout-minutes: 300\n", + "Auto Release job timeout must cover the run watch and guarded merge", + }, + "release job past the hosted limit": { + ".github/workflows/auto-release.yml", " timeout-minutes: 350\n", " timeout-minutes: 400\n", + "within the hosted job limit", + }, + } { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + copyRepositoryInputs(t, root) + data, err := os.ReadFile("../rulesets/main.json") + if err != nil { + t.Fatal(err) + } + writeTestFile(t, root+"/.github/rulesets/main.json", data) + original, err := os.ReadFile(root + "/" + tc.path) + if err != nil { + t.Fatal(err) + } + text := strings.ReplaceAll(string(original), "\r\n", "\n") + if strings.Count(text, tc.old) != 1 { + t.Fatalf("%s must contain %q exactly once", tc.path, tc.old) + } + writeTestFile(t, root+"/"+tc.path, []byte(strings.Replace(text, tc.old, tc.new, 1))) + t.Chdir(root) + if err := validateRepository(); err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("validateRepository() error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + func TestRepositoryValidationFailureReturnsError(t *testing.T) { t.Chdir(t.TempDir()) if err := validateRepository(); err == nil || !strings.Contains(err.Error(), "read .github/rulesets/main.json") { diff --git a/.github/scripts/verify-authoritative-run.sh b/.github/scripts/verify-authoritative-run.sh index 4053d107..11209b40 100644 --- a/.github/scripts/verify-authoritative-run.sh +++ b/.github/scripts/verify-authoritative-run.sh @@ -17,7 +17,8 @@ fail() { exit 1 } -if ! timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status --interval 30; then +# Outlive the slowest gate (Mutation Testing's timeout-minutes) plus queueing. +if ! timeout 240m gh run watch "$run_id" --repo "$repo" --exit-status --interval 30; then fail "stopped before a successful Quality Gate" fi if ! actual_head=$(gh api "repos/${repo}/actions/runs/${run_id}" --jq '.head_sha'); then diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 3b714943..269cbfad 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -15,6 +15,9 @@ jobs: group: auto-release cancel-in-progress: false runs-on: ubuntu-latest + # Covers the authoritative-run watch, the guarded merge wait and the + # release steps, inside GitHub's 360-minute hosted job limit. + timeout-minutes: 350 permissions: actions: write attestations: write diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b038a98..fcb7d9b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -376,10 +376,12 @@ jobs: mutation: name: Mutation Testing runs-on: ubuntu-latest - # Each covered mutant executes real package tests. The full ./src run - # took 74 minutes with 3 workers on Mini's 12 CPUs. Runtime on this - # 4-vCPU runner is unmeasured; retain room for the first hosted result. - timeout-minutes: 300 + # Each covered mutant executes real package tests. 39 successful hosted + # runs from 2026-11-06 to 2026-11-10 took 110 to 175 minutes. Auto Release + # waits for this job inside a 360-minute hosted job, so keep this limit, + # the authoritative-run watch and the release job timeout in step + # (TestMergeBudgetOutlivesRequiredGates). + timeout-minutes: 225 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0