Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/scripts/changelog-merge/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,30 @@ func TestMergeBudgetOutlivesRequiredGates(t *testing.T) {
if executionTimeout([]string{"review"}) >= jobBudget {
t.Fatal("review must finish within its CI job budget")
}
// Auto Release watches the whole authoritative run, so the watch must
// outlive its slowest job, and the release job must hold the watch plus
// the guarded merge wait inside GitHub's 360-minute hosted job limit.
const queueing = 15 * time.Minute
mutationBudget := time.Duration(workflow.Jobs["mutation"].TimeoutMinutes) * time.Minute
if mutationBudget <= 0 || watchBudget < mutationBudget+queueing {
t.Fatalf("authoritative watch %v must outlive Mutation Testing's %v plus %v queueing", watchBudget, mutationBudget, queueing)
}
release, err := os.ReadFile("../../workflows/auto-release.yml")
if err != nil {
t.Fatal(err)
}
var releaseWorkflow struct {
Jobs map[string]struct {
TimeoutMinutes int `yaml:"timeout-minutes"`
}
}
if err := yaml.Unmarshal(release, &releaseWorkflow); err != nil {
t.Fatal(err)
}
releaseBudget := time.Duration(releaseWorkflow.Jobs["release"].TimeoutMinutes) * time.Minute
if releaseBudget < watchBudget+executionTimeout([]string{"enable"})+queueing || releaseBudget > 360*time.Minute {
t.Fatalf("release job %v must cover watch %v, merge wait %v and %v of steps, within 360m", releaseBudget, watchBudget, executionTimeout([]string{"enable"}), queueing)
}
}

func TestReadOnlyReviewNeverRequests(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/changelog-merge/request_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func TestAuthoritativeDispatchHasAnIsolatedConcurrencyLane(t *testing.T) {
t.Fatal(err)
}
for _, required := range []string{
`timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status`,
`timeout 240m gh run watch "$run_id" --repo "$repo" --exit-status`,
`gh api --paginate "repos/${repo}/actions/runs/${run_id}/jobs?per_page=100" |`,
`jq -sr '[.[].jobs[] |`,
`"$gate_conclusion" != "success"`,
Expand Down
25 changes: 24 additions & 1 deletion .github/scripts/validate-main-ruleset.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"reflect"
"regexp"
"slices"
"strconv"
"strings"

"gopkg.in/yaml.v3"
Expand Down Expand Up @@ -163,6 +164,7 @@ func validateVersionedWorkflows() {
validateReleasePublication(releaseJob, autoReleaseContent)
validateExistingRelease(releaseJob)
validateChangelogDelivery(releaseJob, authoritativeRunContent)
validateReleaseWaitBudget(ci, releaseJob, authoritativeRunContent)
fmt.Fprintln(os.Stdout, "main ruleset and release workflows are consistent")
}

Expand Down Expand Up @@ -363,6 +365,27 @@ func validateChangelogPreparation(job workflowJob, prepare workflowStep) {
require(stepIndex(job, prepare.Name) < stepIndex(job, "Build cross-platform binaries"), "candidate creation must precede build and irreversible publication")
}

// Auto Release waits for the dispatched changelog CI run and then for the
// guarded merge, all inside one GitHub-hosted job.
const (
hostedJobLimitMinutes = 360
changelogMergeWaitMinutes = 90 // changelog-merge executionTimeout for "enable"
authoritativeQueueMinutes = 15
)

var authoritativeWatchPattern = regexp.MustCompile(`timeout ([1-9][0-9]*)m gh run watch "\$run_id" --repo "\$repo" --exit-status`)

func validateReleaseWaitBudget(ci workflow, releaseJob workflowJob, authoritativeRunContent string) {
match := authoritativeWatchPattern.FindStringSubmatch(authoritativeRunContent)
require(match != nil, "authoritative verifier must bound its run watch in minutes")
watch, err := strconv.Atoi(match[1])
require(err == nil, "authoritative verifier watch must be a whole number of minutes")
mutation := ci.Jobs["mutation"].TimeoutMinutes
require(mutation > 0 && watch >= mutation+authoritativeQueueMinutes, "authoritative run watch must outlive the Mutation Testing timeout plus queueing")
release := releaseJob.TimeoutMinutes
require(release >= watch+changelogMergeWaitMinutes+authoritativeQueueMinutes && release <= hostedJobLimitMinutes, "Auto Release job timeout must cover the run watch and guarded merge within the hosted job limit")
}

func validateChangelogDelivery(releaseJob workflowJob, authoritativeRunContent string) {
mergeChangelog := namedStep(releaseJob, "Queue guarded changelog auto-merge and await completion")
validatePreparedChangelogDelivery(releaseJob, mergeChangelog)
Expand All @@ -371,7 +394,7 @@ func validateChangelogDelivery(releaseJob workflowJob, authoritativeRunContent s
require(mergeChangelog.Env["RELEASE_SHA"] == "${{ github.event.workflow_run.head_sha }}", "changelog reconciliation must retain the released commit SHA")
require(strings.Contains(mergeChangelog.Run, `"repos/${GITHUB_REPOSITORY}/actions/workflows/ci.yml/dispatches"`) && strings.Contains(mergeChangelog.Run, `-f ref="$branch" --jq '.workflow_run_id'`), "changelog pull request must dispatch CI on its exact branch and capture the run ID")
require(strings.Contains(mergeChangelog.Run, `bash .github/scripts/verify-authoritative-run.sh \`) && strings.Contains(mergeChangelog.Run, `"$GITHUB_REPOSITORY" "$pr_url" "$head_sha" "$ci_run"`), "changelog merge must invoke the authoritative run verifier with exact identities")
require(strings.Contains(authoritativeRunContent, `timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status`) && strings.Contains(authoritativeRunContent, `"$actual_head" != "$expected_head"`), "authoritative verifier must wait for the dispatched run and verify its head")
require(authoritativeWatchPattern.MatchString(authoritativeRunContent) && strings.Contains(authoritativeRunContent, `"$actual_head" != "$expected_head"`), "authoritative verifier must wait for the dispatched run and verify its head")
require(
strings.Contains(authoritativeRunContent, `gh api --paginate "repos/${repo}/actions/runs/${run_id}/jobs?per_page=100" |`) &&
strings.Contains(authoritativeRunContent, `jq -sr '[.[].jobs[] |`) &&
Expand Down
49 changes: 49 additions & 0 deletions .github/scripts/validate-main-ruleset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ func copyRepositoryInputs(t *testing.T, root string) {
".github/workflows/ci.yml",
".github/workflows/auto-release.yml",
".github/scripts/verify-authoritative-run.sh",
".github/scripts/prepare-changelog-pr.sh",
".github/quality-tools.env",
} {
data, err := os.ReadFile("../../" + path)
Expand Down Expand Up @@ -478,6 +479,54 @@ func TestVersionedWorkflowValidation(t *testing.T) {
}
}

// Auto Release's watch and job timeout must keep covering the slowest gate.
// Each edit below brings back a budget that fails a real release (#289).
func TestRepositoryValidationRejectsShortReleaseWaits(t *testing.T) {
for name, tc := range map[string]struct {
path, old, new, wantErr string
}{
"watch shorter than mutation": {
".github/scripts/verify-authoritative-run.sh", "timeout 240m gh run watch", "timeout 60m gh run watch",
"authoritative run watch must outlive the Mutation Testing timeout",
},
"mutation longer than watch": {
".github/workflows/ci.yml", " timeout-minutes: 225\n", " timeout-minutes: 300\n",
"authoritative run watch must outlive the Mutation Testing timeout",
},
"release job without room for the merge wait": {
".github/workflows/auto-release.yml", " timeout-minutes: 350\n", " timeout-minutes: 300\n",
"Auto Release job timeout must cover the run watch and guarded merge",
},
"release job past the hosted limit": {
".github/workflows/auto-release.yml", " timeout-minutes: 350\n", " timeout-minutes: 400\n",
"within the hosted job limit",
},
} {
t.Run(name, func(t *testing.T) {
root := t.TempDir()
copyRepositoryInputs(t, root)
data, err := os.ReadFile("../rulesets/main.json")
if err != nil {
t.Fatal(err)
}
writeTestFile(t, root+"/.github/rulesets/main.json", data)
original, err := os.ReadFile(root + "/" + tc.path)
if err != nil {
t.Fatal(err)
}
text := strings.ReplaceAll(string(original), "\r\n", "\n")
if strings.Count(text, tc.old) != 1 {
t.Fatalf("%s must contain %q exactly once", tc.path, tc.old)
}
writeTestFile(t, root+"/"+tc.path, []byte(strings.Replace(text, tc.old, tc.new, 1)))
t.Chdir(root)
if err := validateRepository(); err == nil || !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("validateRepository() error = %v, want substring %q", err, tc.wantErr)
}
})
}
}

func TestRepositoryValidationFailureReturnsError(t *testing.T) {
t.Chdir(t.TempDir())
if err := validateRepository(); err == nil || !strings.Contains(err.Error(), "read .github/rulesets/main.json") {
Expand Down
3 changes: 2 additions & 1 deletion .github/scripts/verify-authoritative-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ fail() {
exit 1
}

if ! timeout 60m gh run watch "$run_id" --repo "$repo" --exit-status --interval 30; then
# Outlive the slowest gate (Mutation Testing's timeout-minutes) plus queueing.
if ! timeout 240m gh run watch "$run_id" --repo "$repo" --exit-status --interval 30; then
fail "stopped before a successful Quality Gate"
fi
if ! actual_head=$(gh api "repos/${repo}/actions/runs/${run_id}" --jq '.head_sha'); then
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/auto-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ jobs:
group: auto-release
cancel-in-progress: false
runs-on: ubuntu-latest
# Covers the authoritative-run watch, the guarded merge wait and the
# release steps, inside GitHub's 360-minute hosted job limit.
timeout-minutes: 350
permissions:
actions: write
attestations: write
Expand Down
10 changes: 6 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -376,10 +376,12 @@ jobs:
mutation:
name: Mutation Testing
runs-on: ubuntu-latest
# Each covered mutant executes real package tests. The full ./src run
# took 74 minutes with 3 workers on Mini's 12 CPUs. Runtime on this
# 4-vCPU runner is unmeasured; retain room for the first hosted result.
timeout-minutes: 300
# Each covered mutant executes real package tests. 39 successful hosted
# runs from 2026-11-06 to 2026-11-10 took 110 to 175 minutes. Auto Release
# waits for this job inside a 360-minute hosted job, so keep this limit,
# the authoritative-run watch and the release job timeout in step
# (TestMergeBudgetOutlivesRequiredGates).
timeout-minutes: 225
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
Expand Down
Loading