ci: fix the #7205 relapse on the scheduled gate arm, and reap dead PR runs (#7966)

.github/workflows/auto-opt-app-patterns.yml

@@ -63,7 +63,19 @@ permissions:
   contents: read
 
 concurrency:
-  group: auto-opt-app-patterns-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: auto-opt-app-patterns-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:

.github/workflows/benchmark.yml

@@ -13,7 +13,12 @@ on:
 
 # Don't cancel in-progress benchmark runs — we want complete samples
 concurrency:
-  group: bench-${{ github.ref }}
+  # #7966: keyed per RUN for non-PR events. A group that is constant across
+  # scheduled runs lets only the first one execute -- GitHub keeps at most one
+  # PENDING run per group and cancels the rest with `jobs: 0`, regardless of
+  # `cancel-in-progress`. PR runs keep the shared per-ref group so superseded
+  # pushes still coalesce. Enforced by scripts/gc_gate_wiring_check.py.
+  group: bench-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: false
 
 env:

.github/workflows/ci-queue-reaper.yml

@@ -0,0 +1,86 @@
+name: CI Queue Reaper
+
+# Cancels QUEUED pull-request runs whose pull request is already closed.
+#
+# WHY: GitHub does not reliably cancel a queued run when its PR merges and the
+# branch auto-deletes. Perry squash-merges, auto-deletes branches, and fans each
+# PR out to ~11 workflows, so a busy day leaves hundreds of runs queued against
+# branches that no longer exist. They cannot gate anything -- the code is
+# already in `main` -- but they hold runner slots ahead of the six-hourly `main`
+# gates, which is how those gates go dark.
+#
+# Measured 2026-08-12 (#7966): 1,529 queued runs; 794 were `pull_request` runs
+# across 63 head branches, of which 61 no longer existed. ~790 runs -- 51% of
+# the whole queue -- were dead work sitting in front of ten `main` gates that
+# had not completed in 32+ hours.
+#
+# THIS WORKFLOW IS NOT A GATE. It cannot fail a merge and is not a required
+# context. It is a janitor. The policy it enforces, its guard rails and its
+# self-test live in scripts/reap_stale_ci_runs.py -- read that before changing
+# anything here. In particular it only ever touches `event == pull_request`
+# runs in state `queued` whose branch has no OPEN PR, so a `push`, `schedule`,
+# tag or dispatch run is structurally out of reach.
+#
+# BOOTSTRAP NOTE: this job queues like everything else, so it cannot dig the
+# repo out of an already-saturated queue on its own. The first drain is a
+# manual `python3 scripts/reap_stale_ci_runs.py --apply` (or a
+# `workflow_dispatch` of this workflow with `apply=true`); the schedule then
+# keeps the queue clear.
+
+on:
+  schedule:
+    # Every 30 minutes, off the hour and half-hour -- :00 and :30 are the most
+    # contended slots on GitHub's cron scheduler and the most likely to be
+    # dropped or delayed.
+    - cron: "13,43 * * * *"
+  workflow_dispatch:
+    inputs:
+      apply:
+        description: "Actually cancel (unchecked = dry run)"
+        type: boolean
+        default: false
+
+permissions: {}
+
+concurrency:
+  # #7966: keyed per RUN. A constant group across scheduled runs lets only the
+  # first one execute -- GitHub keeps at most one PENDING run per group and
+  # cancels the rest with `jobs: 0`, regardless of `cancel-in-progress`. That is
+  # the exact bug this workflow exists to clean up after; reproducing it here
+  # would be its own joke. Enforced by scripts/gc_gate_wiring_check.py.
+  group: ci-queue-reaper-${{ github.run_id }}
+  cancel-in-progress: false
+
+jobs:
+  reap:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      # Cancelling a run is an Actions write. `pull-requests: read` backs the
+      # open-PR list that protects every live PR from being reaped.
+      actions: write
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      # The policy check runs unconditionally, including on a dry run. A janitor
+      # that cancels things must be able to prove its guard rails still bind
+      # before it is allowed to touch the queue.
+      - name: Self-test the reaping policy
+        run: python3 scripts/reap_stale_ci_runs.py --self-test
+
+      - name: Reap stale queued pull-request runs
+        env:
+          GH_TOKEN: ${{ github.token }}
+          # Scheduled sweeps apply; a manual dispatch applies only if asked.
+          APPLY: ${{ github.event_name == 'schedule' || inputs.apply }}
+        run: |
+          set -euo pipefail
+          if [[ "$APPLY" == "true" ]]; then
+            python3 scripts/reap_stale_ci_runs.py --apply
+          else
+            python3 scripts/reap_stale_ci_runs.py
+          fi

.github/workflows/container-tests.yml

@@ -63,7 +63,12 @@ on:
         options: ["true", "false"]
 
 concurrency:
-  group: container-tests-${{ github.ref }}
+  # #7966: keyed per RUN for non-PR events. A group that is constant across
+  # scheduled runs lets only the first one execute -- GitHub keeps at most one
+  # PENDING run per group and cancels the rest with `jobs: 0`, regardless of
+  # `cancel-in-progress`. PR runs keep the shared per-ref group so superseded
+  # pushes still coalesce. Enforced by scripts/gc_gate_wiring_check.py.
+  group: container-tests-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: true
 
 env:

.github/workflows/coverage.yml

@@ -16,7 +16,12 @@ permissions:
   contents: read
 
 concurrency:
-  group: coverage-${{ github.ref }}
+  # #7966: keyed per RUN for non-PR events. A group that is constant across
+  # scheduled runs lets only the first one execute -- GitHub keeps at most one
+  # PENDING run per group and cancels the rest with `jobs: 0`, regardless of
+  # `cancel-in-progress`. PR runs keep the shared per-ref group so superseded
+  # pushes still coalesce. Enforced by scripts/gc_gate_wiring_check.py.
+  group: coverage-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: true
 
 env:

.github/workflows/eh-transport.yml

@@ -41,7 +41,19 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: eh-transport-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: eh-transport-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   # One group per main COMMIT, cancelling PR runs only. `cancel-in-progress:
   # false` alone does not protect a `main` run: GitHub allows at most one
   # PENDING run per group and cancels the previously pending one when a new run

.github/workflows/feature-matrix.yml

@@ -18,7 +18,12 @@ permissions:
   contents: read
 
 concurrency:
-  group: feature-matrix-${{ github.ref }}
+  # #7966: keyed per RUN for non-PR events. A group that is constant across
+  # scheduled runs lets only the first one execute -- GitHub keeps at most one
+  # PENDING run per group and cancels the rest with `jobs: 0`, regardless of
+  # `cancel-in-progress`. PR runs keep the shared per-ref group so superseded
+  # pushes still coalesce. Enforced by scripts/gc_gate_wiring_check.py.
+  group: feature-matrix-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: false
 
 env:

.github/workflows/gate-freshness.yml

@@ -43,7 +43,19 @@ concurrency:
   # One sweep at a time. Unlike the gates this watches, coalescing is correct here:
   # the freshness verdict is a function of "now", so a superseded run had nothing
   # unique to say. PR runs supersede themselves; scheduled runs queue.
-  group: gate-freshness-${{ github.event_name }}-${{ github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gate-freshness-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:

.github/workflows/gc-moving-witnesses.yml

@@ -130,7 +130,19 @@ concurrency:
   # enters, regardless of that setting. That is #7205, measured on gc-ratchet —
   # three consecutive `main` runs cancelled with `jobs: []`, zero executions.
   # Keying push runs on the SHA gives every merged commit a group of its own.
-  group: gc-moving-witnesses-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gc-moving-witnesses-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:

.github/workflows/gc-native-roots.yml

@@ -127,7 +127,19 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: gc-native-roots-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gc-native-roots-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   # Same shape as llvm-inprocess (#7357), and for the same measured reason.
   #
   # This workflow had NO concurrency group at all, so nothing ever superseded a

.github/workflows/gc-parse-churn-gate.yml

@@ -97,7 +97,19 @@ concurrency:
   # cancel a pending `main` run the moment a new one enters the SAME group --
   # #7205, measured on gc-ratchet with three consecutive `main` runs
   # cancelled and zero executed). PR runs are cancelled on superseding pushes.
-  group: gc-parse-churn-gate-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gc-parse-churn-gate-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:

.github/workflows/gc-ptr-shape-off-witness.yml

@@ -92,7 +92,19 @@ permissions:
 concurrency:
   # One group per main COMMIT, cancelling PR runs only — see
   # gc-moving-witnesses.yml's comment for the #7205 rationale this mirrors.
-  group: gc-ptr-shape-off-witness-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gc-ptr-shape-off-witness-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:

.github/workflows/gc-ratchet.yml

@@ -77,7 +77,19 @@ concurrency:
   # eventually gets its own answer. PR runs keep sharing a per-ref group and
   # keep superseding themselves, which is still correct — only the head
   # commit's result gates the merge.
-  group: gc-ratchet-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gc-ratchet-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:

.github/workflows/gc-root-dominance.yml

@@ -88,7 +88,19 @@ concurrency:
   # gc-ratchet, whose shape this file copied). Keying push runs on the SHA gives
   # every merged commit a group of its own, so no two main runs can contend.
   # Same reasoning as gc-ratchet.yml, which carries the full writeup.
-  group: gc-root-dominance-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  # ***#7966: KEY EVERY MAIN-LINE RUN ON `github.run_id`, NOT `github.sha`.***
+  # The previous expression read `github.event_name == 'push' && github.sha ||
+  # github.ref`. That was #7205's fix and it keyed on the event being `push` --
+  # correct while the main-line arm WAS `push: branches: [main]`. #7856 moved the
+  # main-line arm to `schedule:`, which falls through to `github.ref` (constant
+  # `refs/heads/main`), so every scheduled run shared one group again and #7205
+  # came straight back. Measured 2026-08-12 on all ten scheduled gates, the same
+  # shape every time: oldest run `queued` holding the group, the two after it
+  # `cancelled` with `jobs: 0`, newest `pending`. `github.run_id` is unique per
+  # run, so schedule / tag-push / workflow_dispatch each get a group of their own
+  # and none can supersede another. PR runs keep the shared per-ref group and
+  # keep superseding themselves, which is still what we want.
+  group: gc-root-dominance-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env: