diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8d395343a..d731ede62 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -121,19 +121,19 @@ deployments remain serialized to GitHub Pages. ### `codspeed.yml` — CodSpeed -Builds the divan benchmark targets with `cargo codspeed` and measures them -under CodSpeed's CPU simulation instrument on pull requests to `main`, pushes -to `main`, and manual dispatch. It reports performance deltas against the base -commit as evidence. It is not part of the `CI Gate` aggregate, but its external -check must still be resolved for the PR to reach the required `CLEAN` state. -Its Cargo build stays a diagnostic path next to authoritative Bazel -compilation. Comparable-run and measurement-floor triage is documented in +Runs once nightly against the exact latest `main` SHA, plus explicit manual +dispatch. Pull requests and pushes do not trigger CodSpeed. The latest +successful scheduled workflow SHA skips all nightly benchmark runners when +`main` has not changed; missing or unsuccessful prior evidence fails closed to +running the suite. Its Cargo +build stays a diagnostic path next to authoritative Bazel compilation. +Comparable-run and measurement-floor triage is documented in [`docs/development/benchmarking.md`](../../docs/development/benchmarking.md). M6 pure kernels use simulation on the ordinary pinned CI runner; durable open/recovery/commit/GC/compaction use CodSpeed's isolated bare-metal -`codspeed-macro` ARM64 runner. Weekly/manual runs -also retain exact-SHA replay and compaction peak-RSS artifacts while CodSpeed -memory mode is unavailable for this project. +`codspeed-macro` ARM64 runner. Manual runs also retain exact-SHA replay and +compaction peak-RSS artifacts while CodSpeed memory mode is unavailable for +this project. ### `binding-release-candidate.yml` diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index dc6d23d76..09be7da83 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -1,37 +1,80 @@ name: CodSpeed -# Continuous performance measurement for the Rust surface. Benchmarks are +# Nightly performance measurement for the Rust surface. Benchmarks are # divan targets built through `cargo codspeed` and measured with CodSpeed's # CPU simulation instrument for pure kernels and CodSpeed Macro Runners for # durable I/O walltime. Cargo builds here are diagnostics-only: Bazel remains # the authoritative compile/test surface (see .github/workflows/README.md). on: - push: - branches: ["main"] - pull_request: - branches: ["main"] - # Lets CodSpeed trigger a backtest run to seed the baseline. workflow_dispatch: schedule: - - cron: "17 7 * * 1" + - cron: "17 7 * * *" permissions: + actions: read contents: read # OpenID Connect authentication with CodSpeed (no long-lived token). id-token: write concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ${{ github.workflow }}-main + cancel-in-progress: false jobs: + nightly: + name: Check Latest Main + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + outputs: + sha: ${{ steps.main.outputs.sha }} + should-run: ${{ steps.decision.outputs.should-run }} + steps: + - name: Checkout latest main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + - name: Resolve latest main SHA + id: main + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Find latest successful nightly SHA + id: previous + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + previous="" + if curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/codspeed.yml/runs?branch=main&event=schedule&status=success&per_page=1" \ + --output nightly-runs.json; then + previous="$(python3 -c 'import json; data=json.load(open("nightly-runs.json", encoding="utf-8")); runs=data.get("workflow_runs", []); print(runs[0]["head_sha"] if runs else "")')" + fi + echo "sha=$previous" >> "$GITHUB_OUTPUT" + - name: Decide whether benchmarks are needed + id: decision + shell: bash + run: | + set -euo pipefail + should_run=true + if [[ "${{ github.event_name }}" == "schedule" && "${{ steps.previous.outputs.sha }}" == "${{ steps.main.outputs.sha }}" ]]; then + should_run=false + fi + echo "should-run=$should_run" >> "$GITHUB_OUTPUT" + benchmarks: name: Rust Benchmarks + needs: nightly + if: needs.nightly.outputs.should-run == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.nightly.outputs.sha }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master as of 2026-08-05 @@ -59,12 +102,16 @@ jobs: m6-walltime: name: M6 Durable Walltime (CodSpeed Macro ARM64) + needs: nightly + if: needs.nightly.outputs.should-run == 'true' # Walltime comparisons require CodSpeed's isolated bare-metal runner. # Shared hosted runners are intentionally not a fallback. runs-on: codspeed-macro timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.nightly.outputs.sha }} - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master as of 2026-08-05 with: toolchain: "1.96.0" @@ -80,11 +127,14 @@ jobs: m6-memory-fallback: name: M6 Scheduled Peak Memory Artifact - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: nightly + if: github.event_name == 'workflow_dispatch' && needs.nightly.outputs.should-run == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.nightly.outputs.sha }} - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master as of 2026-08-05 with: toolchain: "1.96.0" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 451611e32..91c6fdb57 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,6 +74,9 @@ jobs: - name: Test changed-path classifier run: scripts/ci/test-classify-changes.sh + - name: Test CodSpeed nightly policy + run: python3 scripts/ci/test-codspeed-nightly.py + - name: Test Repository Policy suite classifier run: scripts/ci/test-classify-policy-suites.sh diff --git a/scripts/ci/test-codspeed-nightly.py b/scripts/ci/test-codspeed-nightly.py new file mode 100644 index 000000000..2b4d9cc2e --- /dev/null +++ b/scripts/ci/test-codspeed-nightly.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Regression tests for the nightly-only CodSpeed workflow.""" + +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/codspeed.yml" + +workflow = yaml.load(WORKFLOW.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) +triggers = workflow["on"] +assert "pull_request" not in triggers +assert "push" not in triggers +assert "workflow_dispatch" in triggers +assert triggers["schedule"] == [{"cron": "17 7 * * *"}] + +jobs = workflow["jobs"] +assert workflow["permissions"]["actions"] == "read" +nightly = jobs["nightly"] +assert nightly["outputs"]["sha"] == "${{ steps.main.outputs.sha }}" +assert nightly["outputs"]["should-run"] == "${{ steps.decision.outputs.should-run }}" +checkout = nightly["steps"][0] +assert checkout["with"]["ref"] == "main" +previous = next(step for step in nightly["steps"] if step.get("id") == "previous")["run"] +assert "actions/workflows/codspeed.yml/runs" in previous +assert "event=schedule" in previous +assert "status=success" in previous +assert 'previous=""' in previous +decision = next(step for step in nightly["steps"] if step.get("id") == "decision")["run"] +assert 'github.event_name }}" == "schedule"' in decision +assert "steps.previous.outputs.sha" in decision +assert "steps.main.outputs.sha" in decision +assert "should_run=true" in decision +assert "should_run=false" in decision + +for job_name in ("benchmarks", "m6-walltime"): + job = jobs[job_name] + assert job["needs"] == "nightly" + assert job["if"] == "needs.nightly.outputs.should-run == 'true'" + checkout = next(step for step in job["steps"] if "actions/checkout@" in step.get("uses", "")) + assert checkout["with"]["ref"] == "${{ needs.nightly.outputs.sha }}" + +memory = jobs["m6-memory-fallback"] +assert memory["if"] == ( + "github.event_name == 'workflow_dispatch' && needs.nightly.outputs.should-run == 'true'" +) + +print("CodSpeed nightly-only policy verified")