From 5973bbe6b961f0091d7cc1aa956958bc9e943b56 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 19 Oct 2026 17:53:49 +0700 Subject: [PATCH] ci(release): check the tag's own CI runs instead of walking and polling --- .claude/rules/ci.md | 16 +- .claude/skills/release/SKILL.md | 2 +- .github/workflows/ios-tests.yml | 5 +- .github/workflows/macos-tests.yml | 5 +- .github/workflows/release-build.yml | 36 +-- .github/workflows/release.yml | 3 +- .github/workflows/repo-hygiene.yml | 3 +- apple/RELEASING.md | 17 +- apple/scripts/release/test_version.py | 316 +++++++++++------------- apple/scripts/release/version.py | 164 +++++------- scripts/ci/detect-changed-paths.sh | 38 ++- scripts/ci/test_detect_changed_paths.py | 60 ++++- 12 files changed, 332 insertions(+), 333 deletions(-) diff --git a/.claude/rules/ci.md b/.claude/rules/ci.md index 3d3e1e78bb..7c1ac2d94f 100644 --- a/.claude/rules/ci.md +++ b/.claude/rules/ci.md @@ -66,10 +66,10 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ ## Release Build (`release-build.yml`) -- Nightly at 19:00 UTC on main, push to `release/**`, and dispatch; never a PR. Never cancelled. Not a required check. -- `Changes (Release)` outputs `run`. A dispatch always builds. A nightly runs the `release` detector against the head of the last nightly that ended green, read with `gh api`, so a quiet day builds nothing. A `release/**` push diffs `github.event.before`, and builds when the push created the branch. -- `Release Build` runs on the self-hosted Mac, or on `xcode-27` when a dispatch picks `runner: github` or, for a push or nightly, the repository variable `RELEASE_BUILD_RUNNER` is `github`. It has a 150-minute limit and VERSION 0.0.0-dev and BUILD 9999999, so build.sh passes the version settings a tag passes. -- While the Mac is offline a run waits for it, and GitHub cancels a job still queued after 24 hours. The tag gate refuses a commit with any red run, and a `runner: github` dispatch on the same commit does not replace it: `apple/RELEASING.md`, When a run fails. +- Nightly at 19:00 UTC on main and dispatch; never a PR or a push. Never cancelled. Not a required check, and no tag waits for it. +- `Changes (Release)` outputs `run`. A dispatch always builds. A nightly runs the `release` detector against the head of the last nightly that ended green, read with `gh api`, so a quiet day builds nothing. +- `Release Build` runs on the self-hosted Mac, or on `xcode-27` when a dispatch picks `runner: github` or, for a nightly, the repository variable `RELEASE_BUILD_RUNNER` is `github`. It has a 150-minute limit and VERSION 0.0.0-dev and BUILD 9999999, so build.sh passes the version settings a tag passes. +- While the Mac is offline a run waits for it, and GitHub cancels a job still queued after 24 hours. `gh variable set RELEASE_BUILD_RUNNER --body github` sends nightlies to `xcode-27` until `gh variable delete RELEASE_BUILD_RUNNER`. - Setup as macOS Tests up to the SPM restore, which it never saves; then Go, `build-dameng.sh both` and `build-hana.sh both`. - `apple/scripts/release/build.sh app --unsigned` archives the `TablePro` scheme; `build.sh plugins --unsigned` builds `RegistryPlugins` and copies the publish set, the targets `apple/scripts/release/registry.py targets` reads from `.github/plugin-registry.json`, to `apple/build/release/plugins`. Both build universal. They run `lipo -verify_arch` once per arch (arm64, x86_64), since Xcode 27's `lipo` takes one arch per call, then `verify-plugin-loads.sh`. - `apple/scripts/release/package.sh app --unsigned` writes the release assets: the DMG, the `.tar.xz` (extracted and compared with the app) and the dSYMs zip; `package.sh plugins` writes one zip per plugin and the plugin dSYMs zip. @@ -117,13 +117,15 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ ## Change detector (`scripts/ci/detect-changed-paths.sh`) -- `detect-changed-paths.sh [--base ] macos|plugins|ios|oracle|hana|release` prints `true` or `false`. An unknown suite or flag exits 2. -- A PR passes its base SHA, a push `github.event.before`. No base prints true. The all-zero SHA of a new ref diffs against `git merge-base HEAD origin/main`, so a branch cut from main skips what main already ran; with no merge base it prints true. +- `detect-changed-paths.sh [--base ] [--head ] [--ref ] ` prints `true` or `false` for `macos`, `plugins`, `ios`, `oracle`, `hana`, `release`, `shipped` or `linux`. An unknown suite or flag exits 2. +- A PR passes its base SHA, a push `github.event.before`; `--head` defaults to `HEAD`. No base prints true. The all-zero SHA of a new ref diffs against `git merge-base origin/main`, so a branch cut from main skips what main already ran; with no merge base it prints true. +- macOS Tests, iOS and Package Tests and Repo Hygiene pass `--ref` with `github.ref`, and `refs/heads/release/*` prints true for every suite: a tag's `verify` needs the line head's own runs, and a release PR changes only the changelog. - `macos` watches the app, test, plugin, package, native, config and libs folders, `apple/TestPlans/`, `apple/project.yml`, the app's `Package.resolved`, the build scripts and CI scripts it runs, the setup actions and its workflow. `apple/Packages/` covers TableProMacCore. - `plugins` watches `apple/Plugins/`, `apple/Packages/TableProCore/`, `apple/Packages/TableProOracle/`, `apple/Native/`, `apple/Libs/`, `apple/Configs/`, `apple/project.yml`, the app's `Package.resolved`, the project, libs and bridge scripts, `verify-plugin-loads.sh`, `apple/scripts/check-pluginkit-abi.sh` and the setup actions. - `ios` watches `apple/TableProMobile/`, `apple/Libs/`, `apple/Plugins/TableProPluginKit/` (the package target is a symlink to it), and every `../` path it reads out of `apple/TableProMobile/project.yml`. - `oracle` watches `apple/Packages/TableProOracle/`; `hana` watches `apple/Native/HanaBridge/`. Each suite also watches its own workflow. - `release` watches what the Release build reads: the app, plugin, package, native, config and libs folders, `apple/project.yml`, the app's `Package.resolved`, `.github/plugin-registry.json`, the release scripts it runs (`build.sh`, `package.sh`, `registry.py`, `version.py`, `notes.py`), the build and CI scripts it runs and the setup actions. It skips `apple/Plugins/*Tests/` and `apple/Packages/*/Tests/`. +- `shipped` is `release` without its `.github/workflows/` and `.github/actions/` entries. `version.py check` reads it for a stable's diff from its newest beta (`apple/RELEASING.md`, What a tag runs). - `linux` watches `linux/`, the detector itself and the three Linux workflows. - A path a suite newly reads joins that suite's pattern in the same change, or the suite skips on it. - `apple/CHANGELOG.md` and `apple/changelog.d/` are on no suite's pattern: the What's New phase writes nothing at `0.0.0-dev`, and Repo Hygiene renders every fragment and section instead. @@ -131,7 +133,7 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ ## What runs where - A pull request: both changes jobs; macOS Tests and iOS and Package Tests unless their detector says false (Oracle only when its detector is not false); SwiftLint; Repo Hygiene; PR title. `AllPlugins` only when the `plugins` detector is not false. -- A push to main or `release/**`: the same minus PR title; never cancelled, the SPM cache saved on a miss. Every code push runs both plans in full; a docs-only or changelog-only push skips the suites through the detector. A `release/**` push also runs Release Build; main gets it nightly. +- A push to main or `release/**`: the same minus PR title; never cancelled, the SPM cache saved on a miss. Every code push runs both plans in full. On main a docs-only or changelog-only push skips the suites through the detector; a `release/**` push runs every suite (Change detector). Release Build runs nightly on main and by dispatch only. - A `v*` tag runs only `release.yml`, and a `linux-v*` tag only `release-linux.yml`, which calls `linux-packages.yml`. The test workflows never run on a tag. - `workflow_dispatch` exists on every workflow except `pr-title.yml` and `release.yml`. - `build-linux.yml` builds and tests the Linux app, and `linux-packages.yml` builds its packages (`linux/docs/testing.md`, CI). What starts each is in its `on:` and job `if:` lines. Flatpak Devel and Minimum Rust version never run on a PR or push, and `linux-packages.yml` never on a push. diff --git a/.claude/skills/release/SKILL.md b/.claude/skills/release/SKILL.md index 39d2e34926..3901094aec 100644 --- a/.claude/skills/release/SKILL.md +++ b/.claude/skills/release/SKILL.md @@ -36,7 +36,7 @@ Before any decision, gather: The days, the gate and the patch bar live in `apple/RELEASING.md`, Schedule. -- **Promote to X.Y.0**: Monday, when the gate passes: the line's age, the newest beta's age and nothing but changelog changes after it (section 2), no open `regression` issue, and the storage check (section 4). +- **Promote to X.Y.0**: Monday, when the gate passes: the line's age, the newest beta's age and no shipped source after it (section 2), no open `regression` issue, and the storage check (section 4). - **beta.1 of X.(Y+1)**: the same Monday, after X.Y.0's forward-port merges (section 9). The tag refuses to build until `main` has `## [X.Y.0]`. - **beta.N**: Tuesday to Thursday, when the line has picks since its last beta. - **Patch X.Y.Z**: Tuesday to Thursday, when the stable line has a fix past the patch bar. diff --git a/.github/workflows/ios-tests.yml b/.github/workflows/ios-tests.yml index cb9e9eb8af..1ac0dc5bbd 100644 --- a/.github/workflows/ios-tests.yml +++ b/.github/workflows/ios-tests.yml @@ -30,10 +30,11 @@ jobs: id: decide env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + REF: ${{ github.ref }} run: | set -euo pipefail - run=$(scripts/ci/detect-changed-paths.sh --base "$BASE" ios) - oracle=$(scripts/ci/detect-changed-paths.sh --base "$BASE" oracle) + run=$(scripts/ci/detect-changed-paths.sh --base "$BASE" --ref "$REF" ios) + oracle=$(scripts/ci/detect-changed-paths.sh --base "$BASE" --ref "$REF" oracle) { echo "run=$run"; echo "oracle=$oracle"; } >> "$GITHUB_OUTPUT" # The required check, with no gate job: see macos-tests.yml. TableProCore and TableProOracle run diff --git a/.github/workflows/macos-tests.yml b/.github/workflows/macos-tests.yml index cf49bbfee6..28ceccb420 100644 --- a/.github/workflows/macos-tests.yml +++ b/.github/workflows/macos-tests.yml @@ -32,10 +32,11 @@ jobs: id: decide env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + REF: ${{ github.ref }} run: | set -euo pipefail - run=$(scripts/ci/detect-changed-paths.sh --base "$BASE" macos) - plugins=$(scripts/ci/detect-changed-paths.sh --base "$BASE" plugins) + run=$(scripts/ci/detect-changed-paths.sh --base "$BASE" --ref "$REF" macos) + plugins=$(scripts/ci/detect-changed-paths.sh --base "$BASE" --ref "$REF" plugins) { echo "run=$run"; echo "plugins=$plugins"; } >> "$GITHUB_OUTPUT" # The required check. A job skipped by its if: reports success, so no gate job is needed. The diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index bb40dc0fa4..7995797fa6 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -1,12 +1,9 @@ name: Release Build # The Release configuration of the app and every registry plugin, universal and unsigned, packaged -# the way a tag packages it, so a packaging failure shows before the tag: nightly on main, on every -# release branch push, and on demand. +# the way a tag packages it, so a packaging failure shows before the tag: nightly on main and on demand. on: - push: - branches: ['release/**'] schedule: - cron: '0 19 * * *' workflow_dispatch: @@ -36,34 +33,21 @@ jobs: fetch-depth: 0 # A nightly diffs against the last nightly that ended green, built or skipped: main up to - # that commit is covered. A push that creates a release branch builds the cut commit, which - # the merge-base rule would skip. + # that commit is covered. - name: Decide whether the Release build runs id: decide env: EVENT: ${{ github.event_name }} - CREATED: ${{ github.event.created }} - BEFORE: ${{ github.event.before }} GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - case "$EVENT" in - workflow_dispatch) - run=true - ;; - schedule) - base=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/release-build.yml/runs?event=schedule&status=success&per_page=1" \ - --jq '.workflow_runs[0].head_sha // ""') - run=$(scripts/ci/detect-changed-paths.sh --base "$base" release) - ;; - *) - if [ "$CREATED" = true ]; then - run=true - else - run=$(scripts/ci/detect-changed-paths.sh --base "$BEFORE" release) - fi - ;; - esac + if [ "$EVENT" = workflow_dispatch ]; then + run=true + else + base=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/release-build.yml/runs?event=schedule&status=success&per_page=1" \ + --jq '.workflow_runs[0].head_sha // ""') + run=$(scripts/ci/detect-changed-paths.sh --base "$base" release) + fi echo "run=$run" >> "$GITHUB_OUTPUT" # Reads "not false" so a failed changes job builds instead of passing. @@ -71,7 +55,7 @@ jobs: name: Release Build needs: changes if: ${{ !cancelled() && needs.changes.outputs.run != 'false' }} - # A push or nightly has no input, so it reads the RELEASE_BUILD_RUNNER repository variable; unset is the Mac. + # A nightly has no input, so it reads the RELEASE_BUILD_RUNNER repository variable; unset is the Mac. runs-on: ${{ (inputs.runner || vars.RELEASE_BUILD_RUNNER) == 'github' && 'xcode-27' || fromJSON('["self-hosted", "macOS", "ARM64", "tablepro-release"]') }} timeout-minutes: 150 # The tag's version settings run here first; 9999999 is the largest build at MAJOR 0 (apple/RELEASING.md, Versions). diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70b7f82f67..6c2024c48f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: verify: name: Verify runs-on: ubuntu-latest - timeout-minutes: 240 + timeout-minutes: 10 permissions: contents: read actions: read @@ -28,6 +28,7 @@ jobs: env: TAG: ${{ github.ref_name }} GH_TOKEN: ${{ github.token }} + PYTHONUNBUFFERED: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 1faf0984f2..a17c71e436 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -124,9 +124,10 @@ jobs: id: native env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + REF: ${{ github.ref }} run: | set -euo pipefail - hana=$(scripts/ci/detect-changed-paths.sh --base "$BASE" hana) + hana=$(scripts/ci/detect-changed-paths.sh --base "$BASE" --ref "$REF" hana) echo "hana=$hana" >> "$GITHUB_OUTPUT" - name: Set up Go diff --git a/apple/RELEASING.md b/apple/RELEASING.md index 2f2fffa437..be48db3bfe 100644 --- a/apple/RELEASING.md +++ b/apple/RELEASING.md @@ -26,14 +26,14 @@ This runbook covers the macOS app, its registry plugins and the static libraries - Every fix lands on `main` first, then reaches the line through a cherry-pick PR from `backport/-`: one fix per PR, titled with the original subject and its ` (#N)`, which is the provenance. A release PR comes from `prep/v`. No PR head branch lives under `release/*`. - A line closes when `X.(Y+1).0` ships. A later security or data-loss fix for its users ships as a critical patch on the current stable line. At most two lines are live. - After a stable or patch publishes, a PR with the same title cherry-picks its release commit to `main`, and to a newer open line when one exists. The next tag refuses to build until it has. -- The 0.82 line was cut before the Apple apps moved into `apple/`: on it every path here sits at the repository root, and a pick across the move relies on git's rename detection. +- The 0.82 line was cut before the Apple apps moved into `apple/`: on it every path here sits at the repository root, and a pick across the move relies on git's rename detection. Its tags run its own `release.yml` and `version.py`, which walk its CI and wait for Release Build. ## Schedule Tags go out between 13:00 and 15:00 ICT (UTC+7). - Monday, the train: promote the open line's newest beta to `vX.Y.0` when the gate passes, forward-port it, then cut `release/X.(Y+1)` from the newest green `main` commit and tag its beta.1. The forward-port merges first: `version.py` refuses beta.1 of X.(Y+1) until `main` has `## [X.Y.0]`. -- The gate: the line is 7 days old; its newest beta is at least 72 h old and nothing outside `CHANGELOG.md` and `changelog.d/` landed on the line after it; no open issue is labeled `regression`; the storage check passes (`.claude/skills/release/SKILL.md`, section 4). +- The gate: the line is 7 days old; its newest beta is at least 72 h old and no shipped source landed on the line after it (What a tag runs, `verify`); no open issue is labeled `regression`; the storage check passes (`.claude/skills/release/SKILL.md`, section 4). - When the gate fails, nothing is promoted or cut. Fix on `main`, pick it, and ship beta.N+1. Promote it on the first Tuesday to Thursday when it is 48 h old with no open `regression` issue, and cut the next beta.1 the same day. That line's train is the first Monday it is 7 days old. - Tuesday to Thursday: a beta.N when the line has picks, at most one a day. A patch `X.Y.Z` goes straight to stable, only for a regression from the previous stable, a crash, a plugin that fails to load, a security fix or data loss. - Friday to Sunday: no tags, except a critical release or the fix for a build that fails to launch, update or connect. @@ -49,12 +49,13 @@ Tags go out between 13:00 and 15:00 ICT (UTC+7). ## Cutting a beta -beta.1 cuts the line at the newest `main` commit a green nightly or dispatched Release Build ran on, once its push runs are green too. The push that creates the branch builds it again, and the tag's `verify` waits for that build. beta.N tags the line's head once the picks are on it. +beta.1 cuts the line at the newest `main` commit a green nightly or dispatched Release Build ran on, once its push runs are green too. beta.N tags the line's head once the picks are on it. Either tag goes on once the line head's own push runs are green (What a tag runs, `verify`). ```bash git fetch origin git push origin :refs/heads/release/X.Y # beta.1 only gh label create "backport release/X.Y" # beta.1 only +gh run list --branch release/X.Y --event push --commit "$(git rev-parse origin/release/X.Y)" git tag -s vX.Y.0-beta.N -m vX.Y.0-beta.N origin/release/X.Y git push origin vX.Y.0-beta.N ``` @@ -72,11 +73,13 @@ git push -u origin prep/vX.Y.Z gh pr create --base release/X.Y --title "chore(release): vX.Y.Z" --body "Release notes for vX.Y.Z." ``` -The PR changes only `CHANGELOG.md` and `changelog.d/`; PR title refuses anything else. Squash-merge it, then tag its merge commit: +The PR changes only `CHANGELOG.md` and `changelog.d/`; PR title refuses anything else. Squash-merge it, then tag its merge commit once that commit's push runs are green (What a tag runs, `verify`): ```bash git fetch origin -git tag -s vX.Y.Z -m vX.Y.Z "$(gh pr view --json mergeCommit -q .mergeCommit.oid)" +release_commit="$(gh pr view --json mergeCommit -q .mergeCommit.oid)" +gh run list --branch release/X.Y --event push --commit "$release_commit" +git tag -s vX.Y.Z -m vX.Y.Z "$release_commit" git push origin vX.Y.Z ``` @@ -97,7 +100,7 @@ python3 apple/scripts/release/notes.py docs X.Y.Z --ref vX.Y.Z --mdx ../tablepro A `v*` tag runs `release.yml` and nothing else. A run is never cancelled, and every job reads the version, build and channel from `verify`. -- `verify`: `version.py check` refuses a tag that breaks the grammar or the bounds; that `origin/release/` does not contain; whose previous tag on the line is not its ancestor; a stable X.Y.0 with no published beta, or a beta after its X.Y.0; a stable whose tree differs from its line's newest published beta outside `CHANGELOG.md` and `changelog.d/`, until a stable tag on the line has matched that beta; a `CHANGELOG.md` missing the stable's heading, or a published stable of this or the previous line below it; a stable whose tag still holds fragments in `changelog.d/`. Then it needs macOS Tests, iOS and Package Tests and Release Build green on the tag commit or the nearest first-parent commit that ran them, with no change to the files they read after it, waiting on a pending run. A push, nightly or dispatched run counts; a PR run builds a merge commit, so it does not. Last, it writes the notes. +- `verify`: `version.py check` refuses a tag that breaks the grammar or the bounds; that `origin/release/` does not contain; whose previous tag on the line is not its ancestor; on a commit that already carries another `v*` tag; a stable X.Y.0 with no published beta, or a beta after its X.Y.0; a stable that changes shipped source (the `shipped` suite of `scripts/ci/detect-changed-paths.sh`) since its line's newest published beta, until a stable tag on the line has matched that beta, so a patch after a matching X.Y.0 passes; a `CHANGELOG.md` missing the stable's heading, or a published stable of this or the previous line below it; a stable whose tag still holds fragments in `changelog.d/`. Then it reads the tag commit's push runs on `release/` once: macOS Tests and iOS and Package Tests must each have completed with success, their test job run and not skipped. A `release/**` push runs every suite, so the line head always gets both (`.claude/rules/ci.md`, Change detector). It never waits: a run still queued or in progress fails it. Release Build is no gate. Last, it writes the notes. - `app` (`signing`): a GA Xcode for a stable (`setup-xcode` `require-release`), the CloudKit Production schema check, `build.sh app` (archive, export with `scripts/release/ExportOptions.plist`, universal and plugin load checks), `notarize.sh` on the app, `package.sh app`, `notarize.sh` on the DMG. - `plugins` (`signing`): the Dameng and HANA bridges, `build.sh plugins` (each target `registry.py targets` prints, universal, signed with Developer ID, the hardened runtime and a timestamp), one notary submission for all, `package.sh plugins` (the zips, described as the manifest will read them). - `github-release`: `actions/attest` attests `dist/*`; `gh release create` drafts, uploads and publishes; `gh release verify` and `verify-asset` check the published release. @@ -111,7 +114,7 @@ A `v*` tag runs `release.yml` and nothing else. A run is never cancelled, and ev - After it publishes: **Re-run failed jobs**. Every later step skips what is done: a published release, a feed item with the same version and build, or a manifest release with the same build and sha256. - A stable that never published: the next release PR runs `git revert ` before `notes.py assemble`, and the leftover draft goes (`gh release delete vX.Y.Z`). - No GA Xcode on the runner image fails a stable in `app`. The fix is a PR pointing `.github/actions/setup-xcode` at a GA Xcode on an image that has one. Betas still ship. -- Release Build runs on the maintainer's Mac, and a tag's `verify` waits for it. While the Mac is offline, `gh variable set RELEASE_BUILD_RUNNER --body github` sends pushes and nightlies to `xcode-27`; a run already waiting needs a cancel and a re-run. `gh variable delete RELEASE_BUILD_RUNNER` once the Mac is back. +- `verify` names the macOS Tests or iOS and Package Tests run that is still going, red, skipped its tests or missing. Re-run `verify` once that run is green (`gh run rerun --failed`), after re-running a run that failed on infrastructure. A code failure ships as the next number, as above. ## Withdrawing a release diff --git a/apple/scripts/release/test_version.py b/apple/scripts/release/test_version.py index dc1efc3a83..e50f05dbb8 100644 --- a/apple/scripts/release/test_version.py +++ b/apple/scripts/release/test_version.py @@ -2,10 +2,12 @@ import io import os +import shutil import subprocess import tempfile import unittest from contextlib import redirect_stderr, redirect_stdout +from dataclasses import replace from pathlib import Path import version @@ -20,6 +22,9 @@ "GIT_COMMITTER_EMAIL": "fixture@example.com", } MAIN_CHANGELOG = "# Changelog\n\n## [0.81.0] - 2026-10-16\n\n### Fixed\n\n- A bug. (#1)\n" +DETECTOR_SOURCE = Path(__file__).resolve().parents[3] / version.DETECTOR +# What the fake detector in CheckTests calls shipped; AppleTreeTests runs the real one. +SHIPPED_PREFIXES = ("TablePro/", "Plugins/") class GrammarTests(unittest.TestCase): @@ -81,22 +86,27 @@ def __init__(self, releases=(), runs=None, jobs=None): def releases(self): return self._releases - def commit_runs(self, workflow, sha): - self.run_queries.append((workflow, sha)) - snapshots = self._runs.get((workflow, sha), [[]]) - return snapshots.pop(0) if len(snapshots) > 1 else snapshots[0] + def push_runs(self, sha, branch): + self.run_queries.append((sha, branch)) + return self._runs.get((sha, branch), []) def job_conclusions(self, run_id, job): return self._jobs.get((run_id, job), []) -def green_on(sha, prefix="run"): - runs, jobs = {}, {} - for workflow, job, _ in version.REQUIRED_RUNS: - run_id = f"{prefix}-{workflow}" - runs[(workflow, sha)] = [[(run_id, "completed", "success")]] - jobs[(run_id, job)] = ["success"] - return runs, jobs +def green_on(sha, branch="release/0.82"): + runs, jobs = [], {} + for path, name in version.TEST_WORKFLOWS: + run_id = f"run-{name}" + runs.append(version.Run(run_id, path, "completed", "success", f"https://ci.example/{run_id}")) + jobs[(run_id, name)] = ["success"] + return {(sha, branch): runs}, jobs + + +def with_run(runs, name, **changes): + path = next(path for path, job in version.TEST_WORKFLOWS if job == name) + return {key: [replace(run, **changes) if run.path == path else run for run in found] + for key, found in runs.items()} class FixtureRepository: @@ -135,141 +145,105 @@ def setUp(self): self.addCleanup(directory.cleanup) self.repo = FixtureRepository(Path(directory.name)) self.git = version.Git(self.repo.path) - self.slept = [] - self.logged = [] self.diffs = [] - self.changed_suites = set() - self.changed_bases = set() self.base = self.repo.commit("0.81.0", MAIN_CHANGELOG) self.repo.git("tag", "v0.81.0") self.cut = self.repo.commit("feat: something") self.repo.line("0.82", self.cut) - def changed(self, base, suite): - self.diffs.append((base, suite)) - return suite in self.changed_suites or (base, suite) in self.changed_bases + def changed(self, base, head, suite): + self.diffs.append((base, head, suite)) + paths = self.repo.git("diff-tree", "-r", "--name-only", base, head).splitlines() + return any(path.startswith(SHIPPED_PREFIXES) for path in paths) def check(self, tag, github): - if self.git.exists(f"refs/tags/{tag}"): - self.repo.git("checkout", "--quiet", "--detach", tag) - return version.check(tag, self.git, github, changed=self.changed, sleep=self.slept.append, - log=self.logged.append) + return version.check(tag, self.git, github, changed=self.changed) + + def stable_on_line(self, name, heading, changelog=MAIN_CHANGELOG): + release_commit = self.repo.commit(f"chore(release): {name}", changelog.replace( + "## [0.81.0]", f"## [{heading}] - 2026-10-26\n\n## [0.81.0]")) + self.repo.line("0.82", release_commit) + self.repo.git("tag", name, release_commit) + return release_commit def test_a_first_beta_on_a_green_cut_commit_builds(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) self.assertEqual(self.check("v0.82.0-beta.1", github), []) + self.assertEqual(github.run_queries, [(self.cut, "release/0.82")]) - def test_a_skipped_job_borrows_the_nearest_first_parent_verdict(self): - release_commit = self.repo.commit("chore(release): notes only") - self.repo.line("0.82", release_commit) - self.repo.git("tag", "v0.82.0-beta.1", release_commit) - runs, jobs = green_on(self.cut) - for workflow, job, _ in version.REQUIRED_RUNS: - runs[(workflow, release_commit)] = [[("skip-" + workflow, "completed", "success")]] - jobs[("skip-" + workflow, job)] = ["skipped"] - github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) - self.assertEqual(self.check("v0.82.0-beta.1", github), []) - self.assertIn(f"macOS Tests: green on {self.cut}", self.logged) - self.assertEqual(self.diffs, [(self.cut, "macos"), (self.cut, "ios"), (self.cut, "release")]) - - def test_a_borrowed_verdict_must_still_cover_the_tag(self): - unrun = self.repo.commit("fix: pushed with no run") - self.repo.line("0.82", unrun) - self.repo.git("tag", "v0.82.0-beta.1", unrun) - self.changed_suites = {"macos"} + def test_a_skipped_test_job_refuses_the_tag(self): + self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) + jobs[("run-macOS Tests", "macOS Tests")] = ["skipped"] github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) self.assertEqual(self.check("v0.82.0-beta.1", github), - [f"macOS Tests passed on {self.cut}, but files it reads changed before v0.82.0-beta.1"]) + ["macOS Tests ran no tests at https://ci.example/run-macOS Tests (job: skipped)"]) - def test_a_green_tag_commit_needs_no_diff(self): + def test_a_run_still_going_refuses_at_once_and_names_its_url(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) - self.changed_suites = {"macos", "ios", "release"} - github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) - self.assertEqual(self.check("v0.82.0-beta.1", github), []) - self.assertEqual(self.diffs, []) - - def test_the_walk_needs_the_tag_checked_out(self): - self.repo.git("tag", "v0.82.0-beta.1", self.base) - self.repo.line("0.82", self.cut) - errors = version.check("v0.82.0-beta.1", self.git, FakeGitHub(releases=[("v0.81.0", False)]), - changed=self.changed, sleep=self.slept.append, log=self.logged.append) - self.assertEqual(errors, ["check out v0.82.0-beta.1 first: the coverage check diffs against HEAD"]) - - def test_a_pending_run_is_waited_for(self): + for status in ("queued", "in_progress"): + with self.subTest(status=status): + pending = with_run(runs, "iOS and Package Tests", status=status, conclusion="") + github = FakeGitHub(releases=[("v0.81.0", False)], runs=pending, jobs=jobs) + self.assertEqual(self.check("v0.82.0-beta.1", github), + [f"iOS and Package Tests is {status} at https://ci.example/run-iOS and Package " + "Tests: re-run Verify once it is green"]) + + def test_a_failed_run_refuses_the_tag(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) - workflow = version.REQUIRED_RUNS[0][0] - runs[(workflow, self.cut)] = [[("run-" + workflow, "in_progress", "")], - [("run-" + workflow, "queued", "")], - [("run-" + workflow, "completed", "success")]] - github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) - self.assertEqual(self.check("v0.82.0-beta.1", github), []) - self.assertEqual(self.slept, [version.POLL_SECONDS, version.POLL_SECONDS]) - - def test_a_failed_job_refuses_the_tag(self): + for conclusion in ("failure", "cancelled", "startup_failure"): + with self.subTest(conclusion=conclusion): + failed = with_run(runs, "iOS and Package Tests", conclusion=conclusion) + github = FakeGitHub(releases=[("v0.81.0", False)], runs=failed, jobs=jobs) + self.assertEqual(self.check("v0.82.0-beta.1", github), + [f"iOS and Package Tests ended '{conclusion}' at " + "https://ci.example/run-iOS and Package Tests"]) + + def test_a_missing_run_refuses_the_tag(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) - jobs[("run-ios-tests.yml", "iOS and Package Tests")] = ["failure"] - github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) + macos_only = {key: [run for run in found if run.path.endswith("macos-tests.yml")] + for key, found in runs.items()} + github = FakeGitHub(releases=[("v0.81.0", False)], runs=macos_only, jobs=jobs) self.assertEqual(self.check("v0.82.0-beta.1", github), - [f"iOS and Package Tests ended 'failure' on {self.cut} (run run-ios-tests.yml)"]) + [f"no iOS and Package Tests push run on release/0.82 at {self.cut}"]) - def test_a_run_that_failed_before_its_jobs_refuses_the_tag(self): + def test_only_a_push_run_on_the_line_counts(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) - runs, jobs = green_on(self.cut) - runs[("release-build.yml", self.cut)] = [[("broken", "completed", "startup_failure")]] + runs, jobs = green_on(self.cut, branch="main") github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) self.assertEqual(self.check("v0.82.0-beta.1", github), - [f"Release Build ended 'startup_failure' on {self.cut} (run broken)"]) + [f"no macOS Tests push run on release/0.82 at {self.cut}", + f"no iOS and Package Tests push run on release/0.82 at {self.cut}"]) - def test_a_success_on_another_branch_run_of_the_same_commit_counts(self): + def test_release_build_is_not_required(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) - for workflow, job, _ in version.REQUIRED_RUNS: - runs[(workflow, self.cut)][0].append(("branch-" + workflow, "completed", "success")) - jobs[("branch-" + workflow, job)] = ["skipped"] + runs[(self.cut, "release/0.82")].append(version.Run( + "build", ".github/workflows/release-build.yml", "completed", "failure", "https://ci.example/build")) github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) self.assertEqual(self.check("v0.82.0-beta.1", github), []) - def test_skipped_commits_past_fifty_still_borrow_the_last_green_run(self): - tail = self.cut - for index in range(60): - tail = self.repo.commit(f"feat(linux): change {index}") - self.repo.line("0.82", tail) - self.repo.git("tag", "v0.82.0-beta.1", tail) - runs, jobs = green_on(self.cut) - github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) - self.assertEqual(self.check("v0.82.0-beta.1", github), []) - self.assertIn(f"macOS Tests: green on {self.cut}", self.logged) - - def test_the_walk_stops_at_the_newest_change_to_the_files_a_job_reads(self): - between = self.repo.commit("feat: before the change") - touch = self.repo.commit("feat: changes the app") - tail = self.repo.commit("docs: after the change") - self.repo.line("0.82", tail) - self.repo.git("tag", "v0.82.0-beta.1", tail) - self.changed_bases = {(between, "macos")} - runs, jobs = green_on(self.cut) - github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) - self.assertEqual(self.check("v0.82.0-beta.1", github), - [f"no macOS Tests run that was not skipped on {touch}, which changed files it reads, " - f"or after it"]) - self.assertNotIn(("macos-tests.yml", self.cut), github.run_queries) + def test_a_second_tag_on_a_tagged_commit_is_refused(self): + self.repo.git("tag", "--annotate", "--message", "v0.82.0-beta.1", "v0.82.0-beta.1", self.cut) + self.repo.git("tag", "--annotate", "--message", "v0.82.0-beta.2", "v0.82.0-beta.2", self.cut) + github = FakeGitHub(releases=[("v0.81.0", False)]) + self.assertEqual(self.check("v0.82.0-beta.2", github), + ["v0.82.0-beta.2 is on the same commit as v0.82.0-beta.1: one v* tag per commit"]) + self.assertEqual(github.run_queries, []) - def test_no_run_in_the_walk_refuses_the_tag(self): + def test_a_tag_of_another_kind_on_the_commit_is_allowed(self): + self.repo.git("tag", "linux-v0.1.0", self.cut) self.repo.git("tag", "v0.82.0-beta.1", self.cut) runs, jobs = green_on(self.cut) - del runs[("release-build.yml", self.cut)] github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) - errors = self.check("v0.82.0-beta.1", github) - self.assertEqual(len(errors), 1) - self.assertIn("no Release Build run that was not skipped", errors[0]) + self.assertEqual(self.check("v0.82.0-beta.1", github), []) - def test_a_tag_off_the_release_line_is_refused_before_the_walk(self): + def test_a_tag_off_the_release_line_is_refused_before_the_query(self): side = self.repo.commit("off the line") self.repo.git("tag", "v0.82.0-beta.1", side) self.repo.line("0.82", self.cut) @@ -306,26 +280,34 @@ def test_a_stable_needs_a_published_beta(self): published = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0-beta.1", False)], runs=runs, jobs=jobs) self.assertEqual(self.check("v0.82.0", published), []) - def test_a_stable_refuses_a_change_after_its_newest_published_beta(self): + def test_a_stable_refuses_shipped_source_after_its_newest_published_beta(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) beta = self.repo.commit("fix: first pick", source="TablePro/First.swift") self.repo.git("tag", "v0.82.0-beta.2", beta) unpublished = self.repo.commit("fix: second pick", source="TablePro/Second.swift") self.repo.git("tag", "v0.82.0-beta.3", unpublished) self.repo.commit("fix: third pick", source="Plugins/Third.swift") - release_commit = self.repo.commit("chore(release): v0.82.0", MAIN_CHANGELOG.replace( - "## [0.81.0]", "## [0.82.0] - 2026-10-26\n\n## [0.81.0]")) - self.repo.line("0.82", release_commit) - self.repo.git("tag", "v0.82.0", release_commit) + self.stable_on_line("v0.82.0", "0.82.0") releases = [("v0.81.0", False), ("v0.82.0-beta.1", False), ("v0.82.0-beta.2", False), ("v0.82.0-beta.3", True)] github = FakeGitHub(releases=releases) self.assertEqual(self.check("v0.82.0", github), - ["v0.82.0 differs from v0.82.0-beta.2, the newest published beta, in " - "Plugins/Third.swift, TablePro/Second.swift: revert them on release/0.82, " - "then tag the next number"]) + ["v0.82.0 changes shipped source since v0.82.0-beta.2, the newest published beta: " + "revert that on release/0.82, then tag the next number"]) + self.assertEqual(self.diffs, [("v0.82.0-beta.2", "v0.82.0", "shipped")]) self.assertEqual(github.run_queries, []) + def test_a_stable_may_differ_from_its_newest_beta_outside_shipped_source(self): + beta = self.repo.commit("fix: a fix", fragment="3610.fixed.md") + self.repo.git("tag", "v0.82.0-beta.1", beta) + self.repo.commit("ci: pin a runner", source=".github/workflows/release.yml") + self.repo.commit("docs(changelog): reword a fragment", fragment="3611.fixed.md") + self.repo.git("rm", "--quiet", "-r", "changelog.d") + release_commit = self.stable_on_line("v0.82.0", "0.82.0") + runs, jobs = green_on(release_commit) + github = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0-beta.1", False)], runs=runs, jobs=jobs) + self.assertEqual(self.check("v0.82.0", github), []) + def test_the_next_number_after_a_refused_stable_is_held_to_the_beta(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) pick = self.repo.commit("fix: late pick", source="TablePro/Late.swift") @@ -343,38 +325,11 @@ def test_the_next_number_after_a_refused_stable_is_held_to_the_beta(self): releases = [("v0.81.0", False), ("v0.82.0-beta.1", False)] runs, jobs = green_on(reverted) self.assertEqual(self.check("v0.82.1", FakeGitHub(releases=releases)), - ["v0.82.1 differs from v0.82.0-beta.1, the newest published beta, in " - "TablePro/Late.swift: revert them on release/0.82, then tag the next number"]) + ["v0.82.1 changes shipped source since v0.82.0-beta.1, the newest published beta: " + "revert that on release/0.82, then tag the next number"]) self.assertEqual(self.check("v0.82.2", FakeGitHub(releases=releases, runs=runs, jobs=jobs)), []) - def test_a_fix_after_a_stable_tag_that_matched_the_beta_is_a_patch(self): - self.repo.git("tag", "v0.82.0-beta.1", self.cut) - stable = self.repo.commit("chore(release): v0.82.0", MAIN_CHANGELOG.replace( - "## [0.81.0]", "## [0.82.0] - 2026-10-26\n\n## [0.81.0]")) - self.repo.git("tag", "v0.82.0", stable) - self.repo.commit("ci: select a GA Xcode", source=".github/actions/setup-xcode/action.yml") - release_commit = self.repo.commit("chore(release): v0.82.1", MAIN_CHANGELOG.replace( - "## [0.81.0]", "## [0.82.1] - 2026-10-27\n\n## [0.81.0]")) - self.repo.line("0.82", release_commit) - self.repo.git("tag", "v0.82.1", release_commit) - runs, jobs = green_on(release_commit) - releases = [("v0.81.0", False), ("v0.82.0-beta.1", False), ("v0.82.0", True)] - self.assertEqual(self.check("v0.82.1", FakeGitHub(releases=releases, runs=runs, jobs=jobs)), []) - - def test_a_stable_may_differ_from_its_newest_beta_in_the_changelog_only(self): - beta = self.repo.commit("fix: a fix", fragment="3610.fixed.md") - self.repo.git("tag", "v0.82.0-beta.1", beta) - self.repo.commit("docs(changelog): reword a fragment", fragment="3611.fixed.md") - self.repo.git("rm", "--quiet", "-r", "changelog.d") - release_commit = self.repo.commit("chore(release): v0.82.0", MAIN_CHANGELOG.replace( - "## [0.81.0]", "## [0.82.0] - 2026-10-26\n\n## [0.81.0]")) - self.repo.line("0.82", release_commit) - self.repo.git("tag", "v0.82.0", release_commit) - runs, jobs = green_on(release_commit) - github = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0-beta.1", False)], runs=runs, jobs=jobs) - self.assertEqual(self.check("v0.82.0", github), []) - - def test_a_patch_is_not_held_to_the_beta_tree(self): + def test_a_patch_after_a_stable_that_matched_the_beta_is_exempt(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) stable = self.repo.commit("chore(release): v0.82.0", MAIN_CHANGELOG.replace( "## [0.81.0]", "## [0.82.0] - 2026-10-26\n\n## [0.81.0]")) @@ -387,6 +342,7 @@ def test_a_patch_is_not_held_to_the_beta_tree(self): runs, jobs = green_on(release_commit) releases = [("v0.81.0", False), ("v0.82.0-beta.1", False), ("v0.82.0", False)] self.assertEqual(self.check("v0.82.1", FakeGitHub(releases=releases, runs=runs, jobs=jobs)), []) + self.assertEqual(self.diffs, [("v0.82.0-beta.1", "v0.82.0", "shipped")]) def test_a_patch_needs_no_beta(self): changelog = MAIN_CHANGELOG.replace( @@ -418,7 +374,7 @@ def test_a_stable_refuses_fragments_left_at_its_tag(self): pick = self.repo.commit("fix: a pick after the release PR was cut", fragment="3610.fixed.md") self.repo.line("0.81", pick) self.repo.git("tag", "v0.81.1", pick) - runs, jobs = green_on(pick) + runs, jobs = green_on(pick, branch="release/0.81") github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) self.assertEqual(self.check("v0.81.1", github), ["v0.81.1 still has 3610.fixed.md in changelog.d/; assemble again on the line head"]) @@ -468,28 +424,39 @@ def test_the_changelog_and_fragments_come_from_the_tree_git_runs_in(self): repo.line("0.81", sha) repo.git("tag", "v0.81.1", sha) errors = version.check("v0.81.1", version.Git(apple), FakeGitHub(releases=[("v0.81.0", False)]), - changed=lambda base, suite: True) + changed=lambda base, head, suite: True) self.assertEqual(errors, ["v0.81.1 still has 3610.fixed.md in changelog.d/; assemble again on the line head"]) - def test_a_stable_may_differ_from_its_beta_in_the_apple_changelog_only(self): - with tempfile.TemporaryDirectory() as directory: - repo = FixtureRepository(Path(directory)) - apple = repo.path / "apple" - (apple / "changelog.d").mkdir(parents=True) - (apple / "changelog.d" / "3610.fixed.md").write_text("A fix.\n", encoding="utf-8") - (apple / "CHANGELOG.md").write_text(MAIN_CHANGELOG, encoding="utf-8") - repo.git("add", "--all") - repo.git("tag", "v0.82.0-beta.1", repo.commit("fix: a fix")) - repo.git("rm", "--quiet", "-r", "apple/changelog.d") - (apple / "CHANGELOG.md").write_text( - MAIN_CHANGELOG.replace("## [0.81.0]", "## [0.82.0] - 2026-10-26\n\n## [0.81.0]"), encoding="utf-8") - (repo.path / "ci.yml").write_text("on: push\n", encoding="utf-8") - repo.git("add", "--all") - repo.git("tag", "v0.82.0", repo.commit("chore(release): v0.82.0")) - errors = version.stable_tree_errors(version.Git(apple), version.parse_tag("v0.82.0"), - [("v0.82.0-beta.1", False)]) - self.assertEqual(errors, ["v0.82.0 differs from v0.82.0-beta.1, the newest published beta, in ci.yml: " - "revert them on release/0.82, then tag the next number"]) + def test_the_shipped_source_rule_reads_the_detector(self): + cases = { + ".github/workflows/release.yml": [], + ".github/actions/setup-xcode/action.yml": [], + "apple/Plugins/OracleDriverPluginTests/OracleTests.swift": [], + "apple/TablePro/App.swift": ["v0.82.0 changes shipped source since v0.82.0-beta.1, the newest published " + "beta: revert that on release/0.82, then tag the next number"], + ".github/plugin-registry.json": ["v0.82.0 changes shipped source since v0.82.0-beta.1, the newest " + "published beta: revert that on release/0.82, then tag the next number"], + } + for path, expected in cases.items(): + with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: + repo = FixtureRepository(Path(directory)) + detector = repo.path / version.DETECTOR + detector.parent.mkdir(parents=True) + shutil.copy(DETECTOR_SOURCE, detector) + apple = repo.path / "apple" + (apple / "changelog.d").mkdir(parents=True) + (apple / "changelog.d" / "3610.fixed.md").write_text("A fix.\n", encoding="utf-8") + (apple / "CHANGELOG.md").write_text(MAIN_CHANGELOG, encoding="utf-8") + repo.git("add", "--all") + repo.git("tag", "v0.82.0-beta.1", repo.commit("fix: a fix")) + repo.git("rm", "--quiet", "-r", "apple/changelog.d") + (apple / "CHANGELOG.md").write_text( + MAIN_CHANGELOG.replace("## [0.81.0]", "## [0.82.0] - 2026-10-26\n\n## [0.81.0]"), encoding="utf-8") + repo.git("add", "--all") + repo.git("tag", "v0.82.0", repo.commit("chore(release): v0.82.0", source=path)) + errors = version.stable_tree_errors(version.Git(apple), version.parse_tag("v0.82.0"), + [("v0.82.0-beta.1", False)], version.detector(repo.path)) + self.assertEqual(errors, expected) class DetectorTests(unittest.TestCase): @@ -498,14 +465,15 @@ def test_the_detector_answer_is_read(self): root = Path(directory) script = root / version.DETECTOR script.parent.mkdir(parents=True) - script.write_text('#!/bin/sh\n[ "$3" = macos ] && echo true || echo false\n', encoding="utf-8") + script.write_text('#!/bin/sh\n[ "$1 $2 $3 $4 $5" = "--base abc --head v1 shipped" ] && echo true ' + '|| echo false\n', encoding="utf-8") script.chmod(0o755) changed = version.detector(root) - self.assertTrue(changed("abc", "macos")) - self.assertFalse(changed("abc", "ios")) + self.assertTrue(changed("abc", "v1", "shipped")) + self.assertFalse(changed("abc", "v1", "release")) script.write_text("#!/bin/sh\necho broken >&2\nexit 2\n", encoding="utf-8") with self.assertRaisesRegex(RuntimeError, "broken"): - changed("abc", "ios") + changed("abc", "v1", "shipped") class GitHubTests(unittest.TestCase): @@ -517,8 +485,8 @@ def run(command, **_): path = command[command.index("--jq") - 1] outputs = { "/releases": "v0.82.0\tfalse\nplugin-oracle-v1.0.0\ttrue\n", - "/runs?": ("11\tcompleted\tsuccess\tpush\n12\tin_progress\t\tschedule\n" - "13\tcompleted\tsuccess\tworkflow_dispatch\n14\tcompleted\tfailure\tpull_request\n"), + "/runs?": ("11\t.github/workflows/macos-tests.yml\tcompleted\tsuccess\thttps://ci.example/11\n" + "12\t.github/workflows/ios-tests.yml\tin_progress\t\thttps://ci.example/12\n"), "/jobs?": "success\n\n", } output = next(text for key, text in outputs.items() if key in path) @@ -526,11 +494,13 @@ def run(command, **_): github = version.GitHub("Owner/Repo", run=run) self.assertEqual(github.releases(), [("v0.82.0", False), ("plugin-oracle-v1.0.0", True)]) - self.assertEqual(github.commit_runs("macos-tests.yml", "abc"), - [("11", "completed", "success"), ("12", "in_progress", ""), ("13", "completed", "success")]) + self.assertEqual(github.push_runs("abc", "release/0.83"), [ + version.Run("11", ".github/workflows/macos-tests.yml", "completed", "success", "https://ci.example/11"), + version.Run("12", ".github/workflows/ios-tests.yml", "in_progress", "", "https://ci.example/12"), + ]) self.assertEqual(github.job_conclusions("11", "macOS Tests"), ["success", ""]) self.assertIn("--paginate", calls[0]) - self.assertIn("repos/Owner/Repo/actions/workflows/macos-tests.yml/runs?head_sha=abc&per_page=100", + self.assertIn("repos/Owner/Repo/actions/runs?head_sha=abc&event=push&branch=release/0.83&per_page=100", calls[1]) def test_a_gh_failure_is_raised(self): diff --git a/apple/scripts/release/version.py b/apple/scripts/release/version.py index 660af9ea23..df7c835cd1 100755 --- a/apple/scripts/release/version.py +++ b/apple/scripts/release/version.py @@ -5,7 +5,6 @@ import re import subprocess import sys -import time from dataclasses import dataclass from pathlib import Path from typing import Callable, List, Optional, Tuple @@ -19,17 +18,14 @@ # [0-9], not \d: \d also matches non-ASCII digits. GRAMMAR = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-beta\.(0|[1-9][0-9]*))?") -REQUIRED_RUNS = ( - ("macos-tests.yml", "macOS Tests", "macos"), - ("ios-tests.yml", "iOS and Package Tests", "ios"), - ("release-build.yml", "Release Build", "release"), +# Each workflow's test job has the workflow's name. +TEST_WORKFLOWS = ( + (".github/workflows/macos-tests.yml", "macOS Tests"), + (".github/workflows/ios-tests.yml", "iOS and Package Tests"), ) DETECTOR = "scripts/ci/detect-changed-paths.sh" +SHIPPED = "shipped" FRAGMENTS = "changelog.d/" -POLL_SECONDS = 60 -# A pull_request run builds a merge commit, not its head_sha; these events build the commit itself. -COMMIT_EVENTS = ("push", "schedule", "workflow_dispatch") -PENDING_STATUSES = {"queued", "in_progress", "waiting", "requested", "pending"} @dataclass(frozen=True) @@ -125,6 +121,9 @@ def is_ancestor(self, ancestor: str, descendant: str) -> bool: def tags(self, pattern: str) -> List[str]: return self._git("tag", "--list", pattern).stdout.split() + def tags_at(self, commit: str, pattern: str) -> List[str]: + return self._git("tag", "--points-at", commit, "--list", pattern).stdout.split() + def show(self, ref: str, path: str) -> Optional[str]: result = self._git("show", f"{ref}:{path}", check=False) return result.stdout if result.returncode == 0 else None @@ -132,15 +131,14 @@ def show(self, ref: str, path: str) -> Optional[str]: def list_tree(self, ref: str, path: str) -> List[str]: return self._git("ls-tree", "--name-only", ref, "--", path).stdout.splitlines() - def prefix(self) -> str: - return self._git("rev-parse", "--show-prefix").stdout.strip() - - def changed_paths(self, base: str, head: str) -> List[str]: - output = self._git("diff-tree", "-r", "--name-only", "-z", base, head).stdout - return [path for path in output.split("\0") if path] - def first_parents(self, ref: str) -> List[str]: - return self._git("rev-list", "--first-parent", f"{ref}^{{commit}}").stdout.split() +@dataclass(frozen=True) +class Run: + id: str + path: str + status: str + conclusion: str + url: str class GitHub: @@ -160,28 +158,23 @@ def releases(self) -> List[Tuple[str, bool]]: ".[] | [.tag_name, .draft] | @tsv", paginate=True) return [(tag, draft == "true") for tag, draft in (line.split("\t") for line in lines if line)] - def commit_runs(self, workflow: str, sha: str) -> List[Tuple[str, str, str]]: - lines = self._api(f"repos/{self.repository}/actions/workflows/{workflow}/runs" - f"?head_sha={sha}&per_page=100", - ".workflow_runs[] | [.id, .status, .conclusion // \"\", .event] | @tsv") - return [(run_id, status, conclusion) - for run_id, status, conclusion, event in (line.split("\t") for line in lines if line) - if event in COMMIT_EVENTS] + def push_runs(self, sha: str, branch: str) -> List[Run]: + lines = self._api(f"repos/{self.repository}/actions/runs?head_sha={sha}&event=push&branch={branch}" + "&per_page=100", + ".workflow_runs[] | [.id, .path, .status, .conclusion // \"\", .html_url] | @tsv") + return [Run(*line.split("\t")) for line in lines if line] def job_conclusions(self, run_id: str, job: str) -> List[str]: return self._api(f"repos/{self.repository}/actions/runs/{run_id}/jobs?filter=latest&per_page=100", f".jobs[] | select(.name == \"{job}\") | .conclusion // \"\"") -class CheckFailure(Exception): - pass - - -def detector(root: Path, run: Callable = subprocess.run) -> Callable[[str, str], bool]: - def changed(base: str, suite: str) -> bool: - result = run([str(root / DETECTOR), "--base", base, suite], cwd=root, capture_output=True, text=True) +def detector(root: Path, run: Callable = subprocess.run) -> Callable[[str, str, str], bool]: + def changed(base: str, head: str, suite: str) -> bool: + result = run([str(root / DETECTOR), "--base", base, "--head", head, suite], + cwd=root, capture_output=True, text=True) if result.returncode != 0: - raise RuntimeError(f"{DETECTOR} --base {base} {suite}: {result.stderr.strip()}") + raise RuntimeError(f"{DETECTOR} --base {base} --head {head} {suite}: {result.stderr.strip()}") return result.stdout.strip() != "false" return changed @@ -219,6 +212,13 @@ def branch_errors(git: Git, version: Version) -> List[str]: return errors +def shared_commit_errors(git: Git, version: Version) -> List[str]: + others = [tag for tag in git.tags_at(git.commit(version.tag), "v*") if tag != version.tag] + if not others: + return [] + return [f"{version.tag} is on the same commit as {', '.join(others)}: one v* tag per commit"] + + def beta_errors(git: Git, version: Version, releases: List[Tuple[str, bool]]) -> List[str]: stable_zero = Version(version.major, version.minor, 0) if version.is_beta: @@ -230,32 +230,25 @@ def beta_errors(git: Git, version: Version, releases: List[Tuple[str, bool]]) -> return [f"{version.tag} needs a published {version.tag}-beta.N release first"] -def untested_paths(git: Git, beta: Version, tag: str) -> List[str]: - # diff-tree names a path from the repository root; the changelog sits in the tree git runs in. - prefix = git.prefix() - return [path for path in git.changed_paths(beta.tag, tag) - if path != f"{prefix}CHANGELOG.md" and not path.startswith(f"{prefix}{FRAGMENTS}")] - - def earlier_stables(git: Git, version: Version) -> List[Version]: tags = (try_parse_tag(tag) for tag in git.tags(f"v{version.line}.*")) return [tag for tag in tags if tag is not None and not tag.is_beta and tag.line == version.line and tag.build < version.build] -def stable_tree_errors(git: Git, version: Version, releases: List[Tuple[str, bool]]) -> List[str]: +def stable_tree_errors(git: Git, version: Version, releases: List[Tuple[str, bool]], + changed: Callable[[str, str, str], bool]) -> List[str]: betas = published_betas(releases, version.line) if version.is_beta or not betas: return [] newest = max(betas, key=lambda beta: beta.build) # No beta can follow a stable tag, so a fix after one that matched the beta ships as a patch. - if any(not untested_paths(git, newest, stable.tag) for stable in earlier_stables(git, version)): + if any(not changed(newest.tag, stable.tag, SHIPPED) for stable in earlier_stables(git, version)): return [] - untested = untested_paths(git, newest, version.tag) - if not untested: + if not changed(newest.tag, version.tag, SHIPPED): return [] - return [f"{version.tag} differs from {newest.tag}, the newest published beta, in {', '.join(untested)}: " - f"revert them on release/{version.line}, then tag the next number"] + return [f"{version.tag} changes shipped source since {newest.tag}, the newest published beta: " + f"revert that on release/{version.line}, then tag the next number"] def changelog_errors(changelog: Optional[str], version: Version, stables: List[Version]) -> List[str]: @@ -289,66 +282,30 @@ def fragment_errors(paths: List[str], version: Version) -> List[str]: return [f"{version.tag} still has {', '.join(fragments)} in {FRAGMENTS}; assemble again on the line head"] -def is_green(github: GitHub, workflow: str, job: str, sha: str, - sleep: Callable[[float], None], log: Callable[[str], None]) -> bool: - while True: - runs = github.commit_runs(workflow, sha) - pending = [run_id for run_id, status, _ in runs if status in PENDING_STATUSES] - if not pending: - break - log(f"waiting for {workflow} run {pending[0]} on {sha[:9]}") - sleep(POLL_SECONDS) - conclusions = [] - for run_id, _, run_conclusion in runs: - found = github.job_conclusions(run_id, job) - # A run that failed before it created any job has only its own conclusion. - if not found and run_conclusion not in ("success", "skipped", ""): - found = [run_conclusion] - conclusions.extend((run_id, conclusion) for conclusion in found) - for run_id, conclusion in conclusions: - if conclusion not in ("success", "skipped"): - raise CheckFailure(f"{job} ended '{conclusion}' on {sha} (run {run_id})") - return any(conclusion == "success" for _, conclusion in conclusions) - - -def ci_errors(git: Git, github: GitHub, version: Version, changed: Callable[[str, str], bool], - sleep: Callable[[float], None] = time.sleep, log: Callable[[str], None] = print) -> List[str]: - tag_commit = git.commit(version.tag) - if git.commit("HEAD") != tag_commit: - return [f"check out {version.tag} first: the coverage check diffs against HEAD"] - shas = git.first_parents(version.tag) +def run_error(github: GitHub, run: Run, name: str) -> Optional[str]: + if run.status != "completed": + return f"{name} is {run.status} at {run.url}: re-run Verify once it is green" + if run.conclusion != "success": + return f"{name} ended '{run.conclusion}' at {run.url}" + jobs = github.job_conclusions(run.id, name) + if jobs != ["success"]: + return f"{name} ran no tests at {run.url} (job: {', '.join(jobs) or 'none'})" + return None + + +def push_run_errors(github: GitHub, sha: str, line: str) -> List[str]: + branch = f"release/{line}" + runs = github.push_runs(sha, branch) errors = [] - for workflow, job, suite in REQUIRED_RUNS: - try: - error = coverage_error(github, workflow, job, suite, shas, version.tag, changed, sleep, log) - except CheckFailure as failure: - error = str(failure) - if error is not None: - errors.append(error) + for path, name in TEST_WORKFLOWS: + found = [run for run in runs if run.path == path] + if not found: + errors.append(f"no {name} push run on {branch} at {sha}") + errors += [error for error in (run_error(github, run, name) for run in found) if error is not None] return errors -def coverage_error(github: GitHub, workflow: str, job: str, suite: str, shas: List[str], tag: str, - changed: Callable[[str, str], bool], sleep: Callable[[float], None], - log: Callable[[str], None]) -> Optional[str]: - # The walk ends at the newest commit that changed the files the job reads: a run on any - # commit before it never covered that change, however many skipped commits follow it. - newer = None - for sha in shas: - is_tag = newer is None - if is_green(github, workflow, job, sha, sleep, log): - if not is_tag and changed(sha, suite): - return f"{job} passed on {sha}, but files it reads changed before {tag}" - log(f"{job}: green on {sha}") - return None - if not is_tag and changed(sha, suite): - return f"no {job} run that was not skipped on {newer}, which changed files it reads, or after it" - newer = sha - return f"no {job} run that was not skipped among the {len(shas)} first-parent commits of {tag}" - - -def check(tag: str, git: Git, github: GitHub, changed: Callable[[str, str], bool], - sleep: Callable[[float], None] = time.sleep, log: Callable[[str], None] = print) -> List[str]: +def check(tag: str, git: Git, github: GitHub, changed: Callable[[str, str, str], bool]) -> List[str]: try: version = parse_tag(tag) except ValueError as error: @@ -357,13 +314,14 @@ def check(tag: str, git: Git, github: GitHub, changed: Callable[[str, str], bool return [f"tag {tag} does not exist here"] releases = github.releases() errors = branch_errors(git, version) + errors += shared_commit_errors(git, version) errors += beta_errors(git, version, releases) - errors += stable_tree_errors(git, version, releases) + errors += stable_tree_errors(git, version, releases, changed) errors += changelog_errors(git.show(tag, "./CHANGELOG.md"), version, published_stables(releases)) errors += fragment_errors(git.list_tree(tag, FRAGMENTS), version) if errors: return errors - return ci_errors(git, github, version, changed, sleep, log) + return push_run_errors(github, git.commit(tag), version.line) def main(argv: Optional[List[str]] = None) -> int: diff --git a/scripts/ci/detect-changed-paths.sh b/scripts/ci/detect-changed-paths.sh index ee60420dce..5f53a16733 100755 --- a/scripts/ci/detect-changed-paths.sh +++ b/scripts/ci/detect-changed-paths.sh @@ -1,14 +1,19 @@ #!/usr/bin/env bash set -euo pipefail -# Prints "true" when a suite has to run for the changes since , "false" when it can skip. -# Usage: detect-changed-paths.sh [--base ] macos|plugins|ios|oracle|hana|release|linux +# Prints "true" when a suite has to run for the changes from to , "false" when it can skip. +# Usage: detect-changed-paths.sh [--base ] [--head ] [--ref ] +# Suites: macos, plugins, ios, oracle, hana, release, shipped, linux. BASE="" +HEAD_REV="HEAD" +REF="" SUITE="" while [ $# -gt 0 ]; do case "$1" in --base) BASE="${2:-}"; shift 2 ;; + --head) HEAD_REV="${2:-}"; shift 2 ;; + --ref) REF="${2:-}"; shift 2 ;; -*) echo "detect-changed-paths.sh: unknown flag $1" >&2; exit 2 ;; *) SUITE="$1"; shift ;; esac @@ -53,7 +58,7 @@ case "$SUITE" in '\.github/actions/setup-xcode/' '\.github/workflows/ios-tests\.yml$') ;; hana) PATTERNS=('apple/Native/HanaBridge/' '\.github/workflows/repo-hygiene\.yml$') ;; - release) + release|shipped) # A Release build compiles no test target, so the test folders under Plugins/ and # Packages/ are skipped too. PATTERNS=('apple/TablePro/' 'apple/Plugins/' 'apple/Packages/' 'apple/Native/' 'apple/Configs/' 'apple/Libs/' @@ -65,18 +70,37 @@ case "$SUITE" in 'apple/scripts/release/(notes|registry|version)\.py$' 'apple/scripts/ci/verify-plugin-loads\.sh$' 'scripts/ci/detect-changed-paths\.sh$' '\.github/actions/(setup-xcode(gen)?|prepare-build)/' '\.github/workflows/release-build\.yml$') - SKIP='apple/Plugins/[^/]+Tests/|apple/Packages/[^/]+/Tests/' ;; + SKIP='apple/Plugins/[^/]+Tests/|apple/Packages/[^/]+/Tests/' + # shipped is what an X.Y.0 may not change after its newest beta: the release build minus its CI. + if [ "$SUITE" = shipped ]; then + KEPT=() + for pattern in "${PATTERNS[@]}"; do + case "$pattern" in + '\.github/workflows/'* | '\.github/actions/'*) ;; + *) KEPT+=("$pattern") ;; + esac + done + PATTERNS=("${KEPT[@]}") + fi ;; linux) PATTERNS=('linux/' 'scripts/ci/detect-changed-paths\.sh$' '\.github/workflows/(build-linux|linux-packages|release-linux)\.yml$') ;; *) - echo "detect-changed-paths.sh: expected macos, plugins, ios, oracle, hana, release or linux, got '$SUITE'" >&2 + echo "detect-changed-paths.sh: expected macos, plugins, ios, oracle, hana, release, shipped or linux, got '$SUITE'" >&2 exit 2 ;; esac +# Every suite runs on a release line: a tag's verify needs its commit's own runs, and a release PR +# changes only the changelog. +case "$REF" in + refs/heads/release/*) + echo "true" + exit 0 ;; +esac + # A new branch has no previous commit, so it is compared with the point where it left main. if [ -n "$BASE" ] && [ -z "${BASE//0/}" ]; then - BASE="$(git merge-base HEAD origin/main 2> /dev/null || true)" + BASE="$(git merge-base "$HEAD_REV" origin/main 2> /dev/null || true)" fi if [ -z "$BASE" ]; then @@ -89,7 +113,7 @@ fi CHANGED="$(mktemp)" RELEVANT="$(mktemp)" trap 'rm -f "$CHANGED" "$RELEVANT"' EXIT -git -c core.quotePath=false diff --no-renames --name-only -z "$BASE" HEAD > "$CHANGED" +git -c core.quotePath=false diff --no-renames --name-only -z "$BASE" "$HEAD_REV" > "$CHANGED" if [ -n "$SKIP" ]; then LC_ALL=C grep -zvE "^($SKIP)" "$CHANGED" > "$RELEVANT" || [ $? -eq 1 ] else diff --git a/scripts/ci/test_detect_changed_paths.py b/scripts/ci/test_detect_changed_paths.py index c7d184fb5e..3cb1d22d2e 100644 --- a/scripts/ci/test_detect_changed_paths.py +++ b/scripts/ci/test_detect_changed_paths.py @@ -50,10 +50,14 @@ def commit(self, paths): self.git("commit", "-q", "-m", "change") return self.git("rev-parse", "HEAD") - def detect(self, suite, base=None): + def detect(self, suite, base=None, head=None, ref=None): args = ["bash", "scripts/ci/detect-changed-paths.sh"] if base is not None: args += ["--base", base] + if head is not None: + args += ["--head", head] + if ref is not None: + args += ["--ref", ref] result = subprocess.run(args + [suite], cwd=self.repo, env=self.env, capture_output=True, text=True) return result.returncode, result.stdout.strip() @@ -90,7 +94,7 @@ def test_linux_skips_the_shared_doc_scripts(self): self.assertEqual(self.verdict_after("linux", path), "false") def test_unknown_flag_exits_2(self): - self.assertEqual(self.detect("--head")[0], 2) + self.assertEqual(self.detect("--since")[0], 2) def test_no_base_runs_the_suite(self): self.assertEqual(self.detect("release"), (0, "true")) @@ -205,8 +209,58 @@ def test_plugins_skips_the_app_its_tests_and_other_workflows(self): with self.subTest(path=path): self.assertEqual(self.verdict_after("plugins", path), "false") + def test_shipped_is_the_release_build_without_its_ci(self): + for path in [ + "apple/TablePro/App.swift", + "apple/Plugins/OracleDriverPlugin/OraclePlugin.swift", + "apple/Packages/TableProCore/Sources/Core.swift", + "apple/project.yml", + ".github/plugin-registry.json", + "apple/scripts/release/build.sh", + "apple/scripts/release/version.py", + ]: + with self.subTest(path=path): + self.assertEqual(self.verdict_after("shipped", path), "true") + for path in [ + ".github/workflows/release-build.yml", + ".github/workflows/release.yml", + ".github/actions/setup-xcode/action.yml", + ".github/actions/prepare-build/action.yml", + "apple/Plugins/MySQLDriverTests/MySQLTests.swift", + "apple/Packages/TableProCore/Tests/CoreTests.swift", + "apple/TableProTests/AppTests.swift", + ]: + with self.subTest(path=path): + self.assertEqual(self.verdict_after("shipped", path), "false") + + def test_a_release_branch_runs_every_suite(self): + # The ios suite reads its shared paths out of the iOS project spec. + spec = self.repo / "apple/TableProMobile/project.yml" + spec.parent.mkdir(parents=True) + spec.write_text("sources:\n - path: ../Packages/TableProCore\n", encoding="utf-8") + base = self.commit([]) + self.git("update-ref", "refs/remotes/origin/main", base) + self.commit(["apple/CHANGELOG.md"]) + for suite in ["macos", "plugins", "ios", "oracle", "hana", "release", "linux"]: + with self.subTest(suite=suite): + self.assertEqual(self.detect(suite, base), (0, "false")) + self.assertEqual(self.detect(suite, base, ref="refs/heads/release/0.83"), (0, "true")) + self.assertEqual(self.detect(suite, ZERO, ref="refs/heads/release/0.83"), (0, "true")) + + def test_main_and_pull_request_refs_keep_the_path_filter(self): + self.commit(["apple/CHANGELOG.md"]) + for ref in ["refs/heads/main", "refs/pull/12/merge", "refs/heads/backport/12-0.83"]: + with self.subTest(ref=ref): + self.assertEqual(self.detect("macos", self.seed, ref=ref), (0, "false")) + + def test_head_ends_the_diff(self): + change = self.commit(["apple/TablePro/App.swift"]) + self.git("checkout", "-q", "--detach", self.seed) + self.assertEqual(self.detect("macos", self.seed), (0, "false")) + self.assertEqual(self.detect("macos", self.seed, head=change), (0, "true")) + def test_changelog_files_run_no_suite(self): - for suite in ["macos", "plugins", "oracle", "hana", "release"]: + for suite in ["macos", "plugins", "oracle", "hana", "release", "shipped"]: for path in ["apple/CHANGELOG.md", "apple/changelog.d/3500.added.md", "apple/towncrier.toml", "appcast.xml"]: with self.subTest(suite=suite, path=path): self.assertEqual(self.verdict_after(suite, path), "false")