diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 863d615..047b9df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,20 +29,11 @@ concurrency: cancel-in-progress: false jobs: - validate: + # Holds HOMEBREW_TAP_TOKEN, so it never checks out repository code. + verify-homebrew-token: runs-on: ubuntu-latest - permissions: - contents: read + permissions: {} steps: - - uses: actions/checkout@v6 - with: - ref: ${{ env.RELEASE_REF }} - persist-credentials: false - - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - name: Verify Homebrew publishing credential env: GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} @@ -57,15 +48,66 @@ jobs: exit 1 fi if [ "$can_push" != "true" ]; then - echo "::error::HOMEBREW_TAP_TOKEN cannot push to coval-ai/homebrew-tap. Replace it with a token that has Contents read/write access." + echo "::error::HOMEBREW_TAP_TOKEN cannot push to coval-ai/homebrew-tap. Replace it with a token that has Contents and Pull requests read/write access." + exit 1 + fi + + # The tap accepts changes only through pull requests. This request + # names branches that do not exist, so it can never open one; GitHub + # returns 422 only after authorizing the Pull requests write. + probe_suffix="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + probe_response="$RUNNER_TEMP/homebrew-tap-pull-probe.json" + if ! pr_status="$( + curl --silent --show-error --connect-timeout 10 --max-time 30 \ + --output "$probe_response" \ + --write-out '%{http_code}' \ + --request POST \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --data "{\"title\":\"permission probe\",\"head\":\"__permission_probe_head_${probe_suffix}__\",\"base\":\"__permission_probe_base_${probe_suffix}__\"}" \ + https://api.github.com/repos/coval-ai/homebrew-tap/pulls + )"; then + echo "::error::Could not reach GitHub to check that HOMEBREW_TAP_TOKEN can open pull requests on coval-ai/homebrew-tap." + exit 1 + fi + if [ "$pr_status" != "422" ]; then + message="$(jq -r '.message // empty' "$probe_response" 2>/dev/null || true)" + echo "::error::HOMEBREW_TAP_TOKEN cannot open pull requests on coval-ai/homebrew-tap (HTTP ${pr_status}${message:+: $message}). Replace it with a token that has Contents and Pull requests read/write access." exit 1 fi + validate: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # Runs the workflow's own version check against the release commit's + # manifests, fetched as data, so no code from RELEASE_REF executes here. + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Match Cargo manifests and release tag - run: python scripts/release_version.py --expected-tag "$RELEASE_TAG" + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + manifests="${RUNNER_TEMP}/release-manifests" + mkdir -p "$manifests" + for file in Cargo.toml Cargo.lock; do + gh api -H 'Accept: application/vnd.github.raw+json' \ + "repos/${GITHUB_REPOSITORY}/contents/${file}?ref=${RELEASE_REF}" \ + > "${manifests}/${file}" + done + python scripts/release_version.py --root "$manifests" --expected-tag "$RELEASE_TAG" build: - needs: validate + needs: [verify-homebrew-token, validate] permissions: contents: read strategy: @@ -166,11 +208,14 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + outputs: + tap_pr_url: ${{ steps.tap.outputs.pr_url }} steps: + # This job holds HOMEBREW_TAP_TOKEN, so it runs the formula renderer from + # the workflow's own ref (main, or the pushed tag), never RELEASE_REF. - uses: actions/checkout@v6 with: - ref: ${{ env.RELEASE_REF }} persist-credentials: false - uses: actions/setup-python@v6 @@ -195,7 +240,8 @@ jobs: echo "linux_x64=$(grep 'coval-linux-x64' SHA256SUMS | awk '{print $1}')" } >> "$GITHUB_OUTPUT" - - name: Update Homebrew tap + - name: Open or update the Homebrew tap pull request + id: tap env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} VERSION: ${{ env.RELEASE_TAG }} @@ -203,6 +249,7 @@ jobs: SHA_MACOS_X64: ${{ steps.sha.outputs.macos_x64 }} SHA_LINUX_ARM64: ${{ steps.sha.outputs.linux_arm64 }} SHA_LINUX_X64: ${{ steps.sha.outputs.linux_x64 }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail if [ -z "$HOMEBREW_TAP_TOKEN" ]; then @@ -210,6 +257,12 @@ jobs: exit 1 fi + # The tap's rulesets reject direct pushes to main, so the release + # pushes its own branch and opens a pull request for review. + tap_repo="coval-ai/homebrew-tap" + branch="release/coval-${VERSION}" + title="chore: update coval to ${VERSION#v}" + body_file="${RUNNER_TEMP}/homebrew-tap-pr-body.md" tap_dir="${RUNNER_TEMP}/homebrew-tap" auth_header="Authorization: Basic $(printf 'x-access-token:%s' "$HOMEBREW_TAP_TOKEN" | base64 | tr -d '\n')" git_auth() { @@ -219,7 +272,7 @@ jobs: git "$@" } - git_auth clone "https://github.com/coval-ai/homebrew-tap.git" "$tap_dir" + git_auth clone --branch main "https://github.com/${tap_repo}.git" "$tap_dir" mkdir -p "$tap_dir/Formula" python scripts/render_homebrew_formula.py \ --version "${VERSION#v}" \ @@ -227,34 +280,63 @@ jobs: --macos-x64 "$SHA_MACOS_X64" \ --linux-arm64 "$SHA_LINUX_ARM64" \ --linux-x64 "$SHA_LINUX_X64" \ - --output "$tap_dir/Formula/coval.rb" + --output "$tap_dir/Formula/coval.rb" \ + --pull-request-body "$body_file" \ + --run-url "$RUN_URL" if [ -z "$(git -C "$tap_dir" status --porcelain -- Formula/coval.rb)" ]; then echo "Homebrew formula already matches ${VERSION}; nothing to push." exit 0 fi - git -C "$tap_dir" config user.name "github-actions[bot]" - git -C "$tap_dir" config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git -C "$tap_dir" add Formula/coval.rb - git -C "$tap_dir" commit -m "chore: update coval to ${VERSION#v}" - for attempt in 1 2 3; do - if git_auth -C "$tap_dir" push origin HEAD:main; then - exit 0 - fi - if [ "$attempt" -eq 3 ]; then - break - fi - echo "Push attempt ${attempt} failed; rebasing on origin/main." - git_auth -C "$tap_dir" fetch origin main - git -C "$tap_dir" rebase origin/main - sleep $((attempt * 5)) - done - echo "::error::Could not push the Homebrew formula update after 3 attempts." - exit 1 + # A rerun reuses the release branch when it already carries this + # formula. Rebuilt release assets change the checksums, so the branch + # is then replaced with a fresh commit on main. + remote_sha="$(git_auth -C "$tap_dir" ls-remote origin "refs/heads/${branch}" | cut -f1)" + if [ -n "$remote_sha" ] \ + && git_auth -C "$tap_dir" fetch --quiet origin "refs/heads/${branch}" \ + && git -C "$tap_dir" diff --quiet FETCH_HEAD -- Formula/coval.rb; then + echo "${branch} already carries the ${VERSION} formula." + else + git -C "$tap_dir" switch --create "$branch" + git -C "$tap_dir" config user.name "github-actions[bot]" + git -C "$tap_dir" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$tap_dir" add Formula/coval.rb + git -C "$tap_dir" commit -m "$title" + git_auth -C "$tap_dir" push \ + --force-with-lease="refs/heads/${branch}:${remote_sha}" \ + origin "HEAD:refs/heads/${branch}" + fi + + export GH_TOKEN="$HOMEBREW_TAP_TOKEN" + pr_number="$( + gh api "repos/${tap_repo}/pulls?head=coval-ai:${branch}&base=main&state=open" \ + --jq '.[0].number // empty' + )" + if [ -n "$pr_number" ]; then + pr_url="$( + gh api --method PATCH "repos/${tap_repo}/pulls/${pr_number}" \ + -f title="$title" \ + -F body=@"$body_file" \ + --jq '.html_url' + )" + echo "Updated ${pr_url}." + else + pr_url="$( + gh api --method POST "repos/${tap_repo}/pulls" \ + -f title="$title" \ + -f head="$branch" \ + -f base=main \ + -F body=@"$body_file" \ + --jq '.html_url' + )" + echo "Opened ${pr_url}." + fi + echo "pr_url=${pr_url}" >> "$GITHUB_OUTPUT" + echo "Homebrew picks up ${VERSION} once ${pr_url} merges." >> "$GITHUB_STEP_SUMMARY" report-failure: - needs: [validate, build, release, update-homebrew] + needs: [verify-homebrew-token, validate, build, release, update-homebrew] if: ${{ always() && contains(needs.*.result, 'failure') }} runs-on: ubuntu-latest permissions: @@ -291,7 +373,7 @@ jobs: } close-recovered-failure: - needs: [validate, build, release, update-homebrew] + needs: [verify-homebrew-token, validate, build, release, update-homebrew] if: ${{ always() && needs.update-homebrew.result == 'success' }} runs-on: ubuntu-latest permissions: @@ -299,9 +381,16 @@ jobs: steps: - name: Close a recovered release failure issue uses: actions/github-script@v7 + env: + TAP_PR_URL: ${{ needs.update-homebrew.outputs.tap_pr_url }} with: script: | const title = 'CLI release workflow is failing'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const prUrl = process.env.TAP_PR_URL; + const body = prUrl + ? `Recovered in ${runUrl}. Homebrew picks up \`${process.env.RELEASE_TAG}\` once ${prUrl} merges.` + : `Recovered in ${runUrl}.`; const issues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, repo: context.repo.repo, @@ -314,7 +403,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: existing.number, - body: `Recovered in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`, + body, }); await github.rest.issues.update({ owner: context.repo.owner, diff --git a/README.md b/README.md index caf3044..ddcccea 100644 --- a/README.md +++ b/README.md @@ -479,8 +479,11 @@ Publishing a new version: 3. Merge only after current-head CI and review are green. The successful CI run for that exact `main` commit triggers `Release on version bump`. 4. Verify the `v*` tag, all five binary artifacts, `SHA256SUMS`, and the GitHub - release. Then verify that `coval-ai/homebrew-tap` contains the same version - and that `brew update && brew upgrade coval-ai/tap/coval` installs it. + release. The release opens a `release/coval-vX.Y.Z` pull request on + `coval-ai/homebrew-tap`, because the tap accepts changes to `main` only + through reviewed pull requests. Get it approved and merged, then verify that + the tap contains the same version and that + `brew update && brew upgrade coval-ai/tap/coval` installs it. If the automatic release needs a retry, dispatch `Release on version bump` from the `main` branch. It reuses an existing matching tag safely. Do not hand-create @@ -490,7 +493,8 @@ all release credentials. A successful GitHub release is not complete publishing proof until the Homebrew formula reports the same version. To recover Homebrew publishing, replace the token interactively, dispatch the -retry from `main`, and verify both destinations: +retry from `main`, merge the tap pull request it opens or updates, and verify +both destinations: ```bash gh secret set HOMEBREW_TAP_TOKEN --repo coval-ai/cli @@ -506,18 +510,20 @@ run_id="$( )" gh run watch "$run_id" --repo coval-ai/cli --exit-status gh release view "$expected_tag" --repo coval-ai/cli +gh pr view "release/coval-$expected_tag" --repo coval-ai/homebrew-tap +# After that pull request merges: gh api -H 'Accept: application/vnd.github.raw+json' \ repos/coval-ai/homebrew-tap/contents/Formula/coval.rb | \ grep -F "version \"$expected_version\"" ``` -The release validation checks that `HOMEBREW_TAP_TOKEN` can push to -`coval-ai/homebrew-tap` before building artifacts. As with `REGEN_PR_TOKEN`, a +The release validation checks that `HOMEBREW_TAP_TOKEN` can push to and open +pull requests on `coval-ai/homebrew-tap` before building artifacts. As with `REGEN_PR_TOKEN`, a secret record alone does not prove that the credential is present or authorized. The release workflow validates tag/version consistency, builds all five target -binaries, creates or updates the GitHub release, and updates -`coval-ai/homebrew-tap`. A manual `Release on version bump` dispatch safely +binaries, creates or updates the GitHub release, and opens or updates the +`coval-ai/homebrew-tap` pull request. A manual `Release on version bump` dispatch safely retries the current version without creating another tag. Repository prerequisite: @@ -526,8 +532,11 @@ Repository prerequisite: read/write access to `coval-ai/cli`. The organization does not allow `GITHUB_TOKEN` to create pull requests. - `HOMEBREW_TAP_TOKEN`: a fine-grained token or GitHub App token with Contents - read/write access to `coval-ai/homebrew-tap`. The Homebrew update is - idempotent, so retrying an already-current formula succeeds without a commit. + and Pull requests read/write access to `coval-ai/homebrew-tap`. The Homebrew + update is idempotent: a retry reuses the release branch and its open pull + request, and an already-current formula succeeds without a commit. The tap + requires approval from someone other than the last pusher, so the token's + owner cannot approve the release pull request. ## Configuration diff --git a/scripts/render_homebrew_formula.py b/scripts/render_homebrew_formula.py index 0c012f4..ea15cbe 100644 --- a/scripts/render_homebrew_formula.py +++ b/scripts/render_homebrew_formula.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Render the Homebrew formula for one validated Coval CLI release.""" +"""Render the Homebrew formula and tap PR body for one validated CLI release.""" from __future__ import annotations @@ -13,13 +13,20 @@ from release_version import VERSION_RE SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +RELEASES_URL = "https://github.com/coval-ai/cli/releases" +PLATFORM_ASSETS = { + "macos_arm64": "coval-macos-arm64.tar.gz", + "macos_x64": "coval-macos-x64.tar.gz", + "linux_arm64": "coval-linux-arm64.tar.gz", + "linux_x64": "coval-linux-x64.tar.gz", +} -def render_formula(version: str, checksums: dict[str, str]) -> str: +def validate_release(version: str, checksums: dict[str, str]) -> None: if VERSION_RE.fullmatch(version) is None: raise ValueError(f"invalid release version: {version!r}") - expected_platforms = {"macos_arm64", "macos_x64", "linux_arm64", "linux_x64"} + expected_platforms = set(PLATFORM_ASSETS) if set(checksums) != expected_platforms: missing = sorted(expected_platforms - set(checksums)) unexpected = sorted(set(checksums) - expected_platforms) @@ -30,6 +37,10 @@ def render_formula(version: str, checksums: dict[str, str]) -> str: if SHA256_RE.fullmatch(checksum) is None: raise ValueError(f"invalid SHA-256 for {platform}: {checksum!r}") + +def render_formula(version: str, checksums: dict[str, str]) -> str: + validate_release(version, checksums) + return f'''class Coval < Formula desc "CLI for Coval AI agent evaluation platform" homepage "https://coval.dev" @@ -67,6 +78,29 @@ def install ''' +def render_pull_request_body( + version: str, checksums: dict[str, str], run_url: str +) -> str: + validate_release(version, checksums) + + download_url = f"{RELEASES_URL}/download/v{version}" + rows = "\n".join( + f"| [`{asset}`]({download_url}/{asset}) | `{checksums[platform]}` |" + for platform, asset in PLATFORM_ASSETS.items() + ) + return f"""Updates `Formula/coval.rb` to Coval CLI {version}. + +Release: {RELEASES_URL}/tag/v{version} +Checksums: {download_url}/SHA256SUMS + +| Asset | SHA-256 | +| --- | --- | +{rows} + +Opened by the CLI release workflow: {run_url} +""" + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--version", required=True) @@ -75,18 +109,23 @@ def main() -> int: parser.add_argument("--linux-arm64", required=True) parser.add_argument("--linux-x64", required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--pull-request-body", type=Path) + parser.add_argument("--run-url") args = parser.parse_args() - - formula = render_formula( - args.version, - { - "macos_arm64": args.macos_arm64, - "macos_x64": args.macos_x64, - "linux_arm64": args.linux_arm64, - "linux_x64": args.linux_x64, - }, - ) - args.output.write_text(formula) + if args.pull_request_body and not args.run_url: + parser.error("--pull-request-body requires --run-url") + + checksums = { + "macos_arm64": args.macos_arm64, + "macos_x64": args.macos_x64, + "linux_arm64": args.linux_arm64, + "linux_x64": args.linux_x64, + } + args.output.write_text(render_formula(args.version, checksums)) + if args.pull_request_body: + args.pull_request_body.write_text( + render_pull_request_body(args.version, checksums, args.run_url) + ) return 0 diff --git a/scripts/test_release_automation.py b/scripts/test_release_automation.py index b21c91f..2a535cc 100644 --- a/scripts/test_release_automation.py +++ b/scripts/test_release_automation.py @@ -120,6 +120,27 @@ def test_rejects_invalid_checksum(self): with self.assertRaisesRegex(ValueError, "invalid SHA-256"): render_homebrew_formula.render_formula("0.6.0", self.checksums) + def test_pull_request_body_lists_release_assets_and_checksums(self): + run_url = "https://github.com/coval-ai/cli/actions/runs/1" + body = render_homebrew_formula.render_pull_request_body( + "0.6.0", self.checksums, run_url + ) + + releases = "https://github.com/coval-ai/cli/releases" + self.assertIn(f"{releases}/tag/v0.6.0", body) + self.assertIn(f"{releases}/download/v0.6.0/SHA256SUMS", body) + self.assertIn( + f"({releases}/download/v0.6.0/coval-linux-arm64.tar.gz) | `{'c' * 64}` |", + body, + ) + self.assertIn(run_url, body) + + def test_pull_request_body_rejects_invalid_version(self): + with self.assertRaisesRegex(ValueError, "invalid release version"): + render_homebrew_formula.render_pull_request_body( + "v0.6.0", self.checksums, "https://example.invalid" + ) + if __name__ == "__main__": unittest.main()