diff --git a/.dockerignore b/.dockerignore index 4c49bd78f..76d736afd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,8 @@ -.env +.env* +.git +.aws +.agents +.codex +__pycache__ +dist +s5cmd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51939fd3b..542c9b0f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,19 @@ name: ci on: - workflow_dispatch: {} + workflow_dispatch: + inputs: + expected_head: + description: Expected release candidate commit (empty for an ordinary manual run) + default: '' + type: string + producer_run_id: + description: Trusted release workflow run + default: '' + type: string + producer_attempt: + description: Trusted release workflow attempt + default: '' + type: string pull_request: branches: - "coreweave" @@ -10,7 +23,26 @@ on: permissions: contents: read jobs: + release-preflight: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Verify release candidate + if: github.event_name == 'workflow_dispatch' && inputs.expected_head != '' + run: python3 scripts/release_gate.py ci-preflight + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_HEAD: ${{ inputs.expected_head }} + PRODUCER_RUN_ID: ${{ inputs.producer_run_id }} + PRODUCER_ATTEMPT: ${{ inputs.producer_attempt }} build: + needs: release-preflight strategy: matrix: go-version: @@ -35,6 +67,7 @@ jobs: - run: make build test: + needs: release-preflight strategy: matrix: go-version: @@ -50,19 +83,6 @@ jobs: runs-on: ${{ matrix.os }}-latest env: GOFLAGS: ${{ matrix.os == 'macos' && '-ldflags=-linkmode=external' || '' }} - services: - minio: - image: ${{ (matrix.os == 'ubuntu') && 'bitnamilegacy/minio:2023.7.18' || ''}} - ports: - - 45677:9000 - options: >- - --health-cmd "curl -I http://localhost:9000/minio/health/live -s" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -71,10 +91,34 @@ jobs: with: go-version: ${{ matrix.go-version }} - - run: make test + - name: Build pinned local test storage + if: matrix.os == 'ubuntu' + run: GOBIN="$RUNNER_TEMP/minio-bin" go install github.com/minio/minio@0120ff93bc4b9cfaf2865e55850e9b20e5ef703d # RELEASE.2023-07-18T17-49-40Z + - name: Run tests + shell: bash + run: | + if [ -n "$S5CMD_TEST_ENDPOINT_URL" ]; then + "$RUNNER_TEMP/minio-bin/minio" server "$RUNNER_TEMP/minio-data" --address 127.0.0.1:45677 >"$RUNNER_TEMP/minio.log" 2>&1 & + minio_pid=$! + trap 'kill "$minio_pid" 2>/dev/null || true; wait "$minio_pid" 2>/dev/null || true' EXIT + for ((attempt=0; attempt<30; attempt++)); do + if curl --fail --silent --max-time 2 "$S5CMD_TEST_ENDPOINT_URL/minio/health/live" >/dev/null; then + break + fi + sleep 1 + done + if ! curl --fail --silent --max-time 2 "$S5CMD_TEST_ENDPOINT_URL/minio/health/live" >/dev/null; then + echo 'Local test storage did not become healthy.' + exit 1 + fi + fi + make test env: S5CMD_TEST_ENDPOINT_URL: ${{ (matrix.os == 'ubuntu') && 'http://localhost:45677' || '' }} + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin qa: + needs: release-preflight strategy: matrix: go-version: @@ -93,10 +137,16 @@ jobs: with: go-version: ${{ matrix.go-version }} + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.11' + - name: Install pinned Semgrep without a container pull + run: python -m pip install --disable-pip-version-check semgrep==1.65.0 - run: make bootstrap - run: make check credential-helper: + needs: release-preflight runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 1b365a4c4..000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: docker -on: - workflow_dispatch: {} - push: - tags: - - 'v*' -jobs: - multi: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log into registry ghcr.io - uses: docker/login-action@v3.3.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Get tag - id: get_tag - run: | - git remote add upstream https://github.com/peak/s5cmd.git - git fetch --tags upstream - VERSION=$(git describe --tags --exclude '*-' --abbrev=0) - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "short_sha=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_OUTPUT - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/arm/v6,linux/arm/v7,linux/386 - tags: | - ghcr.io/coreweave/s5cmd:latest - ghcr.io/coreweave/s5cmd:${{ steps.get_tag.outputs.version }}-${{ steps.get_tag.outputs.short_sha }} - diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml deleted file mode 100644 index 0734e8820..000000000 --- a/.github/workflows/goreleaser.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: goreleaser -on: - workflow_dispatch: {} - push: - tags: - - 'v*' -jobs: - caios_test: - name: caios-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: 1.22.x - - run: make test_without_race - env: - S5CMD_ACCESS_KEY_ID: ${{ secrets.S5CMD_CAIOS_ACCESS_KEY_ID }} - S5CMD_SECRET_ACCESS_KEY: ${{ secrets.S5CMD_CAIOS_SECRET_ACCESS_KEY }} - S5CMD_ENDPOINT_URL: https://cwobject.com - S5CMD_IS_VIRTUAL_HOST: 1 - S5CMD_REGION: ${{ secrets.S5CMD_CAIOS_REGION }} - S5CMD_I_KNOW_WHAT_IM_DOING: 1 - - goreleaser: - needs: [ caios_test ] - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - steps: - - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.22' - - - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: '1.18.2' - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..3201b1e68 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,175 @@ +name: Release +on: + push: + branches: [coreweave] + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: release-coreweave + cancel-in-progress: false + +jobs: + authorize: + if: github.ref == 'refs/heads/coreweave' && vars.RELEASE_AUTOMATION_ENABLED == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + checks: read + actions: read + outputs: + publish: ${{ steps.verify.outputs.publish }} + plan: ${{ steps.verify.outputs.plan }} + tag: ${{ steps.verify.outputs.tag }} + version: ${{ steps.verify.outputs.version }} + commit: ${{ steps.verify.outputs.commit }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - id: verify + run: python3 scripts/release_gate.py publication-plan + env: + GH_TOKEN: ${{ github.token }} + RELEASE_AUTOMATION_ENABLED: ${{ vars.RELEASE_AUTOMATION_ENABLED }} + + publish: + needs: authorize + if: needs.authorize.outputs.publish == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: release + permissions: + contents: write + packages: write + pull-requests: write + env: + RELEASE_PLAN: ${{ needs.authorize.outputs.plan }} + RELEASE_VERSION: ${{ needs.authorize.outputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.authorize.outputs.commit }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.22.12' + cache: false + - name: Reserve the verified tag and draft release + id: prepare + run: python3 scripts/release_gate.py prepare-release + env: + GH_TOKEN: ${{ github.token }} + - uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6 + if: steps.prepare.outputs.build == 'true' + with: + version: v1.18.2 + args: release --snapshot --skip-publish --rm-dist + - name: Upload archives, packages and checksums + if: steps.prepare.outputs.build == 'true' + run: gh release upload "$RELEASE_TAG" dist/*.tar.gz dist/*.zip dist/*.deb dist/*checksums.txt --clobber + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.authorize.outputs.tag }} + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + if: steps.prepare.outputs.build == 'true' + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + if: steps.prepare.outputs.build == 'true' + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + if: steps.prepare.outputs.build == 'true' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Select version tags without moving latest backward + if: steps.prepare.outputs.build == 'true' + id: tags + run: python3 scripts/release_gate.py publication-tags + env: + GH_TOKEN: ${{ github.token }} + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + if: steps.prepare.outputs.build == 'true' + with: + context: . + push: true + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/arm/v6,linux/arm/v7,linux/386 + build-args: | + VERSION=${{ needs.authorize.outputs.version }} + BUILD=${{ needs.authorize.outputs.commit }} + tags: ${{ steps.tags.outputs.tags }} + - name: Publish completed release + run: python3 scripts/release_gate.py finalize-release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.prepare.outputs.release }} + + release-please: + needs: [authorize, publish] + if: always() && github.ref == 'refs/heads/coreweave' && vars.RELEASE_AUTOMATION_ENABLED == 'true' && needs.authorize.result == 'success' && (needs.publish.result == 'success' || needs.publish.result == 'skipped') + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + issues: write + outputs: + candidate: ${{ steps.capture.outputs.candidate }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Refresh PRs only from the current target tip + id: tip + run: python3 scripts/release_gate.py release-tip + env: + GH_TOKEN: ${{ github.token }} + - name: Update the release PR + if: steps.tip.outputs.current == 'true' + id: release + uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 + with: + token: ${{ github.token }} + target-branch: coreweave + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + skip-github-release: true + - name: Record the release candidate + if: steps.tip.outputs.current == 'true' + id: capture + run: python3 scripts/release_gate.py capture-candidate + env: + GH_TOKEN: ${{ github.token }} + RELEASE_PLEASE_PRS: ${{ steps.release.outputs.prs }} + - name: Store candidate source hashes + if: steps.capture.outputs.candidate == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-candidate-${{ github.run_attempt }} + path: release-candidate.json + if-no-files-found: error + retention-days: 90 + + dispatch-validation: + needs: release-please + if: always() && needs.release-please.result == 'success' && needs.release-please.outputs.candidate == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + actions: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Explicitly schedule candidate CI and storage acceptance + run: python3 scripts/release_gate.py dispatch-validation + env: + GH_TOKEN: ${{ github.token }} + PRODUCER_RUN_ID: ${{ github.run_id }} + PRODUCER_ATTEMPT: ${{ github.run_attempt }} diff --git a/.github/workflows/storage-gate.yml b/.github/workflows/storage-gate.yml new file mode 100644 index 000000000..3d1f19b59 --- /dev/null +++ b/.github/workflows/storage-gate.yml @@ -0,0 +1,148 @@ +name: Storage gate +on: + pull_request_target: + branches: [coreweave] + types: [opened, synchronize, reopened, edited, labeled, unlabeled, ready_for_review] + push: + branches: [coreweave] + workflow_dispatch: + inputs: + pr: + description: Release PR number + required: true + type: string + producer_run_id: + description: Trusted release workflow run + required: true + type: string + producer_attempt: + description: Trusted release workflow attempt + required: true + type: string + +permissions: + contents: read + +concurrency: + group: storage-gate-${{ github.event_name == 'workflow_dispatch' && 'dispatch' || 'event' }}-${{ inputs.pr || github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +jobs: + inspect: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + checks: write + actions: read + outputs: + live: ${{ steps.verify.outputs.live }} + proof: ${{ steps.verify.outputs.proof }} + check: ${{ steps.verify.outputs.check }} + base: ${{ steps.verify.outputs.base }} + merge: ${{ steps.verify.outputs.merge }} + pr: ${{ steps.verify.outputs.pr }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - id: verify + run: python3 scripts/release_gate.py inspect + env: + GH_TOKEN: ${{ github.token }} + RELEASE_AUTOMATION_ENABLED: ${{ vars.RELEASE_AUTOMATION_ENABLED }} + + acceptance: + needs: inspect + if: needs.inspect.outputs.live == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: storage-acceptance + permissions: + contents: read + pull-requests: read + id-token: write + concurrency: + group: storage-live-${{ needs.inspect.outputs.pr }} + cancel-in-progress: false + env: + CWIC_API_URL: ${{ secrets.STORAGE_API_URL }} + S5CMD_TEST_ORG: ${{ secrets.STORAGE_ORG }} + S5CMD_TEST_ENDPOINT_URL: ${{ secrets.STORAGE_ENDPOINT_URL }} + S5CMD_REGION: ${{ secrets.STORAGE_REGION }} + S5CMD_OIDC_SUBJECT: ${{ secrets.STORAGE_OIDC_SUBJECT }} + S5CMD_TEST_MODE: live + S5CMD_IS_VIRTUAL_HOST: 'true' + S5CMD_I_KNOW_WHAT_IM_DOING: '1' + AWS_PROFILE: acceptance + AWS_SDK_LOAD_CONFIG: '1' + AWS_EC2_METADATA_DISABLED: 'true' + RELEASE_PROOF: ${{ needs.inspect.outputs.proof }} + steps: + - name: Load trusted control code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inspect.outputs.base }} + path: control + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.22.12' + cache: false + - uses: coreweave/actions-public/cwic/setup-cwic@6b225e30df11c646ddee8641565935318f0918f0 + with: + version: v1.47.1 + - name: Reverify after environment approval + run: python3 control/scripts/run_storage_acceptance.py prepare + env: + GH_TOKEN: ${{ github.token }} + - name: Check out only the verified candidate merge + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inspect.outputs.merge }} + path: candidate + persist-credentials: false + - name: Run bounded live suite + run: python3 control/scripts/run_storage_acceptance.py test + env: + GH_TOKEN: ${{ github.token }} + - name: Recover the exact run bucket and remove private files + if: always() + run: python3 control/scripts/run_storage_acceptance.py cleanup + + record: + needs: [inspect, acceptance] + if: always() && needs.inspect.outputs.live == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + checks: write + env: + GH_TOKEN: ${{ github.token }} + RELEASE_PROOF: ${{ needs.inspect.outputs.proof }} + RELEASE_CHECK: ${{ needs.inspect.outputs.check }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inspect.outputs.base }} + persist-credentials: false + - name: Prepare public source evidence + id: evidence + run: python3 scripts/release_gate.py evidence + env: + ACCEPTANCE_RESULT: ${{ needs.acceptance.result }} + - name: Store source hashes only + id: upload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: storage-acceptance-evidence-${{ github.run_attempt }} + path: acceptance-evidence.json + if-no-files-found: error + retention-days: 90 + - name: Complete the required gate after evidence is stored + if: always() + run: python3 scripts/release_gate.py finish + env: + ACCEPTANCE_RESULT: ${{ needs.acceptance.result == 'success' && steps.evidence.outcome == 'success' && steps.upload.outcome == 'success' && 'success' || 'failure' }} diff --git a/.gitignore b/.gitignore index e25e2ff72..60311ce95 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ .env __pycache__/ +dist/ diff --git a/.goreleaser.yml b/.goreleaser.yml index ce6f31f9c..f0f51fed2 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,12 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/goreleaser/goreleaser/v1.18.2/www/docs/static/schema.json -before: - hooks: - - make build +project_name: s5cmd builds: - binary: s5cmd - ldflags: -s -w -X github.com/coreweave/s5cmd/v2/version.Version={{.Tag}} -X github.com/coreweave/s5cmd/v2/version.GitCommit={{ .ShortCommit }} + ldflags: -s -w -X github.com/peak/s5cmd/v2/version.Version={{ .Env.RELEASE_VERSION }} -X github.com/peak/s5cmd/v2/version.GitCommit={{ .ShortCommit }} env: - CGO_ENABLED=0 goos: @@ -39,7 +37,7 @@ checksum: name_template: '{{ .ProjectName }}_checksums.txt' snapshot: - name_template: "{{ .Tag }}-{{ .ShortCommit }}" + name_template: "{{ .Env.RELEASE_VERSION }}" changelog: sort: asc @@ -52,10 +50,10 @@ changelog: nfpms: - - vendor: Peak - homepage: https://github.com/peak/s5cmd + vendor: CoreWeave + homepage: https://github.com/coreweave/s5cmd license: MIT - maintainer: Peak + maintainer: CoreWeave description: |- Very fast S3 and local filesystem execution tool. It comes with support for a multitude of operations including tab completion and wildcard @@ -65,4 +63,4 @@ nfpms: - deb release: - draft: false + draft: true diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 000000000..9965a3410 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "2.3.0" +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8b957e2e..aa7cce71b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,8 @@ While adding your changes and testing your changes, it would be good to remember For this fork, target the `coreweave` branch. CI runs build, test, and QA checks for pull requests and merges into that branch. Merging a pull request does not automatically tag or publish a release. +Release PRs use Conventional Commits and require protected storage acceptance before publication. See [release setup and operation](doc/releases.md). + Once you add your changes and all the tests/checks pass, you can submit your pull request to the `peak/s5cmd` repository. Github will trigger automated tests in Github Actions. All tests and checks will be run on different operating systems including `linux`, `macos` and `windows`. An `s5cmd` maintainer will review your pull request. They may suggest updates for clarity and style, or request additional unit or integration tests. diff --git a/Dockerfile b/Dockerfile index 0320c5cbb..17ef2b3b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,10 @@ FROM golang:1.22-alpine AS build +ARG VERSION +ARG BUILD COPY . /s5cmd/ RUN apk add --no-cache git make && \ cd /s5cmd/ && \ - CGO_ENABLED=0 make build + CGO_ENABLED=0 make build VERSION="${VERSION:-$(cat version.txt)}" BUILD="${BUILD:-dev}" FROM alpine:3.20 COPY --from=build /s5cmd/s5cmd . diff --git a/Makefile b/Makefile index 7e550c168..85fdb9ccb 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ default: all .PHONY: all all: clean build check test -VERSION := `git describe --abbrev=0 --tags || echo "0.0.0"` -BUILD := `git rev-parse --short HEAD` +VERSION ?= $(shell cat version.txt) +BUILD ?= $(shell git rev-parse --short HEAD) LDFLAGS=-ldflags "-X=github.com/peak/s5cmd/v2/version.Version=$(VERSION) -X=github.com/peak/s5cmd/v2/version.GitCommit=$(BUILD)" TEST_TYPE:=test_with_race diff --git a/doc/releases.md b/doc/releases.md new file mode 100644 index 000000000..528f40d8c --- /dev/null +++ b/doc/releases.md @@ -0,0 +1,73 @@ +# Releases + +Implementation and release PRs target `coreweave`. Conventional Commits update a release-please PR containing only `CHANGELOG.md`, `version.txt`, and `.release-please-manifest.json`. The initial manifest records the upstream baseline, `2.3.0`; the bootstrap commit bounds the initial changelog to subsequent fork changes. Versions advance from those commits. Fork tags use `coreweave-v..`, so upstream tags and historical dirty tags cannot collide with automated releases. + +Ordinary merges update the release PR and do not publish. Only the merge of a verified, approved release PR can publish. The release workflow verifies successful ordinary CI, human approval of the current head, and live acceptance evidence for the identical source tree and immediate parent of the actual merge. Stale head, stale base, failed cleanup, skipped tests, missing evidence, or a mismatched tree blocks publication. Squash merges are supported; publication uses the recorded merge commit rather than whichever branch tip happens to be newest. + +## Administrator setup + +Keep `RELEASE_AUTOMATION_ENABLED` unset until these steps are complete. Code can be reviewed and merged with automation disabled; it cannot obtain live credentials or publish a release in that state. + +1. Set the repository default branch to `coreweave`. GitHub's `pull_request_target` uses the default branch; the verifier also requires the repository default branch to match the release target. Manual runs select `coreweave`. +2. Protect `coreweave` with required human PR approval, dismissal of stale approvals, the existing 22 CI checks, and the stable `storage-acceptance` check from GitHub Actions. Require Code Owner approval with a `CODEOWNERS` rule covering `*` that names human maintainers or a human team with write access; ensure automation cannot satisfy that ownership requirement. This protects ordinary implementation PRs as well as release PRs and is a precondition for enabling automation. Select the actual owners through repository administration. Require branches to be up to date before merging. Disable force pushes and protection bypasses. Use squash merges for release PRs and retain their Conventional Commit titles. Ensure workflow/control changes receive review. Block creation, update, and deletion of legacy `v*` tags; historical commits still contain the superseded tag workflows. Keep fork release tags immutable as well. +3. Enable the repository setting "Allow GitHub Actions to create and approve pull requests". This setting also permits other workflows with Pull requests write permission to approve PRs; generic required approvals alone do not guarantee human review. Configure the human Code Owner requirement above first. Release-please uses the built-in, short-lived `GITHUB_TOKEN` with Contents, Pull requests, and Issues write permissions. A separate job receives Actions write permission solely to dispatch validation. No App installation, App private key, personal token, or release-management environment is required. The release workflow never approves PRs or merges them. +4. Create the `storage-acceptance` environment with required human reviewers, prevent self-review, disable administrator bypass, and permit deployments only from the `coreweave` branch. Use the same controls for the `release` environment. PR merge refs, tags, and other branches must not be permitted to deploy. GitHub evaluates the trusted gate on the default branch; ordinary PR workflow code cannot use this branch-only environment. +5. Store `STORAGE_API_URL`, `STORAGE_ENDPOINT_URL`, `STORAGE_ORG`, `STORAGE_REGION`, and `STORAGE_OIDC_SUBJECT` as secrets on `storage-acceptance`, even when some are configuration rather than credentials. Obtain actual values privately. Do not store a CoreWeave API token, access key, secret key, or session token in either environment. +6. Bootstrap storage WIF as described below, then set `RELEASE_AUTOMATION_ENABLED=true`. Approve and observe the first protected release-PR gate before approving its merge. Verify that explicit dispatch ran all ordinary CI checks and the storage gate against the recorded candidate, that the created check's `external_id` round-trips as `:`, that storage exchange/renewal used OIDC, and that resources and private files were removed. GitHub may render a different check details URL; that display link is not used for authorization. Remove obsolete static storage-test secrets through the secret-administration path. + +The branch and environment controls are part of the gate, not optional documentation settings. If configuration is incomplete, leave automation disabled. Repository settings, secrets, environment protection, and storage federation configuration require an administrator; the release workflow cannot configure them. + +## Storage WIF bootstrap + +Use local administrator credentials and a private cache outside the checkout. The QA token used for local validation must never become a CI secret. Inspect existing provider configurations and policies before adding resources; preserve unrelated policies. + +Register an OIDC provider with issuer `https://token.actions.githubusercontent.com` and audience `https://coreweave.com/iam`, using released CWIC's `iam workload-federation oidc` commands or approved administration tooling. Read the repository's GitHub OIDC customization settings (`GET /repos/coreweave/s5cmd/actions/oidc/customization/sub`). Check whether the subject uses repository names, immutable IDs, or custom claims. Verify the actual `sub` issued for the protected `storage-acceptance` job privately. Set `STORAGE_OIDC_SUBJECT` to that exact value; the helper checks issuer, audience, and subject before exchange and suppresses mismatches without printing the token or claims. Do not grant wildcard repository or environment subjects. + +Create a separate organization policy for that exact principal. Use `scripts/prepare_storage_wif.py --subject '' --output /absolute/private/path/policy.json` to prepare a mode-0600 proposal. Review it before applying it with the organization's approved local administration path. The proposal separates: + +- `cwobject:CreateAccessKeyOIDC` on global resources `["*"]`. +- The suite's S3 actions on `s5cmd-acceptance-*` and `s5cmd-acceptance-*/*` only, including multipart, version deletion, and cleanup. + +The principal format is `role/https://token.actions.githubusercontent.com:`. Storage federation is independent of the token used to manage PRs. The renewable login action is not yet available in the pinned public action revision, so the workflow uses the reviewed local OIDC process helper with pinned CWIC `v1.47.1` and pinned public CWIC setup action. Each renewal requests a fresh GitHub token. Replace the helper only with a reviewed, merged renewable action, preserving refresh and sanitization regressions. + +## Gate operation + +The trusted `Release` workflow runs release-please, verifies the resulting PR, and uploads an attempt-specific record containing only its source hashes and release metadata. After that producer job succeeds, it explicitly dispatches `ci` on the recorded release branch and `Storage gate` on `coreweave`. Both verify the immutable record against the trusted producer workflow run and its successful steps. CI checks its actual checkout SHA against the expected head before allowing builds or tests. This handles token-generated events without requiring an App or relying on approval of automatically generated PR workflow runs. + +The trusted `Storage gate` workflow reads server-side metadata. An ordinary PR receives a successful `storage-acceptance` check without entering the environment or receiving OIDC permission. A release claim must match the built-in Actions bot identity, signed bot-authored commits, same repository, release target/branch, release metadata, increasing version, and the three permitted release files. Live execution additionally requires producer evidence bound to the current head and base. Spoofed titles, labels, branches, forks, human edits, and arbitrary source changes cannot authorize live execution. Automatic release-PR events do not replace the explicitly dispatched live check. + +The gate records head, base, candidate merge, and tree hashes. It rechecks metadata after environment approval and before execution. Only the acceptance job has `id-token: write`. It checks out the verified candidate merge, runs the complete bounded live suite with renewable profile credentials and verified TLS, and attempts ledger-scoped recovery in an `always()` step. Private profiles and CWIC caches are removed even on failure. Raw build, storage, and credential-provider output is suppressed; public logs contain fixed outcomes and source hashes only. Do not enable Actions debug logging for these jobs. + +Every release-please update dispatches new validation. Base-branch pushes invalidate the open release gate; release-please refreshes its PR and dispatches validation again, while strict branch protection prevents merging an outdated release branch. Superseded push handlers do not overwrite a newer base's check, and automatic PR-event runs use a different queue from explicit live dispatches. The completion job checks that head, base, and candidate content still match. Publication independently checks the producer record, the successful gate's trusted workflow run, all 22 successful CI checks from one explicitly dispatched CI run with a successful candidate preflight, and the actual merged tree/parent. The verifier also requires the head tree tested by CI to equal the candidate merge tree tested against storage and eventually published. A race between acceptance and merge cannot publish different content. + +The completion job uploads only a small public source-hash record, separately named for each workflow attempt. Publication verifies that record and the successful live job/steps from the same trusted run. A copied check name or forged check summary cannot substitute for live execution. Evidence expires after 90 days; rerun the gate before merging a release with expired evidence. + +## Reruns and cleanup + +For a test failure, correct protected configuration or permissions privately and rerun all jobs of `Storage gate`. Partial "rerun failed jobs" attempts cannot reuse inspection evidence from an earlier attempt and deliberately fail the gate. If candidate recording or dispatch failed, rerun all jobs of the original `Release` run while its base is current, or run `Release` manually on the current `coreweave` tip to refresh the PR. Partial producer retries cannot reuse an artifact from an earlier attempt. For a revision change, allow the bot to update the release PR and approve the new gate; do not copy an older success. Release-please always rebuilds its branch after target-branch pushes, including changes without release notes. A rejected environment deployment or failed credential exchange remains a failed gate. Missing configuration cannot select fake storage. + +For a CI failure, rerun all jobs of the explicitly dispatched `ci` run, including `release-preflight`. Publication requires the latest successful checks to belong to one such run. Ordinary manual or automatically generated PR runs cannot replace the candidate preflight, even if their check names match. If another CI run supersedes those checks, rerun the explicitly dispatched run before merging. + +See [storage acceptance](storage-acceptance.md) for resource bounds and local validation. Recovery reads only the exact bucket in the run's new private resource ledger, never an account inventory or a prefix-wide sweep. Cancellation is not automatic on a superseding revision, allowing cleanup to finish. Deadlines terminate the process group gracefully and allow an additional cleanup window. Forced runner loss can interrupt cleanup and destroy its local ledger; use an approved private recovery record if that failure mode must be recoverable. Never upload profiles, caches, tokens, or resource ledgers to public Actions artifacts. + +For local recovery with the original private profile/configuration and ledger, run: + +```sh +go test -mod=readonly ./e2e -run '^TestAcceptanceRecovery$' -count=1 -timeout=4m +``` + +Recovery is a separate operation and cannot satisfy the release suite. Failed recovery retains the ledger and fails the job. Investigate service-side diagnostics privately; the public log deliberately omits raw HTTP responses and resource names. + +## Publication and rollback + +After authorization and release-environment approval, publication reserves the version tag at the verified merge commit and creates a draft release. GoReleaser builds archives, checksums, and Debian packages without publishing by itself; the workflow uploads them to that release and builds the existing multi-platform `ghcr.io/coreweave/s5cmd` image destinations. Binary and image builds use `version.txt` and the verified commit. Docker context excludes local credential and environment files. + +The release becomes public only after binary upload and container publication succeed. Rerun the original `Release` run to retry publication of the same immutable merge/version. If its commit is no longer the current target tip, PR management is skipped so an older publication retry cannot rebuild the active release PR. Existing tags must still point to that commit; reruns cannot allocate another version. A completed public release skips rebuild, asset replacement, and image publication. Interrupted drafts are located by release ID and reused. Version and commit image tags are stable. `latest` and GitHub's latest-release pointer move only when the published version still matches the target branch's manifest, so rerunning an older release cannot roll them backward. A manually pushed tag does not invoke publication. + +GitHub concurrency retains one running and one pending release run; a newer push can cancel an older pending run even with `cancel-in-progress: false`, including one waiting behind environment approval. If the cancelled run corresponds to a release merge, rerun all jobs of that original run to preserve its merge SHA. `workflow_dispatch` uses the current branch tip and cannot recover an older release merge. Check pending release runs when approving publication, and resolve an untagged merged release before proceeding with another release PR. + +For rollback, deploy the desired immutable version or image digest. Do not rewrite released tags or replace acceptance evidence. Disable `RELEASE_AUTOMATION_ENABLED` to pause release management and publication, then revert through an ordinary reviewed PR and publish a new tested version. Leave required checks and environment protections in place. + +## Local verification + +Run `python3 -m unittest discover -s scripts -p 'test_*.py'`, focused Go recovery/configuration regressions, `make check`, and `actionlint`. Rehearse acceptance against isolated local storage and then the approved test service using local renewable credentials. GitHub environment enforcement, actual token-authenticated dispatch, and an actual GitHub-issued OIDC exchange require protected workflow validation; synthetic and local tests do not establish those properties. diff --git a/doc/storage-acceptance.md b/doc/storage-acceptance.md index a0a63ef6f..c78c280a0 100644 --- a/doc/storage-acceptance.md +++ b/doc/storage-acceptance.md @@ -45,7 +45,7 @@ GitHub supplies `ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOK Run helper regressions with `python3 -m unittest discover -s scripts -p 'test_*.py'`. Normal PR CI runs these synthetic tests without OIDC permission or live configuration. -Storage federation must trust the actual subject issued for this repository's protected environment, including any subject customization. Grant the credential-exchange permission separately from storage permissions on the acceptance bucket prefix. Configure environment reviewers and release-PR verification before granting the execution job `id-token: write`; this harness PR does not enable privileged live workflows or publication. GitHub App authentication for release-bot event delivery is separate from storage federation. +Storage federation must trust the actual subject issued for this repository's protected environment, including any subject customization. Grant the credential-exchange permission separately from storage permissions on the acceptance bucket prefix. Configure environment reviewers and release-PR verification before granting the execution job `id-token: write`; see [release setup and operation](releases.md). The built-in GitHub token manages release PRs and dispatches validation separately from storage federation. ## Cleanup and diagnosis diff --git a/e2e/acceptance_recovery_helpers_test.go b/e2e/acceptance_recovery_helpers_test.go new file mode 100644 index 000000000..888563659 --- /dev/null +++ b/e2e/acceptance_recovery_helpers_test.go @@ -0,0 +1,59 @@ +package e2e + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +// TestAcceptanceRecovery is a separate cleanup operation. Its completion never +// substitutes for the full suite in the release gate. +func TestAcceptanceRecovery(t *testing.T) { + if os.Getenv("S5CMD_TEST_MODE") != "live" { + t.Skip("live recovery is not selected") + } + ledger := os.Getenv("S5CMD_TEST_RESOURCE_FILE") + if ledger == "" { + t.Fatal("live recovery requires a private resource ledger") + } + client, _ := setup(t) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if err := recoverAcceptanceBucket(ctx, client, ledger); err != nil { + t.Fatal(err) + } + acceptanceCompleted = true +} + +func recoverAcceptanceBucket(ctx context.Context, client s3iface.S3API, ledger string) error { + info, err := os.Lstat(ledger) + if os.IsNotExist(err) { + return nil + } + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() > 128 { + return fmt.Errorf("invalid recovery ledger") + } + file, err := os.Open(ledger) + if err != nil { + return fmt.Errorf("recovery ledger unavailable") + } + contents, readErr := io.ReadAll(io.LimitReader(file, 129)) + closeErr := file.Close() + if readErr != nil || closeErr != nil || len(contents) > 128 { + return fmt.Errorf("invalid recovery ledger") + } + bucket := strings.TrimSuffix(string(contents), "\n") + if err := cleanAcceptanceBucket(ctx, client, bucket); err != nil { + return fmt.Errorf("scoped recovery failed: %w", err) + } + if err := os.Remove(ledger); err != nil { + return fmt.Errorf("recovery ledger removal failed") + } + return nil +} diff --git a/e2e/acceptance_recovery_test.go b/e2e/acceptance_recovery_test.go new file mode 100644 index 000000000..9d1134213 --- /dev/null +++ b/e2e/acceptance_recovery_test.go @@ -0,0 +1,65 @@ +package e2e + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestAcceptanceLedgerRecovery(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("private ledger permissions require a Unix filesystem") + } + const bucket = "s5cmd-acceptance-0123456789abcdef01234567" + for _, stage := range []string{"success", "absent", "failure", "invalid scope", "public file", "symlink"} { + t.Run(stage, func(t *testing.T) { + ledger := filepath.Join(t.TempDir(), "resource") + fixture := &cleanupFixture{bucket: bucket} + if stage != "absent" { + contents := bucket + "\n" + if stage == "invalid scope" { + contents = "unrelated-bucket\n" + } + if err := os.WriteFile(ledger, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + if stage == "failure" { + fixture.fail = "bucket" + } + if stage == "public file" { + if err := os.Chmod(ledger, 0644); err != nil { + t.Fatal(err) + } + } + if stage == "symlink" { + link := ledger + "-link" + if err := os.Symlink(ledger, link); err != nil { + t.Skip("symlink creation unavailable") + } + ledger = link + } + err := recoverAcceptanceBucket(context.Background(), fixture, ledger) + if stage == "success" || stage == "absent" { + if err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(ledger); !os.IsNotExist(err) { + t.Fatal("successful recovery retained its ledger") + } + } else { + if err == nil { + t.Fatal("unsafe or failed recovery succeeded") + } + if _, err := os.Lstat(ledger); err != nil { + t.Fatal("failed recovery lost its ledger") + } + if stage != "failure" && (fixture.aborted > 0 || fixture.deleted > 0 || fixture.removed) { + t.Fatal("unsafe recovery reached storage") + } + } + }) + } +} diff --git a/e2e/main_test.go b/e2e/main_test.go index 0741b2d88..657b549a9 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -24,8 +24,9 @@ func runTests(m *testing.M) int { fmt.Fprintln(os.Stderr, err) return 1 } - if cfg != nil && flag.Lookup("test.run").Value.String() != "^TestAcceptanceStorage$" { - fmt.Fprintln(os.Stderr, "live tests require the complete TestAcceptanceStorage suite") + selector := flag.Lookup("test.run").Value.String() + if cfg != nil && selector != "^TestAcceptanceStorage$" && selector != "^TestAcceptanceRecovery$" { + fmt.Fprintln(os.Stderr, "live tests require the complete acceptance suite or scoped recovery") return 1 } if cfg != nil { diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 000000000..68bc7133f --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "bootstrap-sha": "acb67716e42d14d81cf8446f639e8dcf1a4daab3", + "release-type": "simple", + "include-component-in-tag": true, + "always-update": true, + "packages": { + ".": { + "component": "coreweave", + "changelog-path": "CHANGELOG.md", + "version-file": "version.txt" + } + } +} diff --git a/scripts/oidc_credentials.py b/scripts/oidc_credentials.py index c061387e5..1166eb3ed 100644 --- a/scripts/oidc_credentials.py +++ b/scripts/oidc_credentials.py @@ -2,6 +2,7 @@ """Renewable AWS credential_process for a protected GitHub Actions job.""" import datetime +import base64 import json import os from pathlib import Path @@ -35,6 +36,15 @@ def oidc_token(): token = json.loads(data)['value'] if not isinstance(token, str) or not token.strip(): raise ValueError('empty OIDC token') + expected = os.environ.get('S5CMD_OIDC_SUBJECT') + if expected: + # GitHub supplies this token over authenticated TLS; the exchange service + # verifies its signature. Check the configured identity before sending it. + payload = token.split('.')[1] + claims = json.loads(base64.urlsafe_b64decode(payload + '=' * (-len(payload) % 4))) + if (claims.get('sub') != expected or claims.get('iss') != 'https://token.actions.githubusercontent.com' + or claims.get('aud') != 'https://coreweave.com/iam'): + raise ValueError('unexpected OIDC identity') return token diff --git a/scripts/prepare_storage_wif.py b/scripts/prepare_storage_wif.py new file mode 100644 index 000000000..1fa7a1bd7 --- /dev/null +++ b/scripts/prepare_storage_wif.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Prepare a private WIF policy proposal; never apply trust or permissions.""" + +import argparse +import json +import os +from pathlib import Path + +S3_ACTIONS = ['s3:CreateBucket', 's3:DeleteBucket', 's3:GetBucketLocation', + 's3:ListBucket', 's3:GetObject', 's3:PutObject', 's3:DeleteObject', + 's3:ListBucketVersions', 's3:DeleteObjectVersion', + 's3:ListBucketMultipartUploads', 's3:ListMultipartUploadParts', 's3:AbortMultipartUpload'] + + +def policy(subject): + if not subject or any(character in subject for character in '*?[]\\\r\n'): + raise ValueError('an exact observed subject is required') + principal = 'role/https://token.actions.githubusercontent.com:' + subject + return {'name': 's5cmd-storage-acceptance', 'version': 'v1alpha1', 'statements': [ + {'name': 'temporary-credential-exchange', 'effect': 'Allow', + 'actions': ['cwobject:CreateAccessKeyOIDC'], 'resources': ['*'], 'principals': [principal]}, + {'name': 'disposable-storage', 'effect': 'Allow', 'actions': S3_ACTIONS, + 'resources': ['s5cmd-acceptance-*', 's5cmd-acceptance-*/*'], 'principals': [principal]}]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--subject', required=True) + parser.add_argument('--output', required=True, type=Path) + args = parser.parse_args() + os.umask(0o077) + proposal = policy(args.subject) + if not args.output.is_absolute() or args.output.parent.stat().st_mode & 0o077: + parser.error('output must be in an absolute private directory') + with args.output.open('x', encoding='utf-8') as stream: + json.dump(proposal, stream, indent=2) + stream.write('\n') + + +if __name__ == '__main__': + main() diff --git a/scripts/release_gate.py b/scripts/release_gate.py new file mode 100644 index 000000000..7aed28655 --- /dev/null +++ b/scripts/release_gate.py @@ -0,0 +1,574 @@ +#!/usr/bin/env python3 +"""Authorize release execution from server-side metadata, never PR scripts.""" + +import base64 +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +from oidc_credentials import NoRedirect + +RELEASE_FILES = {'CHANGELOG.md', '.release-please-manifest.json', 'version.txt'} +RELEASE_BRANCHES = {'release-please--branches--coreweave', + 'release-please--branches--coreweave--components--coreweave'} +GATE = 'storage-acceptance' +# GitHub signs API-created commits for the built-in Actions bot. +GITHUB_ACTIONS_BOT_ID = 41898282 +GITHUB_SIGNER_ID = 19864447 +CI_CHECKS = {'credential-helper'} | { + f'{kind} ({system}/go-{version}.x)' + for kind in ('build', 'test') for system in ('ubuntu', 'macos', 'windows') + for version in ('1.20', '1.21', '1.22') +} | {f'qa ({version}.x, ubuntu)' for version in ('1.20', '1.21', '1.22')} + + +def require(condition): + if not condition: + raise ValueError('release authorization failed') + + +def version_parts(value): + require(isinstance(value, str) and re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', value)) + return tuple(int(part) for part in value.split('.')) + + +def sha(value): + require(isinstance(value, str) and re.fullmatch('[0-9a-f]{40}', value)) + return value + + +def claims_release(pr, bot_id): + return (pr['head']['ref'] in RELEASE_BRANCHES or + any(label['name'].startswith('autorelease:') for label in pr['labels']) or + (bot_id != 0 and pr['user']['id'] == bot_id)) + + +def verify_candidate(pr, files, commits, manifest, version_text, previous, + merge, head_commit, repository, bot_id): + require(bot_id > 0 and pr['user']['id'] == bot_id and pr['user']['type'] == 'Bot') + require(pr['base']['ref'] == 'coreweave' and not pr['draft']) + require(pr['base']['repo']['full_name'] == repository == pr['head']['repo']['full_name']) + require(pr['head']['ref'] in RELEASE_BRANCHES) + require(any(label['name'] in ('autorelease: pending', 'autorelease: tagged') for label in pr['labels'])) + require({file['filename'] for file in files} == RELEASE_FILES and + all(file['status'] == 'modified' for file in files)) + require(commits and all(commit['author'] and commit['author']['id'] == bot_id and + commit['committer'] and commit['committer']['id'] == GITHUB_SIGNER_ID and + commit['commit']['verification']['verified'] and + commit['commit']['verification']['reason'] == 'valid' for commit in commits)) + require(set(manifest) == {'.'}) + version = manifest['.'] + require(version_parts(version) > version_parts(previous.strip())) + require(version_text.strip() == version) + body = pr.get('body') or '' + require(body.startswith(':robot: I have created a release *beep* *boop*\n---\n')) + require('This PR was generated with [Release Please](https://github.com/googleapis/release-please).' in body) + require(re.search(r'^#{2,} \[?' + re.escape(version) + r'(?:\]|\s|$)', body, re.MULTILINE)) + head, base = sha(pr['head']['sha']), sha(pr['base']['sha']) + require([parent['sha'] for parent in merge['parents']] == [base, head]) + require(head_commit['sha'] == head and head_commit['tree']['sha'] == merge['tree']['sha']) + return {'pr': pr['number'], 'branch': pr['head']['ref'], 'head': head, 'base': base, 'merge': sha(merge['sha']), + 'tree': sha(merge['tree']['sha']), 'version': version, 'tag': 'coreweave-v' + version} + + +def verify_publication(proof, pr, commit, run, checks, reviews, repository, bot_id, evidence): + require(proof == evidence) + require(pr['merged'] and pr['state'] == 'closed' and pr['user']['id'] == bot_id) + require(pr['base']['ref'] == 'coreweave' and pr['head']['repo']['full_name'] == repository) + require(proof['pr'] == pr['number'] and proof['head'] == pr['head']['sha']) + require(commit['sha'] == pr['merge_commit_sha'] and commit['tree']['sha'] == proof['tree']) + require(commit['parents'] and commit['parents'][0]['sha'] == proof['base']) + require(run['id'] == proof['run_id'] and run['repository']['full_name'] == repository) + require(run['run_attempt'] == proof['run_attempt']) + require(run['path'] == '.github/workflows/storage-gate.yml' and + run['event'] == 'workflow_dispatch' and run['head_branch'] == 'coreweave' and run['head_sha'] == proof['base'] and + run['status'] == 'completed' and run['conclusion'] == 'success') + latest = {check['name']: check for check in checks if check['app']['slug'] == 'github-actions'} + require(all(name in latest and latest[name]['status'] == 'completed' and + latest[name]['conclusion'] == 'success' for name in CI_CHECKS)) + approvals = {} + for review in sorted(reviews, key=lambda item: item['submitted_at'] or ''): + if review['user']['type'] == 'User' and review['state'] != 'COMMENTED': + approvals[review['user']['id']] = review + require(any(review['state'] == 'APPROVED' and review['commit_id'] == proof['head'] and + review.get('author_association') in ('OWNER', 'MEMBER', 'COLLABORATOR') and + review.get('_may_approve') is True + for review in approvals.values())) + version_parts(proof['version']) + require(proof['tag'] == 'coreweave-v' + proof['version']) + return dict(proof, commit=sha(commit['sha'])) + + +class GitHub: + def __init__(self): + self.repository = os.environ['GITHUB_REPOSITORY'] + require(re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', self.repository)) + self.root = 'https://api.github.com/repos/' + self.repository + self.token = os.environ['GH_TOKEN'] + + def request(self, path, method='GET', data=None, missing=False): + require(path == '' or path.startswith('/') and not path.startswith('//')) + request = urllib.request.Request(self.root + path, method=method, + data=json.dumps(data).encode() if data is not None else None, + headers={'Authorization': 'Bearer ' + self.token, + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28'}) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: + body = response.read(4 * 1024 * 1024 + 1) + require(len(body) <= 4 * 1024 * 1024) + return json.loads(body) if body else None + except urllib.error.HTTPError as error: + if missing and error.code == 404: + return None + raise ValueError('GitHub request failed') from None + + def pages(self, path, key=None): + items = [] + for page in range(1, 11): + result = self.request(path + ('&' if '?' in path else '?') + f'per_page=100&page={page}') + values = result[key] if key else result + items.extend(values) + if len(values) < 100: + return items + raise ValueError('metadata limit exceeded') + + def content(self, path, ref): + data = self.request('/contents/' + path + '?ref=' + sha(ref)) + require(data['type'] == 'file' and data['size'] < 1024 * 1024) + return base64.b64decode(data['content'], validate=False).decode() + + +def outputs(values): + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as stream: + for key, value in values.items(): + text = json.dumps(value, separators=(',', ':')) if isinstance(value, dict) else str(value) + require('\n' not in text and '\r' not in text) + stream.write(f'{key}={text}\n') + + +def candidate(api, number, bot_id): + require(api.request('')['default_branch'] == 'coreweave') + pr = api.request(f'/pulls/{number}') + require(pr['state'] == 'open') + base = api.request('/git/ref/heads/coreweave')['object']['sha'] + require(base == pr['base']['sha']) + merge_ref = sha(pr['merge_commit_sha']) + proof = verify_candidate(pr, api.pages(f'/pulls/{number}/files'), + api.pages(f'/pulls/{number}/commits'), + json.loads(api.content('.release-please-manifest.json', pr['head']['sha'])), + api.content('version.txt', pr['head']['sha']), + api.content('version.txt', base), + api.request('/git/commits/' + sha(merge_ref)), + api.request('/git/commits/' + sha(pr['head']['sha'])), api.repository, bot_id) + return proof + + +def prepared_candidate(api, number, bot_id): + # GitHub computes a new PR's merge commit asynchronously. + for attempt in range(10): + pr = api.request(f'/pulls/{number}') + if pr['merge_commit_sha'] is not None: + return candidate(api, number, bot_id) + require(pr['state'] == 'open' and pr.get('mergeable') is None) + if attempt < 9: + time.sleep(2) + raise ValueError('candidate merge is unavailable') + + +def capture_candidate(api, bot_id): + prs = json.loads(os.environ.get('RELEASE_PLEASE_PRS') or '[]') + require(isinstance(prs, list) and len(prs) <= 1) + if not prs: + outputs({'candidate': 'false'}) + return + proof = prepared_candidate(api, int(prs[0]['number']), bot_id) + require(proof['base'] == os.environ['GITHUB_SHA']) + proof.update(producer_run_id=int(os.environ['GITHUB_RUN_ID']), + producer_attempt=int(os.environ['GITHUB_RUN_ATTEMPT'])) + Path('release-candidate.json').write_text(json.dumps(proof, separators=(',', ':')) + '\n') + outputs({'candidate': 'true'}) + + +def release_tip(api): + require(os.environ['GITHUB_REF'] == 'refs/heads/coreweave') + current = api.request('/git/ref/heads/coreweave')['object']['sha'] == os.environ['GITHUB_SHA'] + outputs({'current': 'true' if current else 'false'}) + if not current: + print('Release PR refresh skipped because newer commits are present.') + + +def verify_producer(proof, run, jobs, repository): + require(run['id'] == proof['producer_run_id'] and run['run_attempt'] == proof['producer_attempt']) + require(run['repository']['full_name'] == repository and run['head_sha'] == proof['base'] and + run['head_branch'] == 'coreweave' and run['path'] == '.github/workflows/release.yml' and + run['event'] in ('push', 'workflow_dispatch')) + producers = [job for job in jobs if job['name'] == 'release-please'] + require(len(producers) == 1 and producers[0]['conclusion'] == 'success') + steps = {step['name'] for step in producers[0]['steps'] if step['conclusion'] == 'success'} + require({'Update the release PR', 'Record the release candidate', 'Store candidate source hashes'} <= steps) + + +def source_artifact(api, run_id, artifact_name): + artifacts = api.pages(f'/actions/runs/{int(run_id)}/artifacts', 'artifacts') + records = [artifact for artifact in artifacts if artifact['name'] == artifact_name] + require(len(records) == 1 and not records[0]['expired']) + with tempfile.TemporaryDirectory(prefix='s5cmd-evidence-') as directory: + subprocess.run(['gh', 'run', 'download', str(int(run_id)), '--repo', api.repository, + '--name', artifact_name, '--dir', directory], + check=True, timeout=60, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + filename = 'release-candidate.json' if artifact_name.startswith('release-candidate-') else 'acceptance-evidence.json' + path = Path(directory) / filename + require(path.is_file() and not path.is_symlink() and path.stat().st_size < 4096) + return json.loads(path.read_text()) + + +def producer_evidence(api, run_id, attempt): + run_id, attempt = int(run_id), int(attempt) + require(run_id > 0 and attempt > 0) + proof = source_artifact(api, run_id, f'release-candidate-{attempt}') + require(proof['producer_run_id'] == run_id and proof['producer_attempt'] == attempt) + run = api.request(f'/actions/runs/{run_id}/attempts/{attempt}') + jobs = api.pages(f'/actions/runs/{run_id}/attempts/{attempt}/jobs', 'jobs') + verify_producer(proof, run, jobs, api.repository) + return proof + + +def fresh_producer(api, bot_id): + proof = producer_evidence(api, os.environ['PRODUCER_RUN_ID'], os.environ['PRODUCER_ATTEMPT']) + current = candidate(api, proof['pr'], bot_id) + require(all(current[key] == proof[key] for key in current)) + return proof + + +def dispatch_validation(api, bot_id): + proof = fresh_producer(api, bot_id) + require(proof['base'] == os.environ['GITHUB_SHA']) + inputs = {'producer_run_id': str(proof['producer_run_id']), + 'producer_attempt': str(proof['producer_attempt'])} + api.request('/actions/workflows/ci.yml/dispatches', 'POST', { + 'ref': proof['branch'], + 'inputs': dict(inputs, expected_head=proof['head'])}) + api.request('/actions/workflows/storage-gate.yml/dispatches', 'POST', { + 'ref': 'coreweave', 'inputs': dict(inputs, pr=str(proof['pr']))}) + + +def ci_preflight(api, bot_id): + require(sha(os.environ['EXPECTED_HEAD']) == os.environ['GITHUB_SHA']) + proof = fresh_producer(api, bot_id) + require(proof['head'] == os.environ['GITHUB_SHA']) + + +def ci_evidence(api, proof, checks): + latest = {check['name']: check for check in checks if check['app']['slug'] == 'github-actions'} + require(all(name in latest and latest[name]['status'] == 'completed' and + latest[name]['conclusion'] == 'success' for name in CI_CHECKS)) + suites = {latest[name]['check_suite']['id'] for name in CI_CHECKS} + require(len(suites) == 1) + runs = api.pages(f'/actions/runs?check_suite_id={suites.pop()}', 'workflow_runs') + require(len(runs) == 1) + run = runs[0] + require(run['path'] == '.github/workflows/ci.yml' and run['event'] == 'workflow_dispatch' and + run['head_sha'] == proof['head'] and run['repository']['full_name'] == api.repository and + run['status'] == 'completed' and run['conclusion'] == 'success') + jobs = api.pages(f'/actions/runs/{int(run["id"])}/attempts/{int(run["run_attempt"])}/jobs', 'jobs') + preflight = [job for job in jobs if job['name'] == 'release-preflight'] + require(len(preflight) == 1 and preflight[0]['conclusion'] == 'success' and + any(step['name'] == 'Verify release candidate' and step['conclusion'] == 'success' + for step in preflight[0]['steps'])) + + +def complete_check(api, identifier, conclusion, summary): + api.request(f'/check-runs/{identifier}', 'PATCH', { + 'status': 'completed', 'conclusion': conclusion, + 'output': {'title': GATE, 'summary': summary}}) + + +def gate_uses_current_base(api, head): + checks = api.pages(f'/commits/{head}/check-runs', 'check_runs') + gates = [check for check in checks if check['name'] == GATE and check['app']['slug'] == 'github-actions'] + if not gates: + return False + check = max(gates, key=lambda item: item['id']) + identifier = check.get('external_id') or '' + if not re.fullmatch(r'[1-9][0-9]*:[1-9][0-9]*', identifier): + return False + run_id, attempt = identifier.split(':') + run = api.request(f'/actions/runs/{run_id}/attempts/{attempt}') + return (run['path'] == '.github/workflows/storage-gate.yml' and run['event'] == 'workflow_dispatch' and + run['head_sha'] == os.environ['GITHUB_SHA'] and run['head_branch'] == 'coreweave') + + +def inspect(api, bot_id): + event = json.loads(Path(os.environ['GITHUB_EVENT_PATH']).read_text()) + dispatch = os.environ.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' + if dispatch: + require(os.environ['GITHUB_REF'] == 'refs/heads/coreweave') + number = int(event['inputs']['pr']) + elif 'pull_request' in event: + number = event['pull_request']['number'] + else: + require(os.environ['GITHUB_REF'] == 'refs/heads/coreweave') + if api.request('/git/ref/heads/coreweave')['object']['sha'] != os.environ['GITHUB_SHA']: + outputs({'live': 'false'}) + return + prs = [pr for pr in api.pages('/pulls?state=open&base=coreweave') + if pr['user']['id'] == bot_id and + (pr['head']['repo'] or {}).get('full_name') == api.repository and claims_release(pr, bot_id)] + require(len(prs) <= 1) + if not prs: + outputs({'live': 'false'}) + return + number = prs[0]['number'] + pr = api.request(f'/pulls/{number}') + head = sha(pr['head']['sha']) + # Bot updates use an explicit dispatch with immutable producer evidence. + # Automatic PR events must not replace its pending or completed live gate. + if not dispatch and 'pull_request' in event and pr['user']['id'] == bot_id and claims_release(pr, bot_id): + outputs({'live': 'false'}) + return + # A slow base-push handler must not invalidate the newer explicit dispatch. + if not dispatch and 'pull_request' not in event and gate_uses_current_base(api, head): + outputs({'live': 'false'}) + return + # A check belongs to a SHA, so a fork of a release head must not overwrite + # that release's live gate with an ordinary PR success (or failure). + protected = [other for other in api.pages(f'/commits/{head}/pulls') + if other['number'] != number and other['head']['sha'] == head and + other['user']['id'] == bot_id and + (other['head']['repo'] or {}).get('full_name') == api.repository and claims_release(other, bot_id)] + if protected: + outputs({'live': 'false'}) + return + check = api.request('/check-runs', 'POST', { + 'name': GATE, 'head_sha': head, 'status': 'in_progress', + 'external_id': f'{int(os.environ["GITHUB_RUN_ID"])}:{int(os.environ["GITHUB_RUN_ATTEMPT"])}', + 'details_url': f'https://github.com/{api.repository}/actions/runs/{int(os.environ["GITHUB_RUN_ID"])}'}) + try: + require(check['external_id'] == f'{int(os.environ["GITHUB_RUN_ID"])}:{int(os.environ["GITHUB_RUN_ATTEMPT"])}') + if not claims_release(pr, bot_id): + complete_check(api, check['id'], 'success', 'Ordinary PR: live storage execution is not requested.') + outputs({'live': 'false'}) + return + require(os.environ.get('RELEASE_AUTOMATION_ENABLED') == 'true') + if not dispatch: + complete_check(api, check['id'], 'failure', 'The base changed; waiting for release candidate validation.') + outputs({'live': 'false'}) + return + proof = producer_evidence(api, event['inputs']['producer_run_id'], event['inputs']['producer_attempt']) + require(proof['pr'] == number) + current = candidate(api, number, bot_id) + require(all(current[key] == proof[key] for key in current)) + require(proof['base'] == os.environ['GITHUB_SHA']) + proof['run_id'] = int(os.environ['GITHUB_RUN_ID']) + proof['run_attempt'] = int(os.environ['GITHUB_RUN_ATTEMPT']) + outputs({'live': 'true', 'check': check['id'], 'proof': proof, + 'base': proof['base'], 'merge': proof['merge'], 'pr': number}) + except Exception: + complete_check(api, check['id'], 'failure', 'Release verification failed; no storage credentials were requested.') + raise + + +def finish(api, bot_id): + proof = json.loads(os.environ['RELEASE_PROOF']) + identifier = int(os.environ['RELEASE_CHECK']) + try: + verify_attempt(proof) + require(os.environ['ACCEPTANCE_RESULT'] == 'success') + current = candidate(api, proof['pr'], bot_id) + require(all(current[key] == proof[key] for key in current)) + complete_check(api, identifier, 'success', json.dumps(proof, separators=(',', ':'))) + except Exception: + complete_check(api, identifier, 'failure', 'Live acceptance, cleanup, or revision freshness failed.') + raise + + +def evidence(api, bot_id): + proof = json.loads(os.environ['RELEASE_PROOF']) + verify_attempt(proof) + require(os.environ['ACCEPTANCE_RESULT'] == 'success') + current = candidate(api, proof['pr'], bot_id) + require(all(current[key] == proof[key] for key in current)) + Path('acceptance-evidence.json').write_text(json.dumps(proof, separators=(',', ':')) + '\n') + + +def verify_attempt(proof): + require(proof['run_id'] == int(os.environ['GITHUB_RUN_ID']) and + proof['run_attempt'] == int(os.environ['GITHUB_RUN_ATTEMPT'])) + + +def eligible_reviews(api, reviews): + permissions = {} + for review in reviews: + user = review['user'] + if user['type'] != 'User': + review['_may_approve'] = False + continue + if user['id'] not in permissions: + result = api.request('/collaborators/' + urllib.parse.quote(user['login'], safe='') + '/permission', missing=True) + permissions[user['id']] = bool(result and result['permission'] in ('write', 'maintain', 'admin')) + review['_may_approve'] = permissions[user['id']] + return reviews + + +def acceptance_evidence(api, run_id, attempt): + run_id = int(run_id) + attempt = int(attempt) + jobs = api.pages(f'/actions/runs/{run_id}/attempts/{attempt}/jobs', 'jobs') + live_jobs = [job for job in jobs if job['name'] == 'acceptance'] + require(len(live_jobs) == 1 and live_jobs[0]['conclusion'] == 'success') + required_steps = {'Reverify after environment approval', 'Check out only the verified candidate merge', + 'Run bounded live suite', 'Recover the exact run bucket and remove private files'} + successful_steps = {step['name'] for step in live_jobs[0]['steps'] if step['conclusion'] == 'success'} + require(required_steps <= successful_steps) + return source_artifact(api, run_id, f'storage-acceptance-evidence-{attempt}') + + +def publication_plan(api, bot_id): + require(os.environ['GITHUB_REF'] == 'refs/heads/coreweave') + commit_sha = sha(os.environ['GITHUB_SHA']) + prs = [pr for pr in api.pages(f'/commits/{commit_sha}/pulls') + if pr.get('merged_at') and pr['base']['ref'] == 'coreweave' and + pr['merge_commit_sha'] == commit_sha] + require(len(prs) <= 1) + if not prs or not claims_release(prs[0], bot_id): + outputs({'publish': 'false'}) + return + require(os.environ.get('RELEASE_AUTOMATION_ENABLED') == 'true') + pr = api.request(f'/pulls/{prs[0]["number"]}') + checks = api.pages('/commits/' + sha(pr['head']['sha']) + '/check-runs', 'check_runs') + checks.sort(key=lambda check: check['id']) + gates = [check for check in checks if check['name'] == GATE and check['app']['slug'] == 'github-actions'] + require(gates and gates[-1]['status'] == 'completed' and gates[-1]['conclusion'] == 'success') + proof = json.loads(gates[-1]['output']['summary']) + require(gates[-1]['external_id'] == f'{int(proof["run_id"])}:{int(proof["run_attempt"])}') + producer = producer_evidence(api, proof['producer_run_id'], proof['producer_attempt']) + require(all(producer[key] == proof[key] for key in producer)) + ci_evidence(api, proof, checks) + commit = api.request('/git/commits/' + commit_sha) + plan = verify_publication(proof, pr, commit, api.request(f'/actions/runs/{int(proof["run_id"])}/attempts/{int(proof["run_attempt"])}'), + checks, eligible_reviews(api, api.pages(f'/pulls/{pr["number"]}/reviews')), api.repository, bot_id, + acceptance_evidence(api, proof['run_id'], proof['run_attempt'])) + # Revalidate release metadata at the immutable head before creating a tag. + frozen = dict(pr, base=dict(pr['base'], sha=proof['base'])) + expected_merge = {'sha': proof['merge'], 'tree': {'sha': proof['tree']}, + 'parents': [{'sha': proof['base']}, {'sha': proof['head']}]} + verified = verify_candidate(frozen, api.pages(f'/pulls/{pr["number"]}/files'), + api.pages(f'/pulls/{pr["number"]}/commits'), + json.loads(api.content('.release-please-manifest.json', proof['head'])), + api.content('version.txt', proof['head']), + api.content('version.txt', proof['base']), expected_merge, + api.request('/git/commits/' + sha(proof['head'])), api.repository, bot_id) + require(all(verified[key] == proof[key] for key in verified)) + outputs({'publish': 'true', 'plan': plan, 'tag': plan['tag'], 'version': plan['version'], 'commit': commit_sha}) + + +def find_release(api, tag): + matches = [release for release in api.pages('/releases') if release['tag_name'] == tag] + require(len(matches) <= 1) + return matches[0] if matches else None + + +def prepare_release(api): + plan = json.loads(os.environ['RELEASE_PLAN']) + tag = 'coreweave-v' + '.'.join(str(part) for part in version_parts(plan['version'])) + require(tag == plan['tag']) + commit = sha(plan['commit']) + require(api.request('/git/commits/' + commit)['tree']['sha'] == plan['tree']) + ref = api.request('/git/ref/tags/' + tag, missing=True) + if ref: + require(ref['object']['type'] == 'commit' and ref['object']['sha'] == commit) + else: + api.request('/git/refs', 'POST', {'ref': 'refs/tags/' + tag, 'sha': commit}) + release = find_release(api, tag) + if not release: + release = api.request('/releases', 'POST', { + 'tag_name': tag, 'target_commitish': commit, 'name': tag, 'draft': True, + 'body': release_notes(api.content('CHANGELOG.md', commit), plan['version'])}) + outputs({'release': release['id'], 'build': 'true' if release['draft'] else 'false'}) + + +def finalize_release(api): + plan = json.loads(os.environ['RELEASE_PLAN']) + tag = 'coreweave-v' + '.'.join(str(part) for part in version_parts(plan['version'])) + require(plan['tag'] == tag) + release = api.request('/releases/' + str(int(os.environ['RELEASE_ID']))) + require(release['tag_name'] == tag) + require(api.request('/git/ref/tags/' + tag)['object']['sha'] == sha(plan['commit'])) + latest = api.content('version.txt', api.request('/git/ref/heads/coreweave')['object']['sha']).strip() == plan['version'] + api.request(f'/releases/{release["id"]}', 'PATCH', {'draft': False, 'make_latest': 'true' if latest else 'false'}) + number = int(plan['pr']) + api.request(f'/issues/{number}/labels', 'POST', {'labels': ['autorelease: tagged']}) + api.request(f'/issues/{number}/labels/' + urllib.parse.quote('autorelease: pending', safe=''), + 'DELETE', missing=True) + + +def release_notes(changelog, version): + version_parts(version) + sections = re.split(r'(?=^## )', changelog, flags=re.MULTILINE) + notes = [section for section in sections if re.match(r'^## \[?' + re.escape(version) + r'(?:\]|\s|$)', section)] + require(len(notes) == 1) + return notes[0].strip() + + +def publication_tags(api): + plan = json.loads(os.environ['RELEASE_PLAN']) + version_parts(plan['version']) + require(plan['tag'] == 'coreweave-v' + plan['version']) + prefix = 'ghcr.io/' + api.repository + ':' + tags = [prefix + plan['tag'], prefix + sha(plan['commit'])] + current = api.request('/git/ref/heads/coreweave')['object']['sha'] + if api.content('version.txt', current).strip() == plan['version']: + tags.append(prefix + 'latest') + outputs({'tags': ','.join(tags)}) + + +def execute(args): + require(len(args) == 1) + api = GitHub() + bot_id = GITHUB_ACTIONS_BOT_ID + if args[0] == 'inspect': + inspect(api, bot_id) + elif args[0] == 'finish': + finish(api, bot_id) + elif args[0] == 'evidence': + evidence(api, bot_id) + elif args[0] == 'publication-plan': + publication_plan(api, bot_id) + elif args[0] == 'prepare-release': + prepare_release(api) + elif args[0] == 'finalize-release': + finalize_release(api) + elif args[0] == 'publication-tags': + publication_tags(api) + elif args[0] == 'capture-candidate': + capture_candidate(api, bot_id) + elif args[0] == 'dispatch-validation': + dispatch_validation(api, bot_id) + elif args[0] == 'ci-preflight': + ci_preflight(api, bot_id) + elif args[0] == 'release-tip': + release_tip(api) + else: + raise ValueError('unsupported operation') + + +def main(args): + try: + execute(args) + return 0 + except Exception: + sys.stderr.write('release authorization failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/scripts/run_storage_acceptance.py b/scripts/run_storage_acceptance.py new file mode 100644 index 000000000..049e0da91 --- /dev/null +++ b/scripts/run_storage_acceptance.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Trusted job control: bounded execution, private profiles, silent diagnostics.""" + +import json +import os +from pathlib import Path +import shlex +import shutil +import signal +import subprocess +import sys + +import release_gate as gate + + +def bounded(command, directory, timeout): + environment = dict(os.environ) + environment.pop('GH_TOKEN', None) + process = subprocess.Popen(command, cwd=directory, env=environment, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + def cancelled(signum, frame): + raise subprocess.TimeoutExpired('cancelled storage operation', timeout) + handlers = {number: signal.signal(number, cancelled) for number in (signal.SIGINT, signal.SIGTERM)} + try: + return process.wait(timeout=timeout) == 0 + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=240) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + return False + finally: + for number, handler in handlers.items(): + signal.signal(number, handler) + + +def private_root(): + return Path(os.environ['RUNNER_TEMP']) / 's5cmd-acceptance' + + +def configure(control): + os.umask(0o077) + root = private_root() + root.mkdir(mode=0o700) + (root / 'cache').mkdir(mode=0o700) + helper = shlex.quote(str(control.resolve() / 'scripts' / 'oidc_credentials.py')) + (root / 'config').write_text('[profile acceptance]\ncredential_process = python3 ' + helper + '\n') + (root / 'credentials').write_text('') + return root + + +def fresh_candidate(proof): + current = gate.candidate(gate.GitHub(), proof['pr'], gate.GITHUB_ACTIONS_BOT_ID) + gate.require(all(current[key] == proof[key] for key in current)) + + +def verify_checkout(directory, proof): + for option, expected in [('HEAD', proof['merge']), ('HEAD^{tree}', proof['tree'])]: + result = subprocess.run(['git', '-C', str(directory), 'rev-parse', option], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True, timeout=10) + gate.require(result.stdout.decode().strip() == expected) + + +def execute(args): + gate.require(len(args) == 1) + control = Path(__file__).resolve().parent.parent + candidate = Path(os.environ['GITHUB_WORKSPACE']) / 'candidate' + proof = json.loads(os.environ['RELEASE_PROOF']) + root = private_root() + if args[0] == 'prepare': + fresh_candidate(proof) + gate.require(all(os.environ.get(key) for key in ('CWIC_API_URL', 'S5CMD_TEST_ORG', + 'S5CMD_TEST_ENDPOINT_URL', 'S5CMD_REGION', 'S5CMD_OIDC_SUBJECT'))) + gate.require(not any(os.environ.get(key) for key in ('AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', + 'AWS_ACCESS_KEY', 'AWS_SECRET_KEY', 'AWS_SESSION_TOKEN', + 'S5CMD_ACCESS_KEY_ID', 'S5CMD_SECRET_ACCESS_KEY'))) + gate.require(os.environ.get('RUNNER_DEBUG') != '1') + configure(control) + with open(os.environ['GITHUB_ENV'], 'a', encoding='utf-8') as stream: + for key, value in {'CWIC_CACHE_DIR': root / 'cache', 'AWS_CONFIG_FILE': root / 'config', + 'AWS_SHARED_CREDENTIALS_FILE': root / 'credentials', + 'S5CMD_TEST_RESOURCE_FILE': root / 'resource'}.items(): + gate.require('\n' not in str(value)) + stream.write(f'{key}={value}\n') + elif args[0] == 'test': + verify_checkout(candidate, proof) + fresh_candidate(proof) + gate.require(bounded(['make', 'acceptance'], candidate, 17 * 60)) + print('Live storage suite and scoped cleanup passed for tree ' + gate.sha(proof['tree'])) + elif args[0] == 'cleanup': + if not root.exists(): + return + succeeded = False + try: + if (root / 'resource').exists(): + verify_checkout(candidate, proof) + gate.require(bounded(['go', 'test', '-mod=readonly', './e2e', '-run', + '^TestAcceptanceRecovery$', '-count=1', '-timeout=4m'], candidate, 4 * 60)) + gate.require(not (root / 'resource').exists()) + succeeded = True + finally: + for name in ('cache', 'config', 'credentials'): + path = root / name + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() + if succeeded: + root.rmdir() + else: + raise ValueError('unsupported operation') + + +def main(args): + try: + execute(args) + return 0 + except Exception: + sys.stderr.write('storage acceptance operation failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test_oidc_credentials.py b/scripts/test_oidc_credentials.py index 212aa2e1e..556ddfc57 100644 --- a/scripts/test_oidc_credentials.py +++ b/scripts/test_oidc_credentials.py @@ -1,4 +1,5 @@ import contextlib +import base64 import datetime import io import json @@ -13,6 +14,27 @@ class OIDCCredentialsTests(unittest.TestCase): + def test_configured_subject_and_issuer_must_match_before_exchange(self): + env = {'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://tokens.actions.githubusercontent.com/id', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'synthetic-request-token', + 'S5CMD_OIDC_SUBJECT': 'synthetic-approved-subject'} + for field in (None, 'sub', 'aud', 'iss'): + claims = {'sub': 'synthetic-approved-subject', 'aud': 'https://coreweave.com/iam', + 'iss': 'https://token.actions.githubusercontent.com'} + if field: + claims[field] = 'synthetic-unapproved-value' + payload = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip('=') + token = 'synthetic-header.' + payload + '.synthetic-signature' + class Opener: + def open(self, request, timeout): + return io.BytesIO(json.dumps({'value': token}).encode()) + with patch.dict(os.environ, env, clear=True), patch.object(helper.urllib.request, 'build_opener', return_value=Opener()): + if field: + with self.assertRaises(ValueError): + helper.oidc_token() + else: + self.assertEqual(helper.oidc_token(), token) + def test_failed_exchange_is_sanitized(self): stdout, stderr = io.StringIO(), io.StringIO() with patch.object(helper, 'credentials', side_effect=RuntimeError('synthetic-private-jwt')), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): diff --git a/scripts/test_prepare_storage_wif.py b/scripts/test_prepare_storage_wif.py new file mode 100644 index 000000000..f32b3867d --- /dev/null +++ b/scripts/test_prepare_storage_wif.py @@ -0,0 +1,23 @@ +import unittest + +from prepare_storage_wif import policy + + +class PolicyTests(unittest.TestCase): + def test_exchange_and_storage_permissions_are_separate(self): + proposal = policy('synthetic-observed-subject') + exchange, storage = proposal['statements'] + self.assertEqual(exchange['resources'], ['*']) + self.assertEqual(exchange['actions'], ['cwobject:CreateAccessKeyOIDC']) + self.assertEqual(storage['resources'], ['s5cmd-acceptance-*', 's5cmd-acceptance-*/*']) + self.assertNotIn('s3:*', storage['actions']) + self.assertEqual(exchange['principals'], storage['principals']) + + def test_wildcard_or_multiline_subject_is_rejected(self): + for subject in ('', 'repo:example/*', 'repo:example/?', 'repo:example/[ab]', 'synthetic\nsubject'): + with self.subTest(subject=subject), self.assertRaises(ValueError): + policy(subject) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_release_dispatch.py b/scripts/test_release_dispatch.py new file mode 100644 index 000000000..1b1515157 --- /dev/null +++ b/scripts/test_release_dispatch.py @@ -0,0 +1,184 @@ +"""Release provenance and explicit dispatch regressions, without credentials.""" + +import copy +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import release_gate as gate + + +class DispatchTests(unittest.TestCase): + def setUp(self): + self.proof = {'pr': 7, 'branch': 'release-please--branches--coreweave', 'head': 'b' * 40, 'base': 'a' * 40, + 'merge': 'c' * 40, 'tree': 'd' * 40, + 'version': '2.4.0', 'tag': 'coreweave-v2.4.0', + 'producer_run_id': 21, 'producer_attempt': 1} + self.run = {'id': 21, 'run_attempt': 1, 'head_sha': 'a' * 40, + 'head_branch': 'coreweave', 'event': 'push', + 'path': '.github/workflows/release.yml', + 'repository': {'full_name': 'example/s5cmd'}} + self.jobs = [{'name': 'release-please', 'conclusion': 'success', 'steps': [ + {'name': name, 'conclusion': 'success'} for name in + ('Update the release PR', 'Record the release candidate', 'Store candidate source hashes')]}] + + def test_producer_requires_exact_trusted_source_and_successful_steps(self): + gate.verify_producer(self.proof, self.run, self.jobs, 'example/s5cmd') + for field, value in [('head_sha', 'f' * 40), ('head_branch', 'feature'), + ('event', 'pull_request'), ('path', '.github/workflows/ci.yml'), + ('run_attempt', 2), ('id', 22)]: + run = dict(self.run, **{field: value}) + with self.subTest(field=field), self.assertRaises(ValueError): + gate.verify_producer(self.proof, run, self.jobs, 'example/s5cmd') + for conclusion in ('failure', 'skipped', None): + jobs = copy.deepcopy(self.jobs) + jobs[0]['steps'][0]['conclusion'] = conclusion + with self.assertRaises(ValueError): + gate.verify_producer(self.proof, self.run, jobs, 'example/s5cmd') + + def test_dispatch_checks_freshness_before_scheduling_both_workflows(self): + api = mock.Mock() + with mock.patch.object(gate, 'producer_evidence', return_value=self.proof), mock.patch.object(gate, 'candidate', return_value={k: v for k, v in self.proof.items() if not k.startswith('producer_')}), mock.patch.dict(os.environ, {'PRODUCER_RUN_ID': '21', 'PRODUCER_ATTEMPT': '1', 'GITHUB_SHA': 'a' * 40}): + gate.dispatch_validation(api, gate.GITHUB_ACTIONS_BOT_ID) + self.assertEqual([call.args[0] for call in api.request.call_args_list], + ['/actions/workflows/ci.yml/dispatches', '/actions/workflows/storage-gate.yml/dispatches']) + self.assertEqual(api.request.call_args_list[0].args[2]['inputs']['expected_head'], 'b' * 40) + self.assertEqual(api.request.call_args_list[1].args[2]['ref'], 'coreweave') + stale = dict(self.proof, head='f' * 40) + api.reset_mock() + with mock.patch.object(gate, 'producer_evidence', return_value=self.proof), mock.patch.object(gate, 'candidate', return_value=stale), mock.patch.dict(os.environ, {'PRODUCER_RUN_ID': '21', 'PRODUCER_ATTEMPT': '1', 'GITHUB_SHA': 'a' * 40}), self.assertRaises(ValueError): + gate.dispatch_validation(api, gate.GITHUB_ACTIONS_BOT_ID) + api.request.assert_not_called() + + def test_ci_ref_race_is_rejected_before_builds(self): + with mock.patch.dict(os.environ, {'EXPECTED_HEAD': 'b' * 40, 'GITHUB_SHA': 'f' * 40}), self.assertRaises(ValueError): + gate.ci_preflight(mock.Mock(), gate.GITHUB_ACTIONS_BOT_ID) + + def test_empty_release_please_output_does_not_schedule_validation(self): + with mock.patch.dict(os.environ, {'RELEASE_PLEASE_PRS': ''}), mock.patch.object(gate, 'outputs') as output: + gate.capture_candidate(mock.Mock(), gate.GITHUB_ACTIONS_BOT_ID) + output.assert_called_once_with({'candidate': 'false'}) + + def test_old_publication_retry_does_not_refresh_the_current_release_pr(self): + api = mock.Mock() + api.request.return_value = {'object': {'sha': 'f' * 40}} + with mock.patch.dict(os.environ, {'GITHUB_REF': 'refs/heads/coreweave', 'GITHUB_SHA': 'a' * 40}), mock.patch.object(gate, 'outputs') as output, mock.patch('sys.stdout'): + gate.release_tip(api) + output.assert_called_once_with({'current': 'false'}) + api.request.assert_called_once_with('/git/ref/heads/coreweave') + + def test_captured_record_contains_only_verified_source_metadata(self): + candidate = {k: v for k, v in self.proof.items() if not k.startswith('producer_')} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'release-candidate.json' + environment = {'RELEASE_PLEASE_PRS': json.dumps([{'number': 7}]), + 'GITHUB_SHA': 'a' * 40, 'GITHUB_RUN_ID': '21', 'GITHUB_RUN_ATTEMPT': '1'} + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'prepared_candidate', return_value=candidate), mock.patch.object(gate, 'outputs'), mock.patch.object(gate, 'Path', return_value=path): + gate.capture_candidate(mock.Mock(), gate.GITHUB_ACTIONS_BOT_ID) + self.assertEqual(json.loads(path.read_text()), self.proof) + + def test_new_merge_ref_is_polled_without_retrying_invalid_candidates(self): + api = mock.Mock() + api.request.side_effect = [{'state': 'open', 'mergeable': None, 'merge_commit_sha': None}, + {'merge_commit_sha': 'c' * 40}] + with mock.patch.object(gate.time, 'sleep') as sleep, mock.patch.object(gate, 'candidate', return_value=self.proof): + self.assertEqual(gate.prepared_candidate(api, 7, gate.GITHUB_ACTIONS_BOT_ID), self.proof) + sleep.assert_called_once_with(2) + api.request.side_effect = None + api.request.return_value = {'state': 'open', 'mergeable': False, 'merge_commit_sha': None} + with self.assertRaises(ValueError): + gate.prepared_candidate(api, 7, gate.GITHUB_ACTIONS_BOT_ID) + + def test_late_base_push_cannot_overwrite_gate_for_current_base(self): + api = mock.Mock() + api.pages.return_value = [{'name': gate.GATE, 'id': 7, 'app': {'slug': 'github-actions'}, + 'external_id': '42:1'}] + api.request.return_value = {'path': '.github/workflows/storage-gate.yml', + 'event': 'workflow_dispatch', 'head_sha': 'a' * 40, 'head_branch': 'coreweave'} + with mock.patch.dict(os.environ, {'GITHUB_SHA': 'a' * 40}): + self.assertTrue(gate.gate_uses_current_base(api, 'b' * 40)) + api.request.return_value['head_sha'] = 'f' * 40 + with mock.patch.dict(os.environ, {'GITHUB_SHA': 'a' * 40}): + self.assertFalse(gate.gate_uses_current_base(api, 'b' * 40)) + + def test_superseded_base_push_cannot_create_a_check_on_new_release_head(self): + api = mock.Mock() + api.request.return_value = {'object': {'sha': 'f' * 40}} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'event.json' + path.write_text('{}') + with mock.patch.dict(os.environ, {'GITHUB_EVENT_PATH': str(path), 'GITHUB_SHA': 'a' * 40, + 'GITHUB_REF': 'refs/heads/coreweave'}), mock.patch.object(gate, 'outputs') as output: + gate.inspect(api, gate.GITHUB_ACTIONS_BOT_ID) + output.assert_called_once_with({'live': 'false'}) + api.pages.assert_not_called() + api.request.assert_called_once_with('/git/ref/heads/coreweave') + + def test_storage_dispatch_needs_matching_producer_before_live_output(self): + candidate = {key: value for key, value in self.proof.items() if not key.startswith('producer_')} + pr = {'number': 7, 'head': {'sha': 'b' * 40, 'ref': self.proof['branch']}, + 'user': {'id': gate.GITHUB_ACTIONS_BOT_ID}, 'labels': []} + api = mock.Mock() + api.pages.return_value = [] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'event.json' + path.write_text(json.dumps({'inputs': {'pr': '7', 'producer_run_id': '21', 'producer_attempt': '1'}})) + environment = {'GITHUB_EVENT_PATH': str(path), 'GITHUB_EVENT_NAME': 'workflow_dispatch', + 'GITHUB_REF': 'refs/heads/coreweave', 'GITHUB_SHA': 'a' * 40, + 'GITHUB_RUN_ID': '42', 'GITHUB_RUN_ATTEMPT': '1', 'RELEASE_AUTOMATION_ENABLED': 'true'} + api.request.side_effect = [pr, {'id': 8, 'external_id': '42:1'}] + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'producer_evidence', return_value=dict(self.proof)), mock.patch.object(gate, 'candidate', return_value=candidate), mock.patch.object(gate, 'outputs') as output: + gate.inspect(api, gate.GITHUB_ACTIONS_BOT_ID) + self.assertEqual(output.call_args.args[0]['live'], 'true') + self.assertEqual(output.call_args.args[0]['proof']['producer_run_id'], 21) + api.request.side_effect = [pr, {'id': 9, 'external_id': '42:1'}, None] + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'producer_evidence', return_value=dict(self.proof, head='f' * 40)), mock.patch.object(gate, 'candidate', return_value=candidate), mock.patch.object(gate, 'outputs') as output, self.assertRaises(ValueError): + gate.inspect(api, gate.GITHUB_ACTIONS_BOT_ID) + output.assert_not_called() + self.assertEqual(api.request.call_args.args[2]['conclusion'], 'failure') + + def test_automatic_release_pr_event_cannot_replace_explicit_gate(self): + api = mock.Mock() + api.request.return_value = {'head': {'sha': 'b' * 40, 'ref': self.proof['branch']}, + 'user': {'id': gate.GITHUB_ACTIONS_BOT_ID}, 'labels': []} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'event.json' + path.write_text(json.dumps({'pull_request': {'number': 7}})) + with mock.patch.dict(os.environ, {'GITHUB_EVENT_PATH': str(path), 'GITHUB_EVENT_NAME': 'pull_request_target'}), mock.patch.object(gate, 'outputs') as output: + gate.inspect(api, gate.GITHUB_ACTIONS_BOT_ID) + api.request.assert_called_once_with('/pulls/7') + output.assert_called_once_with({'live': 'false'}) + + def test_ci_check_names_cannot_substitute_for_a_trusted_complete_run(self): + checks = [{'name': name, 'check_suite': {'id': 50}, 'app': {'slug': 'github-actions'}, + 'status': 'completed', 'conclusion': 'success'} for name in gate.CI_CHECKS] + run = {'id': 60, 'run_attempt': 1, 'path': '.github/workflows/ci.yml', + 'event': 'workflow_dispatch', 'head_sha': 'b' * 40, 'status': 'completed', + 'conclusion': 'success', 'repository': {'full_name': 'example/s5cmd'}} + jobs = [{'name': 'release-preflight', 'conclusion': 'success', 'steps': [ + {'name': 'Verify release candidate', 'conclusion': 'success'}]}] + api = mock.Mock(repository='example/s5cmd') + api.pages.side_effect = [[run], jobs] + gate.ci_evidence(api, self.proof, checks) + checks[0]['check_suite']['id'] = 99 + with self.assertRaises(ValueError): + gate.ci_evidence(api, self.proof, checks) + checks[0]['check_suite']['id'] = 50 + for change in ('head', 'path', 'step'): + changed_run, changed_jobs = copy.deepcopy(run), copy.deepcopy(jobs) + if change == 'head': + changed_run['head_sha'] = 'f' * 40 + elif change == 'path': + changed_run['path'] = '.github/workflows/other.yml' + else: + changed_jobs[0]['steps'][0]['conclusion'] = 'skipped' + api.pages.side_effect = [[changed_run], changed_jobs] + with self.subTest(change=change), self.assertRaises(ValueError): + gate.ci_evidence(api, self.proof, checks) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_release_gate.py b/scripts/test_release_gate.py new file mode 100644 index 000000000..bbd10787a --- /dev/null +++ b/scripts/test_release_gate.py @@ -0,0 +1,331 @@ +"""Synthetic release authorization tests; no GitHub or storage credentials.""" + +import copy +import json +import unittest + +import release_gate as gate + + +class ReleaseGateTests(unittest.TestCase): + def setUp(self): + self.pr = { + 'number': 7, 'state': 'open', 'draft': False, + 'user': {'id': 123, 'type': 'Bot'}, + 'base': {'ref': 'coreweave', 'sha': 'a' * 40, + 'repo': {'full_name': 'example/s5cmd'}}, + 'head': {'ref': 'release-please--branches--coreweave', 'sha': 'b' * 40, + 'repo': {'full_name': 'example/s5cmd'}}, + 'labels': [{'name': 'autorelease: pending'}], + 'body': ':robot: I have created a release *beep* *boop*\n---\n\n## [2.4.0](https://example.com)\n\n---\nThis PR was generated with [Release Please](https://github.com/googleapis/release-please).', + } + self.files = [{'filename': name, 'status': 'modified'} for name in gate.RELEASE_FILES] + self.commits = [{'author': {'id': 123}, 'committer': {'id': 19864447}, + 'commit': {'verification': {'verified': True, 'reason': 'valid'}}}] + self.merge = {'sha': 'c' * 40, 'tree': {'sha': 'd' * 40}, + 'parents': [{'sha': 'a' * 40}, {'sha': 'b' * 40}]} + self.head_commit = {'sha': 'b' * 40, 'tree': {'sha': 'd' * 40}} + + def candidate(self): + return gate.verify_candidate(self.pr, self.files, self.commits, + {'.': '2.4.0'}, '2.4.0\n', '2.3.0', + self.merge, self.head_commit, 'example/s5cmd', 123) + + def test_valid_candidate_records_exact_content(self): + proof = self.candidate() + self.assertEqual(proof['tree'], 'd' * 40) + self.assertEqual(proof['base'], 'a' * 40) + self.assertEqual(proof['tag'], 'coreweave-v2.4.0') + + def test_ordinary_pr_does_not_request_storage(self): + self.pr['head']['ref'] = 'feature' + self.pr['labels'] = [] + self.pr['user']['id'] = 456 + self.assertFalse(gate.claims_release(self.pr, 123)) + + def test_spoofed_release_cannot_authorize_execution(self): + for change in ('author', 'fork', 'unsigned', 'signer', 'code', 'base', 'label', 'draft'): + with self.subTest(change=change): + self.setUp() + if change == 'author': + self.pr['user']['id'] = 456 + elif change == 'fork': + self.pr['head']['repo']['full_name'] = 'attacker/s5cmd' + elif change == 'unsigned': + self.commits[0]['commit']['verification']['verified'] = False + elif change == 'signer': + self.commits[0]['committer']['id'] = 456 + elif change == 'code': + self.files.append({'filename': 'scripts/oidc_credentials.py', 'status': 'modified'}) + elif change == 'base': + self.pr['base']['ref'] = 'master' + elif change == 'label': + self.pr['labels'] = [] + else: + self.pr['draft'] = True + with self.assertRaises(ValueError): + self.candidate() + + def test_version_and_release_metadata_must_agree(self): + self.pr['body'] = self.pr['body'].replace('2.4.0', '9.0.0') + with self.assertRaises(ValueError): + self.candidate() + self.setUp() + for manifest, version, previous in [({'.': '2.4.0'}, '2.5.0', '2.3.0'), + ({'.': '2.4.0'}, '2.4.0', '2.4.0'), + ({'.': '2.4.0', 'other': '1.0.0'}, '2.4.0', '2.3.0')]: + with self.assertRaises(ValueError): + gate.verify_candidate(self.pr, self.files, self.commits, manifest, + version, previous, self.merge, self.head_commit, 'example/s5cmd', 123) + + def test_stale_merge_ref_is_rejected(self): + self.merge['parents'][0]['sha'] = 'e' * 40 + with self.assertRaises(ValueError): + self.candidate() + + def test_missing_bot_configuration_fails_closed_for_release(self): + with self.assertRaises(ValueError): + gate.verify_candidate(self.pr, self.files, self.commits, {'.': '2.4.0'}, + '2.4.0', '2.3.0', self.merge, self.head_commit, 'example/s5cmd', 0) + + def test_ci_head_tree_must_equal_accepted_merge_tree(self): + self.head_commit['tree']['sha'] = 'f' * 40 + with self.assertRaises(ValueError): + self.candidate() + self.setUp() + self.head_commit['sha'] = 'f' * 40 + with self.assertRaises(ValueError): + self.candidate() + + def publication(self, mutate=None): + proof = self.candidate() + proof['run_id'] = 42 + proof['run_attempt'] = 1 + merged = copy.deepcopy(self.pr) + merged.update(state='closed', merged=True, merge_commit_sha='e' * 40) + commit = {'sha': 'e' * 40, 'tree': {'sha': 'd' * 40}, + 'parents': [{'sha': 'a' * 40}]} + run = {'id': 42, 'run_attempt': 1, 'path': '.github/workflows/storage-gate.yml', + 'event': 'workflow_dispatch', 'head_branch': 'coreweave', 'status': 'completed', + 'conclusion': 'success', 'head_sha': 'a' * 40, + 'repository': {'full_name': 'example/s5cmd'}} + checks = [{'name': name, 'conclusion': 'success', 'status': 'completed', + 'app': {'slug': 'github-actions'}} for name in gate.CI_CHECKS] + reviews = [{'user': {'id': 999, 'type': 'User', 'login': 'maintainer'}, 'state': 'APPROVED', + 'author_association': 'COLLABORATOR', '_may_approve': True, + 'commit_id': 'b' * 40, 'submitted_at': '2026-01-01T00:00:00Z'}] + if mutate: + mutate(proof, merged, commit, run, checks, reviews) + return gate.verify_publication(proof, merged, commit, run, checks, reviews, + 'example/s5cmd', 123, copy.deepcopy(proof)) + + def test_only_tested_approved_merge_is_publishable(self): + self.assertEqual(self.publication()['tag'], 'coreweave-v2.4.0') + + def test_stale_tree_base_head_and_failed_skipped_missing_ci_block_publication(self): + changes = [lambda p, pr, c, r, checks, reviews: c['tree'].update(sha='f' * 40), + lambda p, pr, c, r, checks, reviews: c['parents'][0].update(sha='f' * 40), + lambda p, pr, c, r, checks, reviews: pr['head'].update(sha='f' * 40), + lambda p, pr, c, r, checks, reviews: checks[0].update(conclusion='failure'), + lambda p, pr, c, r, checks, reviews: checks[0].update(conclusion='skipped'), + lambda p, pr, c, r, checks, reviews: checks.pop(), + lambda p, pr, c, r, checks, reviews: r.update(event='pull_request'), + lambda p, pr, c, r, checks, reviews: reviews.clear(), + lambda p, pr, c, r, checks, reviews: reviews[0].update(author_association='NONE'), + lambda p, pr, c, r, checks, reviews: reviews[0].update(_may_approve=False), + lambda p, pr, c, r, checks, reviews: reviews[0].update(commit_id='f' * 40)] + for change in changes: + with self.subTest(change=change), self.assertRaises(ValueError): + self.publication(change) + + def test_dismissed_approval_cannot_authorize_release(self): + def dismiss(p, pr, c, r, checks, reviews): + reviews.append({'user': {'id': 999, 'type': 'User'}, 'state': 'DISMISSED', + 'commit_id': 'b' * 40, 'submitted_at': '2026-01-02T00:00:00Z'}) + with self.assertRaises(ValueError): + self.publication(dismiss) + + def test_release_notes_do_not_include_older_releases(self): + changelog = '# Changelog\n## [2.4.0](https://example.com)\nnew behavior\n## 2.3.0\nolder behavior\n' + self.assertIn('new behavior', gate.release_notes(changelog, '2.4.0')) + self.assertNotIn('older behavior', gate.release_notes(changelog, '2.4.0')) + with self.assertRaises(ValueError): + gate.release_notes(changelog, '9.0.0') + + def test_rerun_reserves_same_tag_and_refuses_collision(self): + from unittest import mock + import os + api = mock.Mock() + plan = dict(self.candidate(), commit='e' * 40) + api.request.side_effect = [{'tree': {'sha': plan['tree']}}, + {'object': {'type': 'commit', 'sha': plan['commit']}}] + api.pages.return_value = [{'id': 7, 'tag_name': plan['tag'], 'draft': True}] + with mock.patch.dict(os.environ, {'RELEASE_PLAN': json.dumps(plan)}), mock.patch.object(gate, 'outputs'): + gate.prepare_release(api) + self.assertTrue(all(len(call.args) == 1 for call in api.request.call_args_list)) + api.request.side_effect = [{'tree': {'sha': plan['tree']}}, + {'object': {'type': 'commit', 'sha': 'f' * 40}}] + with mock.patch.dict(os.environ, {'RELEASE_PLAN': json.dumps(plan)}), self.assertRaises(ValueError): + gate.prepare_release(api) + + def test_draft_finalize_uses_id_without_published_tag_lookup(self): + from unittest import mock + import os + plan = dict(self.candidate(), commit='e' * 40) + api = mock.Mock() + api.request.side_effect = [{'id': 7, 'tag_name': plan['tag'], 'draft': True}, + {'object': {'sha': plan['commit']}}, + {'object': {'sha': 'f' * 40}}, None, None, None] + api.content.return_value = plan['version'] + with mock.patch.dict(os.environ, {'RELEASE_PLAN': json.dumps(plan), 'RELEASE_ID': '7'}): + gate.finalize_release(api) + paths = [call.args[0] for call in api.request.call_args_list] + self.assertIn('/releases/7', paths) + self.assertNotIn('/releases/tags/' + plan['tag'], paths) + + def test_public_release_rerun_does_not_rebuild(self): + from unittest import mock + import os + api = mock.Mock() + plan = dict(self.candidate(), commit='e' * 40) + api.request.side_effect = [{'tree': {'sha': plan['tree']}}, + {'object': {'type': 'commit', 'sha': plan['commit']}}] + api.pages.return_value = [{'id': 7, 'tag_name': plan['tag'], 'draft': False}] + with mock.patch.dict(os.environ, {'RELEASE_PLAN': json.dumps(plan)}), mock.patch.object(gate, 'outputs') as output: + gate.prepare_release(api) + self.assertEqual(output.call_args.args[0]['build'], 'false') + + def test_partial_rerun_cannot_reuse_earlier_attempt(self): + from unittest import mock + import os + proof = dict(self.candidate(), run_id=42, run_attempt=1) + api = mock.Mock() + environment = {'GITHUB_RUN_ID': '42', 'GITHUB_RUN_ATTEMPT': '2', 'RELEASE_PROOF': json.dumps(proof), + 'RELEASE_CHECK': '8', 'ACCEPTANCE_RESULT': 'success'} + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'candidate', return_value=self.candidate()): + with self.assertRaises(ValueError): + gate.evidence(api, 123) + with self.assertRaises(ValueError): + gate.finish(api, 123) + self.assertEqual(api.request.call_args.args[2]['conclusion'], 'failure') + + def test_fork_of_release_head_cannot_override_required_check(self): + from unittest import mock + from pathlib import Path + import os + import tempfile + release = copy.deepcopy(self.pr) + self.pr['number'] = 8 + self.pr['user']['id'] = 456 + self.pr['head']['ref'] = 'ordinary' + self.pr['head']['repo']['full_name'] = 'attacker/s5cmd' + self.pr['labels'] = [] + api = mock.Mock(repository='example/s5cmd') + api.request.return_value = self.pr + api.pages.return_value = [release, self.pr] + with tempfile.TemporaryDirectory() as directory: + event = Path(directory) / 'event.json' + event.write_text(json.dumps({'pull_request': {'number': 8}})) + with mock.patch.dict(os.environ, {'GITHUB_EVENT_PATH': str(event), 'GITHUB_RUN_ID': '42'}), mock.patch.object(gate, 'outputs') as output: + gate.inspect(api, 123) + self.assertEqual(len(api.request.call_args_list), 1) + output.assert_called_once_with({'live': 'false'}) + + def test_fork_release_name_cannot_suppress_base_push_invalidation(self): + from unittest import mock + from pathlib import Path + import os + import tempfile + fake = copy.deepcopy(self.pr) + fake['number'] = 8 + fake['user']['id'] = 456 + fake['head']['repo']['full_name'] = 'attacker/s5cmd' + api = mock.Mock(repository='example/s5cmd') + api.pages.side_effect = [[fake, self.pr], [], []] + api.request.side_effect = [{'object': {'sha': 'a' * 40}}, self.pr, {'id': 9, 'external_id': '42:1'}, None] + with tempfile.TemporaryDirectory() as directory: + event = Path(directory) / 'event.json' + event.write_text('{}') + environment = {'GITHUB_EVENT_PATH': str(event), 'GITHUB_REF': 'refs/heads/coreweave', + 'GITHUB_RUN_ID': '42', 'GITHUB_RUN_ATTEMPT': '1', 'GITHUB_SHA': 'a' * 40, + 'RELEASE_AUTOMATION_ENABLED': 'true'} + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'candidate', return_value=self.candidate()), mock.patch.object(gate, 'outputs') as output: + gate.inspect(api, 123) + self.assertEqual(output.call_args.args[0]['live'], 'false') + self.assertEqual(api.request.call_args.args[2]['conclusion'], 'failure') + + def test_review_author_requires_repository_write_access(self): + from unittest import mock + api = mock.Mock() + api.request.side_effect = [{'permission': 'read'}, {'permission': 'write'}] + reviews = [{'user': {'id': 1, 'type': 'User', 'login': 'reader'}}, + {'user': {'id': 2, 'type': 'User', 'login': 'writer'}}] + result = gate.eligible_reviews(api, reviews) + self.assertFalse(result[0]['_may_approve']) + self.assertTrue(result[1]['_may_approve']) + + def test_older_publication_does_not_move_latest_image(self): + from unittest import mock + import os + plan = dict(self.candidate(), commit='e' * 40) + api = mock.Mock(repository='example/s5cmd') + api.request.return_value = {'object': {'sha': 'f' * 40}} + for current, latest in [('2.4.0', True), ('2.5.0', False)]: + api.content.return_value = current + with mock.patch.dict(os.environ, {'RELEASE_PLAN': json.dumps(plan)}), mock.patch.object(gate, 'outputs') as output: + gate.publication_tags(api) + self.assertEqual(':latest' in output.call_args.args[0]['tags'], latest) + + def test_inspection_never_enters_storage_for_ordinary_or_spoofed_pr(self): + from unittest import mock + from pathlib import Path + import os + import tempfile + api = mock.Mock() + api.pages.return_value = [] + self.pr['user']['id'] = 456 + ordinary = copy.deepcopy(self.pr) + ordinary['head']['ref'] = 'feature' + ordinary['labels'] = [] + with tempfile.TemporaryDirectory() as directory: + event = Path(directory) / 'event.json' + event.write_text(json.dumps({'pull_request': {'number': 7}})) + environment = {'GITHUB_EVENT_PATH': str(event), 'GITHUB_RUN_ID': '42', 'GITHUB_RUN_ATTEMPT': '1', 'RELEASE_AUTOMATION_ENABLED': 'true'} + api.request.side_effect = [ordinary, {'id': 8, 'external_id': '42:1'}, None] + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'outputs') as output: + gate.inspect(api, 123) + output.assert_called_once_with({'live': 'false'}) + api.request.side_effect = [self.pr, {'id': 8, 'external_id': '42:1'}, None] + environment['RELEASE_AUTOMATION_ENABLED'] = 'false' + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'outputs') as output, self.assertRaises(ValueError): + gate.inspect(api, 123) + output.assert_not_called() + self.assertEqual(api.request.call_args.args[2]['conclusion'], 'failure') + + def test_failed_suite_and_revision_change_cannot_record_success(self): + from unittest import mock + import os + proof = dict(self.candidate(), run_id=42, run_attempt=1) + for result, changed in [('failure', False), ('success', True)]: + current = self.candidate() + if changed: + current['base'] = 'f' * 40 + api = mock.Mock() + environment = {'RELEASE_PROOF': json.dumps(proof), 'RELEASE_CHECK': '8', 'ACCEPTANCE_RESULT': result, + 'GITHUB_RUN_ID': '42', 'GITHUB_RUN_ATTEMPT': '1'} + with mock.patch.dict(os.environ, environment), mock.patch.object(gate, 'candidate', return_value=current), self.assertRaises(ValueError): + gate.finish(api, 123) + self.assertEqual(api.request.call_args.args[2]['conclusion'], 'failure') + + def test_error_output_does_not_echo_api_response(self): + from unittest import mock + import contextlib + import io + output = io.StringIO() + with mock.patch.object(gate, 'execute', side_effect=ValueError('synthetic-private-value')), contextlib.redirect_stderr(output): + self.assertEqual(gate.main(['inspect']), 1) + self.assertEqual(output.getvalue(), 'release authorization failed\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_run_storage_acceptance.py b/scripts/test_run_storage_acceptance.py new file mode 100644 index 000000000..3eb6223d8 --- /dev/null +++ b/scripts/test_run_storage_acceptance.py @@ -0,0 +1,72 @@ +import contextlib +import io +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +import run_storage_acceptance as runner + + +class RunnerTests(unittest.TestCase): + def test_cancellation_uses_the_same_cleanup_window(self): + process = mock.Mock(pid=123) + handlers = {} + def register(number, handler): + previous = handlers.get(number) + handlers[number] = handler + return previous + waits = 0 + def wait(timeout): + nonlocal waits + waits += 1 + if waits == 1: + handlers[runner.signal.SIGTERM](runner.signal.SIGTERM, None) + return 1 + process.wait.side_effect = wait + with mock.patch.object(runner.subprocess, 'Popen', return_value=process), mock.patch.object(runner.signal, 'signal', side_effect=register), mock.patch.object(runner.os, 'killpg') as kill: + self.assertFalse(runner.bounded(['test'], '.', 1)) + kill.assert_called_once_with(123, runner.signal.SIGTERM) + self.assertEqual(process.wait.call_args.kwargs['timeout'], 240) + self.assertEqual(handlers, {runner.signal.SIGINT: None, runner.signal.SIGTERM: None}) + + def test_timeout_terminates_group_and_waits_for_cleanup(self): + process = mock.Mock(pid=123) + process.wait.side_effect = [subprocess.TimeoutExpired('test', 1), 1] + with mock.patch.object(runner.subprocess, 'Popen', return_value=process), mock.patch.object(runner.os, 'killpg') as kill: + self.assertFalse(runner.bounded(['test'], '.', 1)) + kill.assert_called_once_with(123, runner.signal.SIGTERM) + self.assertEqual(process.wait.call_args.kwargs['timeout'], 240) + + def test_process_output_is_never_forwarded_to_logs(self): + with tempfile.TemporaryDirectory() as directory: + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + self.assertFalse(runner.bounded(['python3', '-c', 'import sys; print("synthetic-private-value"); sys.exit(1)'], directory, 5)) + self.assertEqual(output.getvalue(), '') + + def test_private_profile_references_trusted_helper_only(self): + with tempfile.TemporaryDirectory() as directory, mock.patch.dict(os.environ, {'RUNNER_TEMP': directory}, clear=True): + root = runner.configure(Path(directory) / 'control') + self.assertEqual(root.stat().st_mode & 0o777, 0o700) + self.assertEqual((root / 'config').stat().st_mode & 0o777, 0o600) + self.assertIn('control/scripts/oidc_credentials.py', (root / 'config').read_text()) + self.assertEqual((root / 'credentials').read_text(), '') + + def test_existing_private_directory_is_not_reused(self): + with tempfile.TemporaryDirectory() as directory, mock.patch.dict(os.environ, {'RUNNER_TEMP': directory}, clear=True): + runner.configure(Path(directory) / 'control') + with self.assertRaises(FileExistsError): + runner.configure(Path(directory) / 'control') + + def test_fixed_error_does_not_expose_configuration(self): + output = io.StringIO() + with mock.patch.object(runner, 'execute', side_effect=ValueError('synthetic-private-value')), contextlib.redirect_stderr(output): + self.assertEqual(runner.main(['test']), 1) + self.assertEqual(output.getvalue(), 'storage acceptance operation failed\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/version.txt b/version.txt new file mode 100644 index 000000000..276cbf9e2 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +2.3.0