diff --git a/.github/workflows/benchmark-external.yml b/.github/workflows/benchmark-external.yml index eab9fc1..bf2a340 100644 --- a/.github/workflows/benchmark-external.yml +++ b/.github/workflows/benchmark-external.yml @@ -2,8 +2,12 @@ name: NAD Benchmark (label-triggered) # The active NAD benchmark gate. Policy: EVERY PR (internal or external fork) is # gated on demand — a maintainer reviews the PR and adds the `run-benchmark` -# label to run it. (Automatic gating in benchmark.yml is disabled by policy; -# this label gate is the one that runs.) +# label to run it. This is the only NAD benchmark workflow. +# +# Maintainers can also run it MANUALLY from the Actions tab (Run workflow / +# workflow_dispatch) against any branch, with optional subset/count/gate inputs. +# Manual runs are trusted (only maintainers can trigger them), so they check out +# the selected branch directly instead of an untrusted PR head. # # It uses pull_request_target so the run has access to KIRO_API_KEY even for fork # PRs (a plain pull_request gate can't — fork PRs receive no secrets), and is @@ -28,6 +32,20 @@ on: # never auto-runs on push. A maintainer must (re-)apply the label each time. types: [labeled] + # Manual runs from the Actions tab. Only users with write access can dispatch + # this, so the run is trusted — it scores the selected branch's own code. + workflow_dispatch: + inputs: + subset: + description: "Subset(s): 'all', a single name, or a comma-separated list" + default: 'all' + count: + description: 'Pass@k attempts per scenario' + default: '1' + gate: + description: 'Required pass rate (0-100); job fails below this' + default: '90' + # Least privilege: the base-context GITHUB_TOKEN is write-capable by default under # pull_request_target. Restrict it to read so untrusted code can't use it to push, # open releases, or modify the repo. @@ -35,26 +53,28 @@ permissions: contents: read concurrency: - group: nad-benchmark-external-${{ github.event.pull_request.number }} + group: nad-benchmark-external-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: gate: - # Run ONLY when a maintainer added the `run-benchmark` label. This `if` is the - # trust gate — without it, pull_request_target would run untrusted code with - # secrets on every PR event. - if: github.event.label.name == 'run-benchmark' + # Run on a manual dispatch, OR when a maintainer added the `run-benchmark` + # label. For the label path this `if` is the trust gate — without it, + # pull_request_target would run untrusted code with secrets on every PR event. + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-benchmark' name: NAD external benchmark (labeled) runs-on: ubuntu-latest timeout-minutes: 20 steps: - # Check out the PR's HEAD — i.e. the contributor's UNTRUSTED code. This is - # required (we must score their skill) and is the crux of the risk above. - # The workflow FILE itself still comes from base (pull_request_target), so - # the contributor cannot alter these steps — only the skill/agent/test/ - # grader content that the steps then run. - - name: Checkout PR head (untrusted) + # Label runs: check out the PR's HEAD — i.e. the contributor's UNTRUSTED + # code. This is required (we must score their skill) and is the crux of the + # risk above. The workflow FILE itself still comes from base + # (pull_request_target), so the contributor cannot alter these steps — only + # the skill/agent/test/grader content that the steps then run. + # Manual runs: head.sha is empty, so checkout falls back to the selected + # branch (trusted — only maintainers can dispatch). + - name: Checkout PR head (untrusted on label runs) uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -69,13 +89,14 @@ jobs: KIRO_API_KEY: ${{ secrets.KIRO_API_KEY }} run: | set -euo pipefail + # Inputs come from the manual dispatch form; label runs use the defaults. python benchmark/run_skill_tests.py \ --nad-root=. \ - --subset=all \ - --count=1 \ + --subset="${{ github.event.inputs.subset || 'all' }}" \ + --count="${{ github.event.inputs.count || '1' }}" \ --workers=4 \ --judge \ - --gate=90 + --gate="${{ github.event.inputs.gate || '90' }}" - name: Upload results if: always() diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index c3005bd..0000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: NAD Benchmark - -# Single-repo NAD skill benchmark gate. -# -# Everything the gate needs lives in THIS repo: the inputs it scores -# (skills/, agents/, tests/, tests/registry.json) at the repo root, and the -# grader under benchmark/ (run_skill_tests.py + shell/ + environment/Dockerfile). This -# workflow is self-contained — it runs the grader directly against the repo -# root, so a PR that changes the inputs OR the grader is scored against this -# repo's own assets. -# -# NOTE: automatic PR gating is DISABLED. The current policy is that EVERY PR is -# gated on demand via the label-triggered workflow (benchmark-external.yml): a -# maintainer adds the `run-benchmark` label. This workflow is kept for manual -# runs (workflow_dispatch) and can be re-enabled as an automatic gate by -# uncommenting the `pull_request:` trigger below. - -on: - # Automatic PR gate — DISABLED by policy (label-triggered gate is used instead). - # To re-enable automatic gating on trusted PRs, uncomment this block: - # pull_request: - # paths: - # - 'skills/**' - # - 'agents/**' - # - 'tests/**' - # - 'benchmark/**' - # - '.github/workflows/benchmark.yml' - - workflow_dispatch: - inputs: - subset: - description: "Subset(s): 'all', a single name, or a comma-separated list" - default: 'all' - count: - description: 'Pass@k attempts per scenario' - default: '1' - gate: - description: 'Required pass rate (0-100); job fails below this' - default: '90' - -permissions: - contents: read - -concurrency: - group: nad-benchmark-${{ github.ref }} - cancel-in-progress: true - -jobs: - benchmark: - name: NAD ${{ github.event.inputs.subset || 'all' }} benchmark - runs-on: ubuntu-latest - timeout-minutes: 20 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Run NAD benchmark - env: - # kiro-cli reads KIRO_API_KEY; docker.sh passes it into the container. - # The same key drives the agent and the judge. - KIRO_API_KEY: ${{ secrets.KIRO_API_KEY }} - run: | - set -euo pipefail - # nad-root '.' scores this checkout's own skills/agents/tests. The judge - # (25% weight) is on; workers=4. Gate fails the job below the threshold. - python benchmark/run_skill_tests.py \ - --nad-root=. \ - --subset="${{ github.event.inputs.subset || 'all' }}" \ - --count="${{ github.event.inputs.count || '1' }}" \ - --workers=4 \ - --judge \ - --gate="${{ github.event.inputs.gate || '90' }}" - - - name: Upload results - if: always() - uses: actions/upload-artifact@v4 - with: - name: nad-benchmark-results - path: benchmark/build/nad_benchmark_results.json - if-no-files-found: warn