From 3767f9a7959060f129e2c30fe503eaac8e594403 Mon Sep 17 00:00:00 2001
From: Yifan Lu <111810457+yl231@users.noreply.github.com>
Date: Thu, 29 Oct 2026 16:59:25 -0500
Subject: [PATCH 1/2] Add a workflow to re-grade merged routers on GitHub
 runners

Manual workflow (workflow_dispatch) that re-grades each merged prediction
file with the current evaluator, one router per job, and uploads the
re-graded file and its metrics as an artifact. LiveCodeBench grading runs
submitted code, so this belongs on the runners rather than a local machine.
Runner setup matches the PR evaluation workflow.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/regrade-all.yml | 112 ++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 .github/workflows/regrade-all.yml

diff --git a/.github/workflows/regrade-all.yml b/.github/workflows/regrade-all.yml
new file mode 100644
index 00000000..4f3f04e1
--- /dev/null
+++ b/.github/workflows/regrade-all.yml
@@ -0,0 +1,112 @@
+name: Re-grade routers
+
+# Re-grades merged prediction files with the current evaluator (scorers and price table),
+# one router per job. LiveCodeBench grading executes submitted code, so this runs on
+# GitHub's runners rather than locally. Each job uploads the re-graded prediction file
+# (per-row accuracy and cost) and its metrics as an artifact; a maintainer then opens
+# one PR with the results.
+
+on:
+  workflow_dispatch:
+    inputs:
+      routers:
+        description: "Space-separated router names (default: every prediction file on the branch)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+jobs:
+  list-routers:
+    runs-on: ubuntu-latest
+    outputs:
+      routers: ${{ steps.list.outputs.routers }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: List routers to re-grade
+        id: list
+        shell: bash
+        env:
+          ROUTERS: ${{ inputs.routers }}
+        run: |
+          set -euo pipefail
+          if [[ -n "$ROUTERS" ]]; then
+            names="$ROUTERS"
+          else
+            names=$(find router_inference/predictions -maxdepth 1 -name '*.json' ! -name '*-robustness.json' \
+              -exec basename {} .json \; | sort)
+          fi
+          for name in $names; do
+            if [[ ! -f "router_inference/predictions/${name}.json" ]]; then
+              echo "No prediction file for router: $name" >&2
+              exit 1
+            fi
+          done
+          echo "routers=$(printf '%s\n' $names | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
+
+  regrade:
+    needs: list-routers
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      max-parallel: 10
+      matrix:
+        router: ${{ fromJSON(needs.list-routers.outputs.routers) }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Free up disk space on the runner
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        run: |
+          set -euo pipefail
+          python -m pip install --upgrade pip
+          python -m pip install uv
+
+      - name: Prepare dataset
+        run: |
+          set -euo pipefail
+          mkdir -p dataset
+          uv run python scripts/process_datasets/prep_datasets.py
+
+      - name: Enable swap space
+        run: |
+          sudo fallocate -l 10G /swapfile10g
+          sudo chmod 600 /swapfile10g
+          sudo mkswap /swapfile10g
+          sudo swapon /swapfile10g
+
+      - name: Re-grade the full split
+        env:
+          ROUTERARENA_DATASET_DIR: ${{ github.workspace }}/dataset
+          ROUTER: ${{ matrix.router }}
+        run: |
+          set -euo pipefail
+          uv run python llm_evaluation/run.py "$ROUTER" full --force --save-interval 0
+          mkdir -p regrade-out
+          cp "router_inference/predictions/${ROUTER}.json" regrade-out/
+          cp metrics.json "regrade-out/${ROUTER}.metrics.json"
+
+      - name: Upload re-graded files
+        uses: actions/upload-artifact@v4
+        with:
+          name: regrade-${{ matrix.router }}
+          path: regrade-out/
+          retention-days: 14

From b56ea252c379cb59ab40b90bd0f7d28332c38fbf Mon Sep 17 00:00:00 2001
From: Yifan Lu <111810457+yl231@users.noreply.github.com>
Date: Thu, 29 Oct 2026 17:28:59 -0500
Subject: [PATCH 2/2] Use a bash array for the router list (shellcheck SC2086)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/regrade-all.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/regrade-all.yml b/.github/workflows/regrade-all.yml
index 4f3f04e1..15402343 100644
--- a/.github/workflows/regrade-all.yml
+++ b/.github/workflows/regrade-all.yml
@@ -33,18 +33,18 @@ jobs:
         run: |
           set -euo pipefail
           if [[ -n "$ROUTERS" ]]; then
-            names="$ROUTERS"
+            read -r -a names <<< "$ROUTERS"
           else
-            names=$(find router_inference/predictions -maxdepth 1 -name '*.json' ! -name '*-robustness.json' \
-              -exec basename {} .json \; | sort)
+            mapfile -t names < <(find router_inference/predictions -maxdepth 1 -name '*.json' \
+              ! -name '*-robustness.json' -exec basename {} .json \; | sort)
           fi
-          for name in $names; do
+          for name in "${names[@]}"; do
             if [[ ! -f "router_inference/predictions/${name}.json" ]]; then
               echo "No prediction file for router: $name" >&2
               exit 1
             fi
           done
-          echo "routers=$(printf '%s\n' $names | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
+          echo "routers=$(printf '%s\n' "${names[@]}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
 
   regrade:
     needs: list-routers