From 4fa96160fd3080d85186fc03dc2b792748173846 Mon Sep 17 00:00:00 2001 From: Yifan Lu <111810457+yl231@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:54:03 -0500 Subject: [PATCH] Re-grade workflow: 4 evaluator workers by default, log memory The first full re-grade run lost all ten of its first jobs: each runner got a shutdown signal about two minutes into grading with 16 worker threads, while the PR evaluation of the same files succeeds. The worker count is now an input (default 4), progress is logged as in the PR evaluation, and memory and swap use are printed every 30 seconds so a failure shows its cause. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/regrade-all.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regrade-all.yml b/.github/workflows/regrade-all.yml index 15402343..ea4dcbd0 100644 --- a/.github/workflows/regrade-all.yml +++ b/.github/workflows/regrade-all.yml @@ -13,6 +13,10 @@ on: description: "Space-separated router names (default: every prediction file on the branch)" required: false default: "" + num_workers: + description: "Evaluator worker threads per job (16 exhausted runner memory)" + required: false + default: "4" permissions: contents: read @@ -97,9 +101,14 @@ jobs: env: ROUTERARENA_DATASET_DIR: ${{ github.workspace }}/dataset ROUTER: ${{ matrix.router }} + NUM_WORKERS: ${{ inputs.num_workers || '4' }} run: | set -euo pipefail - uv run python llm_evaluation/run.py "$ROUTER" full --force --save-interval 0 + # Log memory and swap use every 30 s, so a runner that dies shows why. + (while sleep 30; do free -m | awk 'NR==2{u=$3} NR==3{s=$3} END{print "mem used " u " MB, swap used " s " MB"}'; done) & + memlog=$! + uv run python llm_evaluation/run.py "$ROUTER" full --force --num-workers "$NUM_WORKERS" + kill "$memlog" mkdir -p regrade-out cp "router_inference/predictions/${ROUTER}.json" regrade-out/ cp metrics.json "regrade-out/${ROUTER}.metrics.json"