From 9b45e96b2aff51706f4ed56bad6ea25464b45208 Mon Sep 17 00:00:00 2001 From: Yifan Lu Date: Sat, 31 Oct 2026 17:53:54 -0500 Subject: [PATCH] CI: stop and retry dataset preparation when a download stalls prep_datasets.py's Hugging Face downloads occasionally stall without failing. On Oct 31 this hung two website syncs for over 90 minutes and a re-grade job for over an hour, each at "Prepare dataset", while other runs of the same step a minute apart finished in about three minutes. scripts/process_datasets/prepare_datasets.sh runs prep_datasets.py with a time limit (15 minutes per attempt by default) and up to three attempts. The /evaluate, re-grade and website-sync workflows call it instead of the script directly. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr-evaluation.yml | 2 +- .github/workflows/regrade-all.yml | 2 +- .github/workflows/sync-website.yml | 2 +- scripts/process_datasets/prepare_datasets.sh | 26 ++++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100755 scripts/process_datasets/prepare_datasets.sh diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml index 139bd717..2c9da6bc 100644 --- a/.github/workflows/pr-evaluation.yml +++ b/.github/workflows/pr-evaluation.yml @@ -258,7 +258,7 @@ jobs: # Uses base repo's script (safe - not from PR) echo "Preparing dataset..." mkdir -p dataset - uv run python scripts/process_datasets/prep_datasets.py + scripts/process_datasets/prepare_datasets.sh - name: Enable Swap Space run: | diff --git a/.github/workflows/regrade-all.yml b/.github/workflows/regrade-all.yml index ea4dcbd0..ce71d044 100644 --- a/.github/workflows/regrade-all.yml +++ b/.github/workflows/regrade-all.yml @@ -88,7 +88,7 @@ jobs: run: | set -euo pipefail mkdir -p dataset - uv run python scripts/process_datasets/prep_datasets.py + scripts/process_datasets/prepare_datasets.sh - name: Enable swap space run: | diff --git a/.github/workflows/sync-website.yml b/.github/workflows/sync-website.yml index ed36164a..764a74ec 100644 --- a/.github/workflows/sync-website.yml +++ b/.github/workflows/sync-website.yml @@ -41,7 +41,7 @@ jobs: - name: Prepare dataset env: ROUTERARENA_DATASET_DIR: ${{ github.workspace }}/dataset - run: uv run python scripts/process_datasets/prep_datasets.py + run: scripts/process_datasets/prepare_datasets.sh - name: Checkout website repo uses: actions/checkout@v5 diff --git a/scripts/process_datasets/prepare_datasets.sh b/scripts/process_datasets/prepare_datasets.sh new file mode 100755 index 00000000..16cf5a32 --- /dev/null +++ b/scripts/process_datasets/prepare_datasets.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 +# +# Run prep_datasets.py with a time limit, retrying a few times. The Hugging Face +# downloads it makes occasionally stall without failing, which used to hang CI jobs +# until their own timeout; a stalled attempt is now stopped and retried. +# +# Usage: scripts/process_datasets/prepare_datasets.sh +# PREP_TIMEOUT seconds per attempt (default 900) +# PREP_ATTEMPTS number of attempts (default 3) +set -euo pipefail + +timeout_s="${PREP_TIMEOUT:-900}" +attempts="${PREP_ATTEMPTS:-3}" +for attempt in $(seq 1 "$attempts"); do + if timeout --kill-after=30 "$timeout_s" uv run python scripts/process_datasets/prep_datasets.py; then + exit 0 + fi + echo "Dataset preparation attempt $attempt of $attempts failed or timed out." >&2 + if [[ "$attempt" -lt "$attempts" ]]; then + sleep 15 + fi +done +echo "Dataset preparation failed after $attempts attempts." >&2 +exit 1