From d785d8e66b1f6fa8d0b2d329b90f315401f7324a Mon Sep 17 00:00:00 2001 From: Nhat Lam Date: Fri, 16 Oct 2026 15:13:17 +0000 Subject: [PATCH 1/2] fix(scorers): raise humaneval/mbpp verify timeout 3s -> 30s The timeout is wall clock measured on the harness event loop. At 128 connections a burst of finished samples saturates the single inspect process and correct answers time out: Qwen3.8-27B humaneval 0.99 -> 0.30 in DeepInfra's quality monitor, while the same answers verified alone pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SGrRPrAXX7iTxNrHDKCKN --- src/openbench/scorers/humaneval.py | 4 +++- src/openbench/scorers/mbpp.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/openbench/scorers/humaneval.py b/src/openbench/scorers/humaneval.py index 9f48c186..463c0e7f 100644 --- a/src/openbench/scorers/humaneval.py +++ b/src/openbench/scorers/humaneval.py @@ -13,7 +13,9 @@ from inspect_ai.solver import TaskState from inspect_ai.util import ExecResult, sandbox -TIMEOUT = 3 +# Wall clock on the harness event loop, not CPU time: under a burst of +# completions a saturated loop expired 3s on correct answers. +TIMEOUT = 30 # Adapted from https://github.com/UKGovernmentBEIS/inspect_evals diff --git a/src/openbench/scorers/mbpp.py b/src/openbench/scorers/mbpp.py index 8559ad93..b7986b4f 100644 --- a/src/openbench/scorers/mbpp.py +++ b/src/openbench/scorers/mbpp.py @@ -10,7 +10,9 @@ ) from inspect_ai.solver import TaskState -TIMEOUT = 3 +# Wall clock on the harness event loop, not CPU time: under a burst of +# completions a saturated loop expired 3s on correct answers. +TIMEOUT = 30 def parse_mbpp_response(response: str) -> str: From d8b7bea6b14b8587d34dee9a26ac8c603aa16bba Mon Sep 17 00:00:00 2001 From: Nhat Lam Date: Fri, 16 Oct 2026 15:18:40 +0000 Subject: [PATCH 2/2] fix(scorers): 10s, not 30s: a runaway holds a verify slot for the whole timeout Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012SGrRPrAXX7iTxNrHDKCKN --- src/openbench/scorers/humaneval.py | 2 +- src/openbench/scorers/mbpp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openbench/scorers/humaneval.py b/src/openbench/scorers/humaneval.py index 463c0e7f..7713a28e 100644 --- a/src/openbench/scorers/humaneval.py +++ b/src/openbench/scorers/humaneval.py @@ -15,7 +15,7 @@ # Wall clock on the harness event loop, not CPU time: under a burst of # completions a saturated loop expired 3s on correct answers. -TIMEOUT = 30 +TIMEOUT = 10 # Adapted from https://github.com/UKGovernmentBEIS/inspect_evals diff --git a/src/openbench/scorers/mbpp.py b/src/openbench/scorers/mbpp.py index b7986b4f..e7c8a546 100644 --- a/src/openbench/scorers/mbpp.py +++ b/src/openbench/scorers/mbpp.py @@ -12,7 +12,7 @@ # Wall clock on the harness event loop, not CPU time: under a burst of # completions a saturated loop expired 3s on correct answers. -TIMEOUT = 30 +TIMEOUT = 10 def parse_mbpp_response(response: str) -> str: