diff --git a/src/openbench/scorers/humaneval.py b/src/openbench/scorers/humaneval.py index 9f48c186..7713a28e 100644 --- a/src/openbench/scorers/humaneval.py +++ b/src/openbench/scorers/humaneval.py @@ -13,7 +13,9 @@ from inspect_ai.solver import TaskState from inspect_ai.util import ExecResult, sandbox -TIMEOUT = 3 +# Wall clock on the harness event loop, not CPU time: under a burst of +# completions a saturated loop expired 3s on correct answers. +TIMEOUT = 10 # Adapted from https://github.com/UKGovernmentBEIS/inspect_evals diff --git a/src/openbench/scorers/mbpp.py b/src/openbench/scorers/mbpp.py index 8559ad93..e7c8a546 100644 --- a/src/openbench/scorers/mbpp.py +++ b/src/openbench/scorers/mbpp.py @@ -10,7 +10,9 @@ ) from inspect_ai.solver import TaskState -TIMEOUT = 3 +# Wall clock on the harness event loop, not CPU time: under a burst of +# completions a saturated loop expired 3s on correct answers. +TIMEOUT = 10 def parse_mbpp_response(response: str) -> str: