From b705589e26ef56b7b79b69d8bb2538d68d67376e Mon Sep 17 00:00:00 2001 From: MagMueller Date: Fri, 25 Sep 2026 03:42:08 +0000 Subject: [PATCH 1/2] Default BrowserCode evaluations to 100 concurrent tasks --- .github/workflows/run-benchmark.yml | 4 ++-- README.md | 12 +++++++----- run_eval.py | 4 +++- tests/test_bcode_runner.py | 3 ++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 0f29fab..0af96a2 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -32,7 +32,7 @@ on: max_parallel: description: Maximum concurrent task runners type: number - default: 12 + default: 100 fetch_use: description: Enable BrowserCode fetch service (requires Browser Use key) type: boolean @@ -67,7 +67,7 @@ jobs: run: | python3 - <<'PY' import os - assert 1 <= int(os.environ["MAX_PARALLEL"]) <= 50 + assert 1 <= int(os.environ["MAX_PARALLEL"]) <= 200 assert 1 <= int(os.environ["TASK_TIMEOUT"]) <= 14400 PY uv run python bcode_results.py prepare --tasks "$TASK_COUNT" --task-ids "$TASK_IDS" diff --git a/README.md b/README.md index c1efba4..0db742b 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Defaults: | Executor | BrowserCode 0.1.20, `openai/gpt-6-luna`, xhigh reasoning | | Tasks | All 200 in `BU_Bench_V2.enc` | | Browser | Browser Use Cloud, one session per task | -| Limits | 3 concurrent tasks; 3,600 seconds per task | +| Limits | Up to 100 concurrent tasks; 3,600 seconds per task | | Judge | `gpt-5.6-luna`, xhigh reasoning | | Score | Continuous weighted V2 rubric score, including partial credit | @@ -170,8 +170,9 @@ bu2-001/ ``` Use `--tasks 5` for a short run, or `--task-ids bu2-171 bu2-185` for exact cases. -`--parallel` controls tasks per process; `--task-timeout` is in seconds. `--model` -and `--agent-reasoning` select another BrowserCode model/variant, while +`--parallel` controls tasks per process (default 100); use a smaller value such as +`--parallel 5` to fit your browser capacity or API limits. `--task-timeout` is in +seconds. `--model` and `--agent-reasoning` select another BrowserCode model/variant, while `--judge-model` and `--judge-reasoning` change the findings judge. `--check` performs the binary/model preflight without executing tasks. @@ -185,8 +186,9 @@ batch/orchestrator path. ### Running the default evaluation on GitHub Actions The manual **Run BU Bench V2** workflow runs one task per GitHub-hosted Ubuntu -runner, with at most 12 task runners active. Every runner invokes the same -`run_eval.py` command and judges its task. All 200 tasks run by default. The +runner, with up to 100 task runners active by default, subject to your GitHub +account's concurrency limits. Set `max_parallel` to override this (1–200). +Every runner invokes the same `run_eval.py` command and judges its task. All 200 tasks run by default. The aggregate job requires every selected task exactly once and reports incomplete judging as an error instead of treating it as a zero or silently dropping it. diff --git a/run_eval.py b/run_eval.py index 4811449..e584809 100644 --- a/run_eval.py +++ b/run_eval.py @@ -342,11 +342,13 @@ def parse_args(argv=None): parser.add_argument("--shard-count", type=int, help="Total contiguous shards") parser.add_argument("--task-timeout", type=int, default=None) parser.add_argument("--max-steps", type=int, default=None) - parser.add_argument("--concurrency", "--parallel", type=int, default=MAX_CONCURRENT) + parser.add_argument("--concurrency", "--parallel", type=int, default=None) args = parser.parse_args(argv) args.executor = args.executor or ( "bcode" if args.benchmark == DEFAULT_BENCHMARK else "browser-use" ) + if args.concurrency is None: + args.concurrency = 100 if args.executor == "bcode" else MAX_CONCURRENT if (args.shard_index is None) != (args.shard_count is None): parser.error("--shard-index and --shard-count must be provided together") if args.shard_count is not None and ( diff --git a/tests/test_bcode_runner.py b/tests/test_bcode_runner.py index b8a480f..e0d7ad0 100644 --- a/tests/test_bcode_runner.py +++ b/tests/test_bcode_runner.py @@ -57,7 +57,7 @@ def test_defaults_select_all_200_and_only_bcode_findings_path(self): ) self.assertEqual( (args.bcode_version, args.task_timeout, args.concurrency), - ("0.1.20", 3600, 3), + ("0.1.20", 3600, 100), ) with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): run_eval.parse_args(["--framework", "browser-use"]) @@ -98,6 +98,7 @@ def test_stealth_and_v1_keep_their_existing_executor_and_datasets(self): ("browser-use", "bu-2-0", "local_headless"), ) self.assertEqual(len(load_tasks(benchmark)), count) + self.assertEqual(args.concurrency, 3) explicit = run_eval.parse_args(["--executor", "browser-use"]) self.assertEqual(explicit.benchmark, "BU_Bench_V2") self.assertEqual(explicit.model, "bu-2-0") From 66aec3402ffbf462ab37727b9a60f40fde9211f6 Mon Sep 17 00:00:00 2001 From: MagMueller Date: Fri, 25 Sep 2026 03:46:07 +0000 Subject: [PATCH 2/2] Wrap concurrency documentation consistently --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0db742b..1de0ae2 100644 --- a/README.md +++ b/README.md @@ -172,8 +172,8 @@ bu2-001/ Use `--tasks 5` for a short run, or `--task-ids bu2-171 bu2-185` for exact cases. `--parallel` controls tasks per process (default 100); use a smaller value such as `--parallel 5` to fit your browser capacity or API limits. `--task-timeout` is in -seconds. `--model` and `--agent-reasoning` select another BrowserCode model/variant, while -`--judge-model` and `--judge-reasoning` change the findings judge. `--check` +seconds. `--model` and `--agent-reasoning` select another BrowserCode model/variant, +while `--judge-model` and `--judge-reasoning` change the findings judge. `--check` performs the binary/model preflight without executing tasks. The historical V1, Stealth, and framework comparison runners remain available @@ -188,8 +188,9 @@ batch/orchestrator path. The manual **Run BU Bench V2** workflow runs one task per GitHub-hosted Ubuntu runner, with up to 100 task runners active by default, subject to your GitHub account's concurrency limits. Set `max_parallel` to override this (1–200). -Every runner invokes the same `run_eval.py` command and judges its task. All 200 tasks run by default. The -aggregate job requires every selected task exactly once and reports incomplete +Every runner invokes the same `run_eval.py` command and judges its task. +All 200 tasks run by default. The aggregate job requires every selected task +exactly once and reports incomplete judging as an error instead of treating it as a zero or silently dropping it. For your own runs, fork the repository, enable Actions, and add your own secrets