diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 0f29fab..0af96a2 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -32,7 +32,7 @@ on: max_parallel: description: Maximum concurrent task runners type: number - default: 12 + default: 100 fetch_use: description: Enable BrowserCode fetch service (requires Browser Use key) type: boolean @@ -67,7 +67,7 @@ jobs: run: | python3 - <<'PY' import os - assert 1 <= int(os.environ["MAX_PARALLEL"]) <= 50 + assert 1 <= int(os.environ["MAX_PARALLEL"]) <= 200 assert 1 <= int(os.environ["TASK_TIMEOUT"]) <= 14400 PY uv run python bcode_results.py prepare --tasks "$TASK_COUNT" --task-ids "$TASK_IDS" diff --git a/README.md b/README.md index c1efba4..1de0ae2 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Defaults: | Executor | BrowserCode 0.1.20, `openai/gpt-6-luna`, xhigh reasoning | | Tasks | All 200 in `BU_Bench_V2.enc` | | Browser | Browser Use Cloud, one session per task | -| Limits | 3 concurrent tasks; 3,600 seconds per task | +| Limits | Up to 100 concurrent tasks; 3,600 seconds per task | | Judge | `gpt-5.6-luna`, xhigh reasoning | | Score | Continuous weighted V2 rubric score, including partial credit | @@ -170,9 +170,10 @@ bu2-001/ ``` Use `--tasks 5` for a short run, or `--task-ids bu2-171 bu2-185` for exact cases. -`--parallel` controls tasks per process; `--task-timeout` is in seconds. `--model` -and `--agent-reasoning` select another BrowserCode model/variant, while -`--judge-model` and `--judge-reasoning` change the findings judge. `--check` +`--parallel` controls tasks per process (default 100); use a smaller value such as +`--parallel 5` to fit your browser capacity or API limits. `--task-timeout` is in +seconds. `--model` and `--agent-reasoning` select another BrowserCode model/variant, +while `--judge-model` and `--judge-reasoning` change the findings judge. `--check` performs the binary/model preflight without executing tasks. The historical V1, Stealth, and framework comparison runners remain available @@ -185,9 +186,11 @@ batch/orchestrator path. ### Running the default evaluation on GitHub Actions The manual **Run BU Bench V2** workflow runs one task per GitHub-hosted Ubuntu -runner, with at most 12 task runners active. Every runner invokes the same -`run_eval.py` command and judges its task. All 200 tasks run by default. The -aggregate job requires every selected task exactly once and reports incomplete +runner, with up to 100 task runners active by default, subject to your GitHub +account's concurrency limits. Set `max_parallel` to override this (1–200). +Every runner invokes the same `run_eval.py` command and judges its task. +All 200 tasks run by default. The aggregate job requires every selected task +exactly once and reports incomplete judging as an error instead of treating it as a zero or silently dropping it. For your own runs, fork the repository, enable Actions, and add your own secrets diff --git a/run_eval.py b/run_eval.py index 4811449..e584809 100644 --- a/run_eval.py +++ b/run_eval.py @@ -342,11 +342,13 @@ def parse_args(argv=None): parser.add_argument("--shard-count", type=int, help="Total contiguous shards") parser.add_argument("--task-timeout", type=int, default=None) parser.add_argument("--max-steps", type=int, default=None) - parser.add_argument("--concurrency", "--parallel", type=int, default=MAX_CONCURRENT) + parser.add_argument("--concurrency", "--parallel", type=int, default=None) args = parser.parse_args(argv) args.executor = args.executor or ( "bcode" if args.benchmark == DEFAULT_BENCHMARK else "browser-use" ) + if args.concurrency is None: + args.concurrency = 100 if args.executor == "bcode" else MAX_CONCURRENT if (args.shard_index is None) != (args.shard_count is None): parser.error("--shard-index and --shard-count must be provided together") if args.shard_count is not None and ( diff --git a/tests/test_bcode_runner.py b/tests/test_bcode_runner.py index b8a480f..e0d7ad0 100644 --- a/tests/test_bcode_runner.py +++ b/tests/test_bcode_runner.py @@ -57,7 +57,7 @@ def test_defaults_select_all_200_and_only_bcode_findings_path(self): ) self.assertEqual( (args.bcode_version, args.task_timeout, args.concurrency), - ("0.1.20", 3600, 3), + ("0.1.20", 3600, 100), ) with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): run_eval.parse_args(["--framework", "browser-use"]) @@ -98,6 +98,7 @@ def test_stealth_and_v1_keep_their_existing_executor_and_datasets(self): ("browser-use", "bu-2-0", "local_headless"), ) self.assertEqual(len(load_tasks(benchmark)), count) + self.assertEqual(args.concurrency, 3) explicit = run_eval.parse_args(["--executor", "browser-use"]) self.assertEqual(explicit.benchmark, "BU_Bench_V2") self.assertEqual(explicit.model, "bu-2-0")