From 59b12da93ea04b947d8870e6276af080cbaf27f5 Mon Sep 17 00:00:00 2001 From: MagMueller Date: Fri, 25 Sep 2026 05:37:48 +0000 Subject: [PATCH] Prepare v2.1.1 release of current runner and dataset --- README.md | 16 +++++++++++----- RUBRIC_REVISION.md | 21 +++++++++++++++++++++ rubric_revision.json | 11 ++++++----- tests/test_rubric_revision.py | 5 +++-- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 36bd06e..e88dbe4 100644 --- a/README.md +++ b/README.md @@ -36,10 +36,15 @@ **200 web tasks scored against weighted findings rubrics — the default task set.** +Latest release: [v2.1.1](https://github.com/browser-use/benchmark/releases/latest). +It includes the updated BrowserCode runner and the September 25 CAPTCHA and Reuters +changes. Download its source code for the complete runner, or use the setup below; +the encrypted dataset, review cases, revision metadata and checksums are also attached. + ### Default evaluation command After [installing the runner and setting your API keys](#running-bu-bench-v21-default), -run this from the repository root on `main`: +run this from the repository root in `v2.1.1` or current `main`: ```bash uv run python run_eval.py @@ -58,7 +63,7 @@ Results go to `run_data/BU_Bench_V2_bcode_/`. | [BU_Bench_V2_review_cases.enc](BU_Bench_V2_review_cases.enc) | Judge review scenarios; not a runnable task set | | [rubric_revision.json](rubric_revision.json) | Revision metadata and integrity hashes | -Main includes the September 25 Reuters playback revision +Release `v2.1.1` and current main include the September 25 Reuters playback revision (`2026-09-25-reuters-playback`), following the CAPTCHA-alignment update and the corrections in the [original V2.1 snapshot](https://github.com/browser-use/benchmark/releases/tag/v2.1). All 200 task IDs remain. Task 185 has a prospective playback contract and @@ -68,8 +73,9 @@ remains `BU_Bench_V2.enc` for compatibility; its contents are V2.1. The old 55-task selectors and original 200-task snapshot are available through Git history, rather than as extra files in the current checkout. The original -`v2.1` tag and its download assets retain the September 24 snapshot; use main -for the latest V2.1 data and record the commit and encrypted-file checksum +`v2.1` tag and its download assets retain the September 24 snapshot. Use the +[latest release](https://github.com/browser-use/benchmark/releases/latest) for the +updated runner and V2.1 data; record its tag, commit and encrypted-file checksum with your results. See [changes, validation and remaining work](RUBRIC_REVISION.md). The earlier @@ -101,7 +107,7 @@ platform is needed. Use macOS or Linux (including GitHub's Ubuntu runners). Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then: ```bash -git clone https://github.com/browser-use/benchmark.git +git clone --branch v2.1.1 --single-branch https://github.com/browser-use/benchmark.git cd benchmark uv sync --frozen --python 3.12 curl -fsSL https://bcode.sh/install | bash -s -- --version 0.1.20 --no-modify-path diff --git a/RUBRIC_REVISION.md b/RUBRIC_REVISION.md index 91afe58..c67750d 100644 --- a/RUBRIC_REVISION.md +++ b/RUBRIC_REVISION.md @@ -1,3 +1,24 @@ +# BU Bench V2.1.1 release + +[Release v2.1.1](https://github.com/browser-use/benchmark/releases/tag/v2.1.1) +packages the current V2.1 dataset with all merged runner and rubric updates +through the September 25 Reuters playback redesign. + +- The complete source includes the BrowserCode 0.1.20 runner, Cloud/local browser + support, Luna xhigh executor and findings judge, 200 tasks, 100-concurrency + defaults, one-hour task budgets and GitHub Actions execution. +- Download assets contain the encrypted dataset, encrypted semantic review cases, + release metadata and SHA-256 checksums. Obsolete 55-task selectors are omitted. +- Dataset and review-case bytes are unchanged from the reviewed Reuters revision. + This release changes only documentation, release metadata and their test fixtures. +- Historical scores are unchanged. Task 185 has one successful live validation; + the other pending saved-trace reviews listed below remain pending. + +The original v2.1 release/tag remains available for reproducing older results. +Use the exact release tag and recorded dataset hash when comparing scores. + +--- + # BU Bench V2.1 Reuters playback revision **Content revision:** `2026-09-25-reuters-playback` diff --git a/rubric_revision.json b/rubric_revision.json index 0a97c20..29c648a 100644 --- a/rubric_revision.json +++ b/rubric_revision.json @@ -1,8 +1,8 @@ { "revision": "2026-09-25-reuters-playback", - "status": "main_not_regraded", - "release_version": "2.1", - "release_tag": null, + "status": "released_not_regraded", + "release_version": "2.1.1", + "release_tag": "v2.1.1", "base_commit": "421390ea7fa4708f3d89d7695f9a16debb861daf", "base_encrypted_sha256": "fe0fc1eede3197d9eaffd42af3ac7b11cc15743d401e95496f3eb03f0d023a0e", "hash_convention": "sha256 of UTF-8 text exactly as distributed; revised weights use sorted compact JSON", @@ -243,7 +243,8 @@ "bu2-020", "bu2-044", "bu2-047", - "bu2-154" + "bu2-154", + "bu2-185" ], - "release_note": "V2.1 rubric update on main; the original v2.1 tag and release assets retain the September 24 snapshot." + "release_note": "V2.1.1 includes the merged BrowserCode runner/defaults, CAPTCHA alignment, dataset cleanup and validated Reuters playback redesign. Historical results are unchanged; the original v2.1 tag/assets retain their snapshot." } diff --git a/tests/test_rubric_revision.py b/tests/test_rubric_revision.py index bc5cff3..18778c1 100644 --- a/tests/test_rubric_revision.py +++ b/tests/test_rubric_revision.py @@ -98,7 +98,7 @@ def test_only_declared_contracts_change(self): self.assertTrue(changed_fields <= allowed) self.assertEqual(after[task_id]["weights"], before[task_id]["weights"]) self.assertEqual(len(cases["cases"]), 32) - self.assertEqual(manifest["status"], "main_not_regraded") + self.assertEqual(manifest["status"], "released_not_regraded") def test_baseline_from_history_is_exact_published_snapshot(self): self.assertEqual( @@ -112,7 +112,8 @@ def test_current_dataset_is_pinned_and_legacy_v2_files_are_absent(self): hashlib.sha256((ROOT / "BU_Bench_V2.enc").read_bytes()).hexdigest(), manifest["candidate_encrypted_sha256"], ) - self.assertEqual(manifest["release_version"], "2.1") + self.assertEqual(manifest["release_version"], "2.1.1") + self.assertEqual(manifest["release_tag"], "v2.1.1") for name in ( "BU_Bench_V2_55.json", "snapshots/BU_Bench_V2_55_2026-08-25.json",