browser-use · MagMueller · Sep 25, 2026 · Sep 25, 2026 · Sep 25, 2026 · Sep 25, 2026
diff --git a/BU_Bench_V2.enc b/BU_Bench_V2.enc
diff --git a/BU_Bench_V2_review_cases.enc b/BU_Bench_V2_review_cases.enc
diff --git a/README.md b/README.md
@@ -58,20 +58,22 @@ Results go to `run_data/BU_Bench_V2_bcode_<timestamp>/`.
 | [BU_Bench_V2_review_cases.enc](BU_Bench_V2_review_cases.enc) | Judge review scenarios; not a runnable task set |
 | [rubric_revision.json](rubric_revision.json) | Revision metadata and integrity hashes |
 
-Main includes the September 25 CAPTCHA-alignment update
-(`2026-09-25-captcha-alignment`), with eight further task/rubric corrections on
-top of the nine in the [original V2.1 snapshot](https://github.com/browser-use/benchmark/releases/tag/v2.1).
-All 200 task IDs, item IDs and weights are unchanged. The filename remains
-`BU_Bench_V2.enc` for compatibility; its contents are V2.1.
+Main includes the September 25 Reuters playback revision
+(`2026-09-25-reuters-playback`), following the CAPTCHA-alignment update and
+the corrections in the [original V2.1 snapshot](https://github.com/browser-use/benchmark/releases/tag/v2.1).
+All 200 task IDs remain. Task 185 has a prospective playback contract and
+scoring redesign; the other 199 tasks retain their prior contracts and weights.
+Historical task 185 scores are not comparable to the revised task. The filename
+remains `BU_Bench_V2.enc` for compatibility; its contents are V2.1.
 
 The old 55-task selectors and original 200-task snapshot are available through
 Git history, rather than as extra files in the current checkout. The original
 `v2.1` tag and its download assets retain the September 24 snapshot; use main
 for the latest V2.1 data and record the commit and encrypted-file checksum
 with your results.
 
-See [changes, validation and remaining work](RUBRIC_REVISION.md). The update
-requires successful Walmart source verification and gives no item credit to a
+See [changes, validation and remaining work](RUBRIC_REVISION.md). The earlier
+CAPTCHA-alignment update requires successful Walmart source verification and gives no item credit to a
 run blocked before obtaining any Walmart task result. Real partial results and
 the grocery task's evidenced offer shortfalls still count. It also aligns
 CAPTCHA recovery instructions, resolves an observation-record ambiguity, and

diff --git a/RUBRIC_REVISION.md b/RUBRIC_REVISION.md
@@ -1,3 +1,42 @@
+# BU Bench V2.1 Reuters playback revision
+
+**Content revision:** `2026-09-25-reuters-playback`
+**Dataset version:** 2.1. Historical executions and scores are unchanged.
+The original v2.1 release assets retain their earlier snapshot.
+
+Task 185 now measures actual news-video playback. It permits navigation to
+Reuters articles and removes the unreliable footer-location premise and the
+successful-absence alternative. Most credit requires evidenced normal playback;
+reporting an honest failure does not complete the playback requirement and is
+not automatically reward hacking. Native player evidence is required; seeking
+or an agent-drawn progress indicator cannot substitute for playback.
+
+This is a prospective task and scoring redesign, not a historical regrade.
+Only 185 changes its instruction, title, summary, rubric, item IDs and weights.
+The other 199 task contracts and weights, all task IDs/canaries, judge/scorer
+code, runner defaults and existing result artifacts remain unchanged.
+Existing revised tasks receive the common dataset revision identifier only.
+
+## Validation
+
+- The validator pins the original and replacement weight hashes for 185.
+  Every other task retains its original weights. Undeclared edits still fail.
+- Eight encrypted semantic cases cover valid article playback, absence, seeking,
+  buffering, advertising, short clips, disclosed overlays and honest blocks.
+  These cases are review inputs, not executed-model accuracy results.
+- One frozen-candidate execution (private Actions run 36097801340) scored
+  100/100 with BrowserCode 0.1.20, OpenAI Luna xhigh, Cloud and the unchanged
+  Luna xhigh findings judge. Six actual player samples demonstrated 25.54 seconds
+  of normal playback; the final screenshot visibly showed genuine player controls.
+  Trace, screenshot and configuration were inspected; no judge or evidence errors.
+  This demonstrates one successful execution, not site reliability across runs.
+- Earlier pending saved-trace reviews remain pending; this change does not
+  claim to complete them.
+
+The records below describe earlier revisions and their validation at the time.
+
+---
+
 # BU Bench V2.1 CAPTCHA-alignment update
 
 **Content revision:** `2026-09-25-captcha-alignment`

diff --git a/review_rubric_revision.py b/review_rubric_revision.py
@@ -13,6 +13,32 @@
 
 ROOT = Path(__file__).resolve().parent
 
+# One prospective scoring redesign; all other tasks retain original weights.
+APPROVED_WEIGHT_REVISIONS = {
+    "bu2-185": (
+        "a3a3b9bf46595905b842c65560b9054a46746dccd8bb70469de29165ba0814a4",
+        "de3256e1d436612e9b546532da0aa21d45c7b9e0ceabdc6fdd91e3b8528f0ba4",
+    ),
+}
+
+
+APPROVED_METADATA_REVISIONS = {
+    "bu2-185": {
+        "title": (
+            "a2e7dd85053f9c85b3251f2905009f24f15e89ba0df1317d41dcbf3eec0052a7",
+            "596ad9375e266e32f2be30feb5d80942ed57716a86b708add83c291b070033d5",
+        ),
+        "summary": (
+            "1f7be224bd117f9e7f03fab45dceabb81e3ce1494f8a4ebf8e29479d2485e6b3",
+            "4a611a12737aa08ac06befa841d5331af779d8b3174f564c4421326ddcdf111d",
+        ),
+    },
+}
+
+
+def weights_digest(weights: dict) -> str:
+    return digest(json.dumps(weights, sort_keys=True, separators=(",", ":")))
+
 
 def digest(text: str) -> str:
     return hashlib.sha256(text.encode()).hexdigest()
@@ -104,12 +130,38 @@ def validate_revision(
             for key in set(old) | set(task)
             if old.get(key, missing) != task.get(key, missing)
         }
-        if not changed_fields <= {"task", "rubric", "task_sha", "rubric_sha", "revision"}:
+        allowed_fields = {"task", "rubric", "task_sha", "rubric_sha", "revision"}
+        if task_id in APPROVED_WEIGHT_REVISIONS:
+            allowed_fields |= {"title", "summary", "weights", "weights_sha"}
+        if not changed_fields <= allowed_fields:
             raise ValueError(f"Unapproved task field change: {task_id}")
         if task.get("revision") != manifest["revision"]:
             raise ValueError(f"Task revision mismatch: {task_id}")
-        if task["weights"] != old["weights"] or sum(task["weights"].values()) != 100:
+        if task_id in APPROVED_WEIGHT_REVISIONS:
+            expected_before, expected_after = APPROVED_WEIGHT_REVISIONS[task_id]
+            if (
+                weights_digest(old["weights"]) != expected_before
+                or weights_digest(task["weights"]) != expected_after
+                or change.get("before_weights_sha256") != expected_before
+                or change.get("after_weights_sha256") != expected_after
+                or task.get("weights_sha") != expected_after
+            ):
+                raise ValueError(f"Unapproved weight revision: {task_id}")
+            for field, hashes in APPROVED_METADATA_REVISIONS[task_id].items():
+                expected_before, expected_after = hashes
+                if (
+                    digest(old[field]) != expected_before
+                    or digest(task[field]) != expected_after
+                    or change.get(f"before_{field}_sha256") != expected_before
+                    or change.get(f"after_{field}_sha256") != expected_after
+                ):
+                    raise ValueError(f"Metadata hash mismatch: {task_id}/{field}")
+        elif task["weights"] != old["weights"]:
             raise ValueError(f"Weight change: {task_id}")
+        if any(type(w) is not int or w <= 0 for w in task["weights"].values()):
+            raise ValueError(f"Invalid weights: {task_id}")
+        if sum(task["weights"].values()) != 100:
+            raise ValueError(f"Weight total changed: {task_id}")
         if task["canary"] != old["canary"] or task["canary"] in task["task"]:
             raise ValueError(f"Canary changed or exposed: {task_id}")
         item_ids = set(
@@ -150,7 +202,8 @@ def main():
     args = parser.parse_args()
     manifest, before, after, cases = validate_revision(base_artifact=args.base_artifact)
     print(
-        f"Validated {len(manifest['changes'])} revised tasks; other tasks and all weights unchanged."
+        f"Validated {len(manifest['changes'])} revised tasks; "
+        "only explicitly pinned scoring revisions may change weights."
     )
     print(
         f"{len(cases['cases'])} synthetic semantic review cases; no judge accuracy result implied."

diff --git a/rubric_revision.json b/rubric_revision.json
@@ -1,11 +1,11 @@
 {
-  "revision": "2026-09-25-captcha-alignment",
+  "revision": "2026-09-25-reuters-playback",
   "status": "main_not_regraded",
   "release_version": "2.1",
   "release_tag": null,
   "base_commit": "421390ea7fa4708f3d89d7695f9a16debb861daf",
   "base_encrypted_sha256": "fe0fc1eede3197d9eaffd42af3ac7b11cc15743d401e95496f3eb03f0d023a0e",
-  "hash_convention": "sha256 of UTF-8 text exactly as distributed; weights hashes unchanged",
+  "hash_convention": "sha256 of UTF-8 text exactly as distributed; revised weights use sorted compact JSON",
   "changes": [
     {
       "task_id": "bu2-001",
@@ -172,6 +172,32 @@
       "before_rubric_sha256": "6f84496a32e5d885253810079090239519b377d02badc36ee509350fb1479886",
       "after_rubric_sha256": "a7d479aefabfa0193cca178e7dd762d4f779b26dc892bfcf402f42e28f992c3b",
       "review_status": "pending_saved_trace_review"
+    },
+    {
+      "task_id": "bu2-185",
+      "summary": "Replace the footer/absence contract with actual news-video playback; reallocate scoring prospectively to the demonstrated outcome.",
+      "task_changed": true,
+      "review_status": "single_live_trace_validated",
+      "before_task_sha256": "162f44ed2f8269824ecb9696783b08328265761e7ea826feea22092ffb942c36",
+      "after_task_sha256": "ae5f78df23b9f2784932565dd8a1d6649750fcfa39b1b8174930b89e1c708604",
+      "before_rubric_sha256": "2820f3c00db846389b72b90b45785d364dcd15b532aea05dd07151ae4d1b0a4f",
+      "after_rubric_sha256": "d03a749363005b20e98e0042a9aee494e66c4e8e3f5c6f371d4d491095b8780f",
+      "before_title_sha256": "a2e7dd85053f9c85b3251f2905009f24f15e89ba0df1317d41dcbf3eec0052a7",
+      "after_title_sha256": "596ad9375e266e32f2be30feb5d80942ed57716a86b708add83c291b070033d5",
+      "before_summary_sha256": "1f7be224bd117f9e7f03fab45dceabb81e3ce1494f8a4ebf8e29479d2485e6b3",
+      "after_summary_sha256": "4a611a12737aa08ac06befa841d5331af779d8b3174f564c4421326ddcdf111d",
+      "before_weights_sha256": "a3a3b9bf46595905b842c65560b9054a46746dccd8bb70469de29165ba0814a4",
+      "after_weights_sha256": "de3256e1d436612e9b546532da0aa21d45c7b9e0ceabdc6fdd91e3b8528f0ba4",
+      "live_validation": {
+        "run_id": 36097801340,
+        "tested_public_commit": "46da763ffe97fc031f82e0e924ca0fdbbc4a867e",
+        "dataset_sha256": "0c014b056192e261b06f04ed6da6dcf37f2135150cf58c87ea839bd4e150e5c3",
+        "executor": "BrowserCode 0.1.20 / openai/gpt-6-luna xhigh",
+        "judge": "gpt-5.6-luna xhigh findings",
+        "browser": "browser-use-cloud",
+        "score": 100,
+        "scope": "One execution; trace and screenshot inspected. Not a reliability estimate or historical regrade."
+      }
     }
   ],
   "deferred": [
@@ -186,15 +212,11 @@
     {
       "task_id": "bu2-040",
       "issue": "Keep success requirements; compare equal budgets. Partner-reported 100% run is not independently verified."
-    },
-    {
-      "task_id": "bu2-185",
-      "issue": "No in-scope playback demonstrated in seven reviewed historical executions; retain evidenced-absence outcome, require verified target before a playback-only revision."
     }
   ],
-  "candidate_encrypted_sha256": "7ef05a1d0abdf6ab4b570cae5b93adf3299cc5c3c2749961473d68f5cdc7bf94",
-  "previous_revision": "2026-09-24-feedback-review-draft-4",
-  "previous_encrypted_sha256": "87101f7ebcf3bfbde00091e278741427e906c9b7ccc223892c95e4549704d7c6",
+  "candidate_encrypted_sha256": "0c014b056192e261b06f04ed6da6dcf37f2135150cf58c87ea839bd4e150e5c3",
+  "previous_revision": "2026-09-25-captcha-alignment",
+  "previous_encrypted_sha256": "7ef05a1d0abdf6ab4b570cae5b93adf3299cc5c3c2749961473d68f5cdc7bf94",
   "reverted_changes": [
     {
       "task_id": "bu2-171",

diff --git a/tests/test_rubric_revision.py b/tests/test_rubric_revision.py
@@ -12,7 +12,12 @@
 
 from cryptography.fernet import Fernet
 
-from review_rubric_revision import ROOT, read_base_artifact, validate_revision
+from review_rubric_revision import (
+    ROOT,
+    read_base_artifact,
+    validate_revision,
+    weights_digest,
+)
 
 
 class RevisionTests(unittest.TestCase):
@@ -55,6 +60,7 @@ def test_only_declared_contracts_change(self):
             "bu2-113",
             "bu2-163",
             "bu2-187",
+            "bu2-185",
         }
         self.assertEqual({c["task_id"] for c in manifest["changes"]}, expected)
         self.assertEqual(
@@ -72,6 +78,7 @@ def test_only_declared_contracts_change(self):
                 "bu2-029",
                 "bu2-088",
                 "bu2-113",
+                "bu2-185",
             },
         )
         allowed = {"task", "rubric", "task_sha", "rubric_sha", "revision"}
@@ -82,9 +89,15 @@ def test_only_declared_contracts_change(self):
                 for key in set(before[task_id]) | set(after[task_id])
                 if before[task_id].get(key, missing) != after[task_id].get(key, missing)
             }
-            self.assertTrue(changed_fields <= allowed)
-            self.assertEqual(after[task_id]["weights"], before[task_id]["weights"])
-        self.assertEqual(len(cases["cases"]), 24)
+            if task_id == "bu2-185":
+                self.assertTrue(
+                    changed_fields <= allowed | {"title", "summary", "weights", "weights_sha"}
+                )
+                self.assertNotEqual(after[task_id]["weights"], before[task_id]["weights"])
+            else:
+                self.assertTrue(changed_fields <= allowed)
+                self.assertEqual(after[task_id]["weights"], before[task_id]["weights"])
+        self.assertEqual(len(cases["cases"]), 32)
         self.assertEqual(manifest["status"], "main_not_regraded")
 
     def test_baseline_from_history_is_exact_published_snapshot(self):
@@ -149,6 +162,64 @@ def test_weight_edit_rejected_even_with_updated_artifact_hash(self):
             with self.assertRaisesRegex(ValueError, "Unapproved task field change"):
                 validate_revision(root, base_artifact=baseline)
 
+    def test_redesign_cannot_rebalance_even_with_matching_manifest(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            baseline = self.copy_revision_fixture(root)
+            manifest, _, after, _ = validate_revision(root, base_artifact=baseline)
+            task = after["bu2-185"]
+            keys = list(task["weights"])
+            task["weights"][keys[0]] += 1
+            task["weights"][keys[1]] -= 1
+            task["weights_sha"] = weights_digest(task["weights"])
+            change = next(c for c in manifest["changes"] if c["task_id"] == "bu2-185")
+            change["after_weights_sha256"] = task["weights_sha"]
+            payload = {"tasks": list(after.values()), "revision": manifest["revision"]}
+            fernet = Fernet(
+                base64.urlsafe_b64encode(hashlib.sha256(b"BU_Bench_V2").digest())
+            )
+            artifact = base64.b64encode(fernet.encrypt(json.dumps(payload).encode()))
+            (root / "BU_Bench_V2.enc").write_bytes(artifact)
+            manifest["candidate_encrypted_sha256"] = hashlib.sha256(artifact).hexdigest()
+            (root / "rubric_revision.json").write_text(json.dumps(manifest))
+            with self.assertRaisesRegex(ValueError, "Unapproved weight revision"):
+                validate_revision(root, base_artifact=baseline)
+
+    def test_redesign_requires_declared_original_and_current_weight_hashes(self):
+        for key in ("before_weights_sha256", "after_weights_sha256"):
+            with self.subTest(key=key), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                baseline = self.copy_revision_fixture(root)
+                manifest = json.loads((root / "rubric_revision.json").read_text())
+                change = next(c for c in manifest["changes"] if c["task_id"] == "bu2-185")
+                del change[key]
+                (root / "rubric_revision.json").write_text(json.dumps(manifest))
+                with self.assertRaisesRegex(ValueError, "Unapproved weight revision"):
+                    validate_revision(root, base_artifact=baseline)
+
+    def test_redesign_metadata_cannot_change_with_matching_manifest(self):
+        for field in ("title", "summary"):
+            with self.subTest(field=field), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                baseline = self.copy_revision_fixture(root)
+                manifest, _, after, _ = validate_revision(root, base_artifact=baseline)
+                task = after["bu2-185"]
+                task[field] += " unapproved change"
+                change = next(c for c in manifest["changes"] if c["task_id"] == "bu2-185")
+                change[f"after_{field}_sha256"] = hashlib.sha256(
+                    task[field].encode()
+                ).hexdigest()
+                payload = {"tasks": list(after.values()), "revision": manifest["revision"]}
+                fernet = Fernet(
+                    base64.urlsafe_b64encode(hashlib.sha256(b"BU_Bench_V2").digest())
+                )
+                artifact = base64.b64encode(fernet.encrypt(json.dumps(payload).encode()))
+                (root / "BU_Bench_V2.enc").write_bytes(artifact)
+                manifest["candidate_encrypted_sha256"] = hashlib.sha256(artifact).hexdigest()
+                (root / "rubric_revision.json").write_text(json.dumps(manifest))
+                with self.assertRaisesRegex(ValueError, "Metadata hash mismatch"):
+                    validate_revision(root, base_artifact=baseline)
+
     def test_duplicate_task_ids_rejected_before_indexing(self):
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)