diff --git a/README.md b/README.md index 7aaae60..99a3fd5 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ These results use the earlier 60-task set. They are not results for the current ## BU Bench V2.1 +The findings judge excludes agent `thinking:` and `reasoning:` summaries from its input. +Saved traces retain them for debugging; tool evidence, final answers, files, screenshot +step references, and deterministic integrity checks are unchanged. Historical scores +are not recalculated automatically. + + **200 web tasks scored against weighted findings rubrics — the default task set.** Latest release: [v2.1.1](https://github.com/browser-use/benchmark/releases/latest). diff --git a/evaluation.py b/evaluation.py index 13dd43d..c6199c8 100644 --- a/evaluation.py +++ b/evaluation.py @@ -122,7 +122,7 @@ def judge_config(benchmark: str, llm) -> dict: endpoint_host = urlparse(str(endpoint)).hostname return { "type": "findings" if findings else "legacy_binary", - "adapter_version": "2.1.3" if findings else "legacy-v1", + "adapter_version": "2.1.4" if findings else "legacy-v1", "adapter_source_sha256": hashlib.sha256( Path(__file__).read_bytes() ).hexdigest(), diff --git a/findings_judge.py b/findings_judge.py index 71f2799..205e52b 100644 --- a/findings_judge.py +++ b/findings_judge.py @@ -174,8 +174,13 @@ def construct_findings_judge_messages( screenshot_timing: Literal['before', 'after'] = 'after', evidence_notes: list[str] | None = None, ) -> list[BaseMessage]: + # Keep original step IDs for screenshot references; retain raw reasoning in saved traces. + visible_steps = [ + (i, step) for i, step in enumerate(agent_steps, start=1) + if not step.lstrip().startswith(('thinking:', 'reasoning:')) + ] if screenshot_steps is None: - trajectory = '\n'.join(agent_steps) + trajectory = '\n'.join(step for _, step in visible_steps) screenshots_note = ( '{n} screenshots from execution are attached below in chronological order. ' f'They were captured {screenshot_timing} browser actions. ' @@ -183,7 +188,7 @@ def construct_findings_judge_messages( ) else: # Number the steps so screenshot labels ([step N]) can be located. - trajectory = '\n'.join(f'[step {i}] {s}' for i, s in enumerate(agent_steps, start=1)) + trajectory = '\n'.join(f'[step {i}] {step}' for i, step in visible_steps) screenshots_note = ( '{n} screenshots are attached below in chronological order. They were captured ' f'automatically by the harness immediately {screenshot_timing} browser actions (not chosen by the ' diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index a59a135..687e993 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -157,6 +157,36 @@ async def test_full_rubric_images_files_and_partial_weighted_score(self): any(isinstance(part, ContentPartImageParam) for part in content) ) + async def test_reasoning_excluded_without_changing_saved_evidence_or_image_ids(self): + for screenshot_steps in (None, [4]): + with self.subTest(screenshot_steps=screenshot_steps): + data = trace() + data["agent_steps"] = [ + "thinking: PRIVATE_DRAFT", + "text: Inspecting the page", + " reasoning: PRIVATE_SUMMARY", + "tool: observed thinking: and reasoning: on the page", + ] + original = data["agent_steps"].copy() + if screenshot_steps is not None: + data["screenshot_steps"] = screenshot_steps + llm = judge() + await judge_trace(task(), data, llm) + content = llm.ainvoke.call_args.args[0][1].content + prompt = content[0].text + self.assertNotIn("PRIVATE_DRAFT", prompt) + self.assertNotIn("PRIVATE_SUMMARY", prompt) + self.assertIn("text: Inspecting the page", prompt) + self.assertIn("tool: observed thinking: and reasoning: on the page", prompt) + self.assertIn(data["final_result"], prompt) + self.assertIn(data["output_files_text"], prompt) + self.assertTrue(any(isinstance(p, ContentPartImageParam) for p in content)) + if screenshot_steps is not None: + self.assertIn("[step 4] tool:", prompt) + self.assertIn("[step 2] text:", prompt) + self.assertIn("[step 4]", content[-2].text) + self.assertEqual(data["agent_steps"], original) + async def test_global_penalty_preserves_raw_score(self): result = await judge_trace( task(), trace(), judge(findings(reward_hacking_suspected=True))