Exclude agent reasoning from findings judge input

README.md

@@ -40,6 +40,12 @@ These results use the earlier 60-task set. They are not results for the current
 
 ## BU Bench V2.1
 
+The findings judge excludes agent `thinking:` and `reasoning:` summaries from its input.
+Saved traces retain them for debugging; tool evidence, final answers, files, screenshot
+step references, and deterministic integrity checks are unchanged. Historical scores
+are not recalculated automatically.
+
+
 **200 web tasks scored against weighted findings rubrics — the default task set.**
 
 Latest release: [v2.1.1](https://github.com/browser-use/benchmark/releases/latest).

evaluation.py

@@ -122,7 +122,7 @@ def judge_config(benchmark: str, llm) -> dict:
     endpoint_host = urlparse(str(endpoint)).hostname
     return {
         "type": "findings" if findings else "legacy_binary",
-        "adapter_version": "2.1.3" if findings else "legacy-v1",
+        "adapter_version": "2.1.4" if findings else "legacy-v1",
         "adapter_source_sha256": hashlib.sha256(
             Path(__file__).read_bytes()
         ).hexdigest(),

findings_judge.py

@@ -174,16 +174,21 @@ def construct_findings_judge_messages(
 	screenshot_timing: Literal['before', 'after'] = 'after',
 	evidence_notes: list[str] | None = None,
 ) -> list[BaseMessage]:
+	# Keep original step IDs for screenshot references; retain raw reasoning in saved traces.
+	visible_steps = [
+		(i, step) for i, step in enumerate(agent_steps, start=1)
+		if not step.lstrip().startswith(('thinking:', 'reasoning:'))
+	]
 	if screenshot_steps is None:
-		trajectory = '\n'.join(agent_steps)
+		trajectory = '\n'.join(step for _, step in visible_steps)
 		screenshots_note = (
 			'{n} screenshots from execution are attached below in chronological order. '
 			f'They were captured {screenshot_timing} browser actions. '
 			'Image ordinals are not trajectory step numbers; no exact step mapping is available.'
 		)
 	else:
 		# Number the steps so screenshot labels ([step N]) can be located.
-		trajectory = '\n'.join(f'[step {i}] {s}' for i, s in enumerate(agent_steps, start=1))
+		trajectory = '\n'.join(f'[step {i}] {step}' for i, step in visible_steps)
 		screenshots_note = (
 			'{n} screenshots are attached below in chronological order. They were captured '
 			f'automatically by the harness immediately {screenshot_timing} browser actions (not chosen by the '

tests/test_evaluation.py

@@ -157,6 +157,36 @@ async def test_full_rubric_images_files_and_partial_weighted_score(self):
             any(isinstance(part, ContentPartImageParam) for part in content)
         )
 
+    async def test_reasoning_excluded_without_changing_saved_evidence_or_image_ids(self):
+        for screenshot_steps in (None, [4]):
+            with self.subTest(screenshot_steps=screenshot_steps):
+                data = trace()
+                data["agent_steps"] = [
+                    "thinking: PRIVATE_DRAFT",
+                    "text: Inspecting the page",
+                    "  reasoning: PRIVATE_SUMMARY",
+                    "tool: observed thinking: and reasoning: on the page",
+                ]
+                original = data["agent_steps"].copy()
+                if screenshot_steps is not None:
+                    data["screenshot_steps"] = screenshot_steps
+                llm = judge()
+                await judge_trace(task(), data, llm)
+                content = llm.ainvoke.call_args.args[0][1].content
+                prompt = content[0].text
+                self.assertNotIn("PRIVATE_DRAFT", prompt)
+                self.assertNotIn("PRIVATE_SUMMARY", prompt)
+                self.assertIn("text: Inspecting the page", prompt)
+                self.assertIn("tool: observed thinking: and reasoning: on the page", prompt)
+                self.assertIn(data["final_result"], prompt)
+                self.assertIn(data["output_files_text"], prompt)
+                self.assertTrue(any(isinstance(p, ContentPartImageParam) for p in content))
+                if screenshot_steps is not None:
+                    self.assertIn("[step 4] tool:", prompt)
+                    self.assertIn("[step 2] text:", prompt)
+                    self.assertIn("[step 4]", content[-2].text)
+                self.assertEqual(data["agent_steps"], original)
+
     async def test_global_penalty_preserves_raw_score(self):
         result = await judge_trace(
             task(), trace(), judge(findings(reward_hacking_suspected=True))