Skip to content

Commit d6cd3a2

Browse files
authored
improvement(browser): accept observe on every page action (#9085)
Models carry observe: {} over from click and type to set_checked, select_option, click_at, insert_text and drag, and the driver refused those calls outright, costing a round each time (26 refusals across 650 benchmark episodes). The observation runs after the action exactly as it does for the other actions.
1 parent 30631ad commit d6cd3a2

2 files changed

Lines changed: 27 additions & 0 deletions

File tree

‎apps/desktop/src/main/browser-agent/post-action-observation.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,4 +94,25 @@ describe('post-action observation', () => {
9494
})
9595
}
9696
)
97+
98+
it.each([
99+
'browser_click_at',
100+
'browser_insert_text',
101+
'browser_select_option',
102+
'browser_set_checked',
103+
'browser_drag',
104+
] as const)('observes after %s instead of refusing the call', async (tool) => {
105+
await expect(
106+
withPostActionObservation(
107+
tool,
108+
{ observe: {} },
109+
async () => ({ dispatched: true }),
110+
async () => ({ outline: 'After' }),
111+
() => {}
112+
)
113+
).resolves.toMatchObject({
114+
dispatched: true,
115+
observation: { ok: true, result: { outline: 'After' } },
116+
})
117+
})
97118
})

‎apps/desktop/src/main/browser-agent/post-action-observation.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,19 @@ import { getErrorMessage } from '@sim/utils/errors'
33
import { isRecordLike } from '@sim/utils/object'
44
import { ToolError } from '@/main/browser-agent/errors'
55

6+
/** Every page action accepts observe: models carry it over from click and type to the rest. */
67
const OBSERVABLE_ACTIONS: ReadonlySet<BrowserToolName> = new Set([
78
'browser_click',
9+
'browser_click_at',
810
'browser_type',
11+
'browser_insert_text',
912
'browser_press_key',
1013
'browser_fill_form',
14+
'browser_select_option',
15+
'browser_set_checked',
1116
'browser_scroll',
1217
'browser_hover',
18+
'browser_drag',
1319
'browser_batch',
1420
])
1521

0 commit comments

Comments
 (0)