fix(browser): align coordinate guidance with screenshot mapping

Author: waleedlatif1

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -3507,7 +3507,7 @@ describe('credential protection', () => {
     const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 9999, y: 5 })
 
     expect(result.ok).toBe(false)
-    expect(result.error).toMatch(/divide image pixels by its scale/)
+    expect(result.error).toMatch(/X\/Y coordinate mapping and crop origin/)
   })
 
   it('inserts text into the focused editable at the caret', async () => {

apps/desktop/src/main/browser-agent/driver.ts

@@ -1237,7 +1237,7 @@ function unwrapPageResult(result: unknown): unknown {
     }
     if (code === 'outside-viewport') {
       throw new ToolError(
-        'That point is outside the visible viewport. Coordinates are CSS pixels within the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale, and scroll the target into view first.'
+        "That point is outside the visible viewport. Coordinates are CSS pixels within the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin, and scroll the target into view first."
       )
     }
     if (code === 'ambiguous-editable') {
@@ -4219,7 +4219,7 @@ async function executeToolInner(
       )
       if (!isRecordLike(pointTarget) || pointTarget.found !== true) {
         throw new ToolError(
-          'Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.'
+          "Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin."
         )
       }
       if (pointTarget.fileInput === true) {
@@ -4463,7 +4463,7 @@ async function executeToolInner(
         )
         if (!isRecordLike(probe) || probe.found !== true) {
           throw new ToolError(
-            `Nothing is rendered at the ${which} point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.`
+            `Nothing is rendered at the ${which} point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin.`
           )
         }
         return {

apps/sim/lib/copilot/generated/tool-catalog-v1.ts

@@ -500,12 +500,12 @@ export const BrowserClickAt: ToolCatalogEntry = {
       x: {
         type: 'number',
         description:
-          'X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by scale and add clip.x when present.',
+          "X in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's X mapping and crop origin.",
       },
       y: {
         type: 'number',
         description:
-          'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.',
+          "Y in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's Y mapping and crop origin.",
       },
     },
     required: ['x', 'y'],
@@ -1519,7 +1519,7 @@ export const BrowserScreenshot: ToolCatalogEntry = {
       elementId: {
         type: 'number',
         description:
-          "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Use the returned clip offset when converting image coordinates.",
+          "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Follow the image caption's coordinate mapping, including its crop origin.",
       },
     },
   },

apps/sim/lib/copilot/generated/tool-schemas-v1.ts

@@ -225,12 +225,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
         x: {
           type: 'number',
           description:
-            'X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by scale and add clip.x when present.',
+            "X in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's X mapping and crop origin.",
         },
         y: {
           type: 'number',
           description:
-            'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.',
+            "Y in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's Y mapping and crop origin.",
         },
       },
       required: ['x', 'y'],
@@ -1430,7 +1430,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
         elementId: {
           type: 'number',
           description:
-            "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Use the returned clip offset when converting image coordinates.",
+            "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Follow the image caption's coordinate mapping, including its crop origin.",
         },
       },
     },

apps/sim/lib/copilot/tools/client/browser-tool-result.ts

@@ -58,7 +58,7 @@ export function sanitizeBrowserToolResultForModel(
       : null
     : { x: 0, y: 0 }
   const content = [
-    `Screenshot${location}. This is the rendered ${clip ? 'element' : 'viewport'} only — it carries no element ids, so use browser_snapshot before interacting.`,
+    `Screenshot${location}. This is the rendered ${clip ? 'element' : 'viewport'} only — it carries no element ids. Use browser_snapshot for element-ref actions and the mapping below for coordinate actions.`,
     viewportSize && `Viewport: ${viewportSize.width} × ${viewportSize.height} CSS pixels.`,
     imageSize && `Encoded image: ${imageSize.width} × ${imageSize.height} pixels.`,
     hasScale && `Image scale: X=${scaleX}, Y=${scaleY} encoded image pixels per CSS pixel.`,