Address Yutori native tool review

Author: rgarcia

packages/agent/src/translator/translator.ts

@@ -102,52 +102,76 @@ type DragMouseButton = "left" | "right" | "middle";
 function toSdkAction(type: string, action: ModelAction): KernelBatchAction {
 	switch (type) {
 		case "click":
+			const clickHoldKeys = readHoldKeys(action.hold_keys);
 			return {
 				type: "click_mouse",
 				click_mouse: {
 					x: toInt(action.x),
 					y: toInt(action.y),
 					button: clickMouseButtonOr(action.button, "left"),
+					...(clickHoldKeys.length > 0 ? { hold_keys: clickHoldKeys } : {}),
 				},
 			};
 		case "double_click":
+			const doubleClickHoldKeys = readHoldKeys(action.hold_keys);
 			return {
 				type: "click_mouse",
 				click_mouse: {
 					x: toInt(action.x),
 					y: toInt(action.y),
 					num_clicks: 2,
+					...(doubleClickHoldKeys.length > 0 ? { hold_keys: doubleClickHoldKeys } : {}),
 				},
 			};
 		case "mouse_down":
 		case "mouse_up":
+			const mouseHoldKeys = readHoldKeys(action.hold_keys);
 			return {
 				type: "click_mouse",
 				click_mouse: {
 					x: toInt(action.x),
 					y: toInt(action.y),
 					button: clickMouseButtonOr(action.button, "left"),
 					click_type: type === "mouse_down" ? "down" : "up",
+					...(mouseHoldKeys.length > 0 ? { hold_keys: mouseHoldKeys } : {}),
 				},
 			};
 		case "type":
 			return { type: "type_text", type_text: { text: typeof action.text === "string" ? action.text : "" } };
 		case "keypress":
-			return keypress(toStringArray(action.keys));
+			return keypress(toStringArray(action.keys), action.duration);
 		case "scroll":
+			const scrollHoldKeys = readHoldKeys(action.hold_keys);
 			return {
 				type: "scroll",
 				scroll: {
 					x: toInt(action.x),
 					y: toInt(action.y),
 					delta_x: toInt(action.scroll_x),
 					delta_y: toInt(action.scroll_y),
+					...(scrollHoldKeys.length > 0 ? { hold_keys: scrollHoldKeys } : {}),
 				},
 			};
 		case "move":
-			return { type: "move_mouse", move_mouse: { x: toInt(action.x), y: toInt(action.y) } };
+			const moveHoldKeys = readHoldKeys(action.hold_keys);
+			return {
+				type: "move_mouse",
+				move_mouse: {
+					x: toInt(action.x),
+					y: toInt(action.y),
+					...(moveHoldKeys.length > 0 ? { hold_keys: moveHoldKeys } : {}),
+				},
+			};
 		case "drag":
-			return { type: "drag_mouse", drag_mouse: { path: toPath(action.path), button: dragMouseButtonOr(action.button, "left") } };
+			const dragHoldKeys = readHoldKeys(action.hold_keys);
+			return {
+				type: "drag_mouse",
+				drag_mouse: {
+					path: toPath(action.path),
+					button: dragMouseButtonOr(action.button, "left"),
+					...(dragHoldKeys.length > 0 ? { hold_keys: dragHoldKeys } : {}),
+				},
+			};
 		case "wait":
 			return { type: "sleep", sleep: { duration_ms: typeof action.ms === "number" ? Math.trunc(action.ms) : 1000 } };
 		default:
@@ -188,7 +212,11 @@ function toStringArray(value: unknown): string[] {
 	return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
 }
 
-function keypress(keys: string[]): KernelBatchAction {
+function readHoldKeys(value: unknown): string[] {
+	return toStringArray(value).flatMap((key) => translateKeys([key]));
+}
+
+function keypress(keys: string[], duration: unknown = undefined): KernelBatchAction {
 	const translated = translateKeys(keys);
 	const pressedKeys = translated.filter((key) => !isModifierKey(key));
 	const holdKeys = pressedKeys.length > 0 ? translated.filter(isModifierKey) : translated.slice(0, -1);
@@ -197,6 +225,7 @@ function keypress(keys: string[]): KernelBatchAction {
 		press_key: {
 			keys: pressedKeys.length > 0 ? pressedKeys : translated.slice(-1),
 			...(holdKeys.length > 0 ? { hold_keys: holdKeys } : {}),
+			...(typeof duration === "number" && Number.isFinite(duration) && duration > 0 ? { duration: Math.trunc(duration) } : {}),
 		},
 	};
 }
@@ -214,6 +243,7 @@ const KEY_ALIASES: Record<string, string> = {
 	shiftleft: "Shift_L",
 	meta: "Super_L",
 	super: "Super_L",
+	super_l: "Super_L",
 	cmd: "Super_L",
 	command: "Super_L",
 	enter: "Return",

packages/agent/test/translator.test.ts

@@ -56,4 +56,23 @@ describe("InternalComputerTranslator", () => {
 			[{ type: "press_key", press_key: { keys: ["l"], hold_keys: ["Control_L"] } }],
 		]);
 	});
+
+	it("passes canonical modifier and key duration fields through to Kernel actions", async () => {
+		const { batches, client } = createClient();
+		const translator = new InternalComputerTranslator({ browser, client });
+
+		await translator.executeBatch([
+			{ type: "click", x: 10, y: 20, hold_keys: ["Control_L"] },
+			{ type: "scroll", x: 10, y: 20, scroll_y: 120, hold_keys: ["Shift_L"] },
+			{ type: "keypress", keys: ["Shift_L"], duration: 1500 },
+		]);
+
+		expect(batches).toEqual([
+			[
+				{ type: "click_mouse", click_mouse: { x: 10, y: 20, button: "left", hold_keys: ["Control_L"] } },
+				{ type: "scroll", scroll: { x: 10, y: 20, delta_x: 0, delta_y: 120, hold_keys: ["Shift_L"] } },
+				{ type: "press_key", press_key: { keys: ["Shift_L"], duration: 1500 } },
+			],
+		]);
+	});
 });

packages/ai/src/providers/common.ts

@@ -133,6 +133,83 @@ export type CuaAction =
 	| CuaActionUrl
 	| CuaActionCursorPosition;
 
+export const CUA_MODIFIER_KEYSYMS = ["Control_L", "Alt_L", "Shift_L", "Super_L"] as const;
+
+export type CuaModifierKeysym = (typeof CUA_MODIFIER_KEYSYMS)[number];
+
+const CUA_KEY_ALIASES: Record<string, string> = {
+	alt: "Alt_L",
+	alt_l: "Alt_L",
+	altleft: "Alt_L",
+	backspace: "BackSpace",
+	cmd: "Super_L",
+	command: "Super_L",
+	control: "Control_L",
+	control_l: "Control_L",
+	controlleft: "Control_L",
+	ctrl: "Control_L",
+	delete: "Delete",
+	down: "Down",
+	end: "End",
+	enter: "Return",
+	esc: "Escape",
+	escape: "Escape",
+	home: "Home",
+	left: "Left",
+	meta: "Super_L",
+	option: "Alt_L",
+	pagedown: "Next",
+	page_down: "Next",
+	pageup: "Prior",
+	page_up: "Prior",
+	return: "Return",
+	right: "Right",
+	shift: "Shift_L",
+	shift_l: "Shift_L",
+	shiftleft: "Shift_L",
+	space: "space",
+	super: "Super_L",
+	tab: "Tab",
+	up: "Up",
+};
+
+const CUA_MODIFIER_KEYSYM_SET = new Set<string>(CUA_MODIFIER_KEYSYMS);
+
+/**
+ * Normalize provider-emitted key aliases to Kernel's xdotool/X11 keysyms.
+ * Unknown values pass through so provider-specific or printable keys can still
+ * be executed when the Kernel computer API already understands them.
+ */
+export function normalizeCuaKey(value: string): string {
+	const trimmed = value.trim();
+	const lookup = trimmed.replace(/[-\s]/g, "_").toLowerCase();
+	const alias = CUA_KEY_ALIASES[lookup];
+	if (alias) return alias;
+	if (/^f\d{1,2}$/i.test(trimmed)) return trimmed.toUpperCase();
+	if (/^arrow/i.test(trimmed)) return normalizeCuaKey(trimmed.slice("arrow".length));
+	if (trimmed.length === 1 && trimmed >= "A" && trimmed <= "Z") return trimmed.toLowerCase();
+	return trimmed;
+}
+
+export function normalizeCuaModifierKey(value: string): CuaModifierKeysym | undefined {
+	const key = normalizeCuaKey(value);
+	return CUA_MODIFIER_KEYSYM_SET.has(key) ? (key as CuaModifierKeysym) : undefined;
+}
+
+export function normalizeCuaKeyCombo(value: string): string[] {
+	return value
+		.split("+")
+		.map((part) => normalizeCuaKey(part))
+		.filter(Boolean);
+}
+
+export function normalizeCuaKeySequence(value: string): string[][] {
+	return value
+		.split(/\s+/)
+		.map((part) => normalizeCuaKeyCombo(part))
+		.filter((combo) => combo.length > 0);
+}
+
 const PointSchema = Type.Object(
 	{
 		x: Type.Number(),

packages/ai/src/providers/yutori/actions.ts

@@ -2,20 +2,40 @@ import {
 	CUA_ACTION_TYPES,
 	CUA_BATCH_TOOL_NAME,
 	CUA_NAVIGATION_TOOL_NAME,
+	normalizeCuaKeyCombo,
+	normalizeCuaKeySequence,
+	normalizeCuaModifierKey,
 	type CuaAction,
 	type CuaActionType,
 } from "../common";
 
+/**
+ * Native Yutori Navigator n1.5 tool-set ids.
+ *
+ * Source of truth:
+ * - https://docs.yutori.com/reference/n1-5
+ * - https://docs.yutori.com/llm-quickstart.md
+ */
 export const YUTORI_N15_CORE_TOOL_SET = "browser_tools_core-20260403";
 export const YUTORI_N15_EXPANDED_TOOL_SET = "browser_tools_expanded-20260403";
 
+/**
+ * DOM/ref-backed Navigator n1.5 actions. We intentionally disable these until
+ * CuaAgent has the ref/DOM execution path that Yutori documents for the
+ * expanded tool set.
+ */
 export const YUTORI_N15_EXPANDED_ACTION_TYPES = [
 	"extract_elements",
 	"find",
 	"set_element_value",
 	"execute_js",
 ] as const;
 
+/**
+ * Navigator n1's fixed legacy browser action space.
+ *
+ * Source of truth: https://docs.yutori.com/reference/n1
+ */
 export const YUTORI_N1_ACTION_TYPES = [
 	"left_click",
 	"double_click",
@@ -32,6 +52,13 @@ export const YUTORI_N1_ACTION_TYPES = [
 	"wait",
 ] as const;
 
+/**
+ * Navigator n1.5 core visual action space. These are the actions available
+ * when `tool_set` is `browser_tools_core-20260403`, which keeps CuaAgent in the
+ * pure screenshot/coordinate path and avoids DOM refs.
+ *
+ * Source of truth: https://docs.yutori.com/reference/n1-5
+ */
 export const YUTORI_N15_CORE_ACTION_TYPES = [
 	"left_click",
 	"double_click",
@@ -74,17 +101,11 @@ const DEFAULT_WAIT_MS = 2000;
 const NAVIGATION_WAIT_MS = 1500;
 const GOTO_WAIT_MS = 2000;
 
-const MODIFIER_MAP: Record<string, string> = {
-	alt: "alt",
-	command: "super",
-	control: "ctrl",
-	ctrl: "ctrl",
-	cmd: "super",
-	meta: "super",
-	shift: "shift",
-	super: "super",
-};
-
+/**
+ * Yutori n1.5 exposes built-in browser tools via `tool_set`, not by accepting
+ * client-supplied JSON tool definitions. Keeping this empty prevents us from
+ * sending duplicate CUA batch/browser tools alongside Yutori's native tools.
+ */
 export function createComputerToolDefinitions(_options?: unknown): [] {
 	return [];
 }
@@ -195,34 +216,36 @@ function toTypeActions(args: Record<string, unknown>): CuaAction[] | undefined {
 	if (text === undefined) return undefined;
 	const actions: CuaAction[] = [];
 	if (args.clear_before_typing === true) {
-		actions.push({ type: "keypress", keys: ["Control", "a"] }, { type: "keypress", keys: ["Backspace"] });
+		actions.push({ type: "keypress", keys: ["Control_L", "a"] }, { type: "keypress", keys: ["BackSpace"] });
 	}
 	actions.push({ type: "type", text });
-	if (args.press_enter_after === true) actions.push({ type: "keypress", keys: ["Enter"] });
+	if (args.press_enter_after === true) actions.push({ type: "keypress", keys: ["Return"] });
 	return actions;
 }
 
 function toKeypressAction(args: Record<string, unknown>): CuaAction[] | undefined {
-	const keys = readKeys(args.key_comb ?? args.key);
-	return keys.length > 0 ? [{ type: "keypress", keys }] : undefined;
+	const sequence = readKeySequence(args.key_comb ?? args.key);
+	return sequence.length > 0 ? sequence.map((keys) => ({ type: "keypress", keys })) : undefined;
 }
 
 function toHoldKeyAction(args: Record<string, unknown>): CuaAction[] | undefined {
-	const keys = readKeys(args.key_comb ?? args.key);
+	const keys = readKeyCombo(args.key_comb ?? args.key);
 	return keys.length > 0 ? [{ type: "keypress", keys, duration: secondsToMs(args.duration, 1000) }] : undefined;
 }
 
-function readKeys(value: unknown): string[] {
+function readKeyCombo(value: unknown): string[] {
+	if (typeof value !== "string") return [];
+	return normalizeCuaKeyCombo(value);
+}
+
+function readKeySequence(value: unknown): string[][] {
 	if (typeof value !== "string") return [];
-	return value
-		.split("+")
-		.map((part) => part.trim())
-		.filter(Boolean);
+	return normalizeCuaKeySequence(value);
 }
 
 function holdKeys(value: unknown): { hold_keys?: string[] } {
 	if (typeof value !== "string") return {};
-	const key = MODIFIER_MAP[value.trim().toLowerCase()];
+	const key = normalizeCuaModifierKey(value);
 	return key ? { hold_keys: [key] } : {};
 }
 

packages/ai/src/providers/yutori/index.ts

@@ -33,12 +33,15 @@ export {
 //   n1.5 expanded (browser_tools_expanded-20260403): core +
 //     extract_elements, find, set_element_value, execute_js
 // Sources:
-//   https://github.com/yutori-ai/yutori-sdk-python/blob/main/api.md
-//   https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/models.py
+//   https://docs.yutori.com/reference/n1
 //   https://docs.yutori.com/reference/n1-5
+//   https://docs.yutori.com/llm-quickstart.md
 //   https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/coordinates.py
 export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 1000] } as const satisfies ComputerToolCoordinateSystem;
 
+// Yutori's Navigator quickstart recommends putting extra instructions in the
+// first user message instead of supplying a custom system prompt.
+// Source: https://docs.yutori.com/llm-quickstart.md
 export const YUTORI_INSTRUCTIONS_RAW = "";
 
 export function buildYutoriSystemPrompt(opts: { suffix?: string } = {}): string {

packages/ai/test/common-keys.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+import {
+	normalizeCuaKey,
+	normalizeCuaKeyCombo,
+	normalizeCuaKeySequence,
+	normalizeCuaModifierKey,
+} from "../src/index";
+
+describe("canonical CUA key normalization", () => {
+	it("normalizes common provider key names to X11 keysyms", () => {
+		expect(normalizeCuaModifierKey("ctrl")).toBe("Control_L");
+		expect(normalizeCuaModifierKey("command")).toBe("Super_L");
+		expect(normalizeCuaKey("Backspace")).toBe("BackSpace");
+		expect(normalizeCuaKey("ArrowLeft")).toBe("Left");
+		expect(normalizeCuaKey("enter")).toBe("Return");
+		expect(normalizeCuaKey("F5")).toBe("F5");
+	});
+
+	it("normalizes key combos and sequential key expressions separately", () => {
+		expect(normalizeCuaKeyCombo("ctrl+shift+Tab")).toEqual(["Control_L", "Shift_L", "Tab"]);
+		expect(normalizeCuaKeySequence("down down enter")).toEqual([["Down"], ["Down"], ["Return"]]);
+	});
+});