diff --git a/apps/hub/src/inference/code-mode.test.ts b/apps/hub/src/inference/code-mode.test.ts new file mode 100644 index 00000000..f4d05b07 --- /dev/null +++ b/apps/hub/src/inference/code-mode.test.ts @@ -0,0 +1,577 @@ +import { expect, test } from "bun:test"; +import type { + ConversationTurn, + LastCycleSource, + ToolDefinition, +} from "@intx/types/runtime"; +import { createCodexResponsesAdapter } from "./codex-adapter.ts"; +import { scriptCall } from "./code-mode.ts"; + +const source: LastCycleSource = { + sourceId: "codex-default", + provider: "codex", + model: "gpt-6.1-terra", +}; +const quirks = { + productName: "Corbits Builder", + environmentTagName: "builder_environment", +}; +const readArtifact = { + name: "read_artifact", + description: "See which project documents exist, or read one.", + inputSchema: { + type: "object", + properties: { + kind: { type: "string", description: "The document kind." }, + part: { type: "integer" }, + }, + required: ["kind"], + }, +}; +const tools = [ + readArtifact, + { + name: "ask_questions", + description: "Ask the person questions.", + inputSchema: { + type: "object", + properties: { + questions: { + type: "array", + items: { + type: "object", + properties: { + id: { type: "string" }, + kind: { enum: ["single", "multi"] }, + }, + required: ["id", "kind"], + }, + }, + }, + required: ["questions"], + }, + }, +]; +const user: ConversationTurn = { + role: "user", + content: [{ type: "text", text: "Are you there?" }], + timestamp: 0, +}; + +function build( + messages: ConversationTurn[] = [user], + requestTools: ToolDefinition[] = tools, +) { + const adapter = createCodexResponsesAdapter(source, quirks); + const request = adapter.buildRequest(messages, "gpt-6.1-terra", { + tools: requestTools, + }); + return { adapter, body: JSON.parse(request.body) }; +} + +function done(input: string) { + return JSON.stringify({ + type: "response.output_item.done", + sequence_number: 4, + output_index: 1, + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "exec", + input, + }, + }); +} + +function toolCalls(events: { type: string; data: unknown }[]) { + const calls: { callId: unknown; name: unknown; args: string }[] = []; + for (const event of events) { + const data = event.data as Record; + if (event.type === "inference.tool_call.start") + calls.push({ callId: data["callId"], name: data["name"], args: "" }); + if (event.type === "inference.tool_call.delta") { + const last = calls.at(-1); + if (last) last.args += String(data["argumentFragment"]); + } + } + return calls.map(({ args, ...call }) => ({ + ...call, + args: JSON.parse(args), + })); +} + +test("offers the tools as one exec code tool with a typed declaration for each", () => { + const { body } = build(); + expect(body.tools).toHaveLength(1); + const [exec] = body.tools; + expect(exec).toMatchObject({ + type: "custom", + name: "exec", + format: { type: "grammar", syntax: "lark" }, + }); + expect(exec.description).toContain( + " read_artifact(args: {\n // The document kind.\n kind: string;\n part?: number;\n }): Promise;", + ); + expect(exec.description).toContain( + 'questions: Array<{\n id: string;\n kind: "single" | "multi";\n }>;', + ); + expect(body.parallel_tool_calls).toBe(true); +}); + +test("replays past tool calls as exec scripts and their outputs", () => { + const { body } = build([ + user, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call_9", + name: "read_artifact", + arguments: { kind: "problem_brief" }, + }, + ], + timestamp: 1, + }, + { + role: "user", + content: [ + { + type: "tool_result", + callId: "call_9", + content: [{ type: "text", text: '{"version":1}' }], + }, + ], + timestamp: 2, + }, + ]); + expect(body.input).toContainEqual({ + type: "custom_tool_call", + call_id: "call_9", + name: "exec", + input: + 'const result = await tools.read_artifact({"kind":"problem_brief"});\ntext(result);\n', + }); + expect(body.input).toContainEqual({ + type: "custom_tool_call_output", + call_id: "call_9", + output: 'Script completed\nOutput:\n{"version":1}', + }); +}); + +test("turns the tool call in a script into an ordinary tool call", () => { + const { adapter } = build(); + const events = adapter.parseResponse( + done( + "// Ask about access\nconst q = await tools.ask_questions({questions: [{id: \"access\", kind: 'single',}]});\ntext(JSON.stringify(q));\n", + ), + ); + expect(toolCalls(events)).toEqual([ + { + callId: "call_1", + name: "ask_questions", + args: { questions: [{ id: "access", kind: "single" }] }, + }, + ]); +}); + +test("replays a call to a tool no longer offered under its decoded alias", () => { + const { body } = build([ + user, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call_9", + name: "exa.web_search_exa", + arguments: { query: "bun" }, + }, + ], + timestamp: 1, + }, + ]); + expect(body.input).toContainEqual( + expect.objectContaining({ + call_id: "call_9", + input: expect.stringContaining("await tools.exa_web_search_exa({"), + }), + ); +}); + +test("replays a failed tool call as a failed script", () => { + const { body } = build([ + user, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call_9", + name: "read_artifact", + arguments: { kind: "nope" }, + }, + ], + timestamp: 1, + }, + { + role: "user", + content: [ + { + type: "tool_result", + callId: "call_9", + content: [{ type: "text", text: "no such document" }], + isError: true, + }, + ], + timestamp: 2, + }, + ]); + expect(body.input).toContainEqual({ + type: "custom_tool_call_output", + call_id: "call_9", + output: "Script failed\nError:\nno such document", + }); +}); + +test("emits one tool call across the whole Codex event sequence", () => { + const { adapter } = build(); + const item = { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "exec", + input: "await tools.read_artifact({kind: 'a'});", + }; + const events = [ + { type: "response.created", response: { id: "resp_1" } }, + { + type: "response.output_item.added", + output_index: 0, + item: { ...item, input: "" }, + }, + { + type: "response.custom_tool_call_input.delta", + item_id: "ctc_1", + delta: "await", + }, + { + type: "response.custom_tool_call_input.done", + item_id: "ctc_1", + input: item.input, + }, + { type: "response.output_item.done", output_index: 0, item }, + { + type: "response.completed", + response: { + id: "resp_1", + status: "completed", + output: [item], + usage: { input_tokens: 1, output_tokens: 1 }, + }, + }, + ].flatMap((event) => adapter.parseResponse(JSON.stringify(event))); + expect(toolCalls(events)).toEqual([ + { callId: "call_1", name: "read_artifact", args: { kind: "a" } }, + ]); +}); + +test("turns parallel scripts in one response into separate tool calls", () => { + const { adapter } = build(); + const second = JSON.parse(done("await tools.read_artifact({kind: 'b'});")); + second.output_index = 2; + second.item = { ...second.item, id: "ctc_2", call_id: "call_2" }; + const events = [ + ...adapter.parseResponse(done("await tools.read_artifact({kind: 'a'});")), + ...adapter.parseResponse(JSON.stringify(second)), + ]; + expect(toolCalls(events)).toEqual([ + { callId: "call_1", name: "read_artifact", args: { kind: "a" } }, + { callId: "call_2", name: "read_artifact", args: { kind: "b" } }, + ]); +}); + +test("waits for the finished script before emitting tool calls", () => { + const { adapter } = build(); + const added = JSON.stringify({ + type: "response.output_item.added", + sequence_number: 2, + output_index: 1, + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "exec", + input: "", + }, + }); + expect(adapter.parseResponse(added)).toEqual([]); +}); + +test("sends a script it cannot read as one exec call, answered with how to call tools", () => { + const { adapter } = build(); + const script = "text(ALL_TOOLS.map((t) => t.name));\n"; + expect(toolCalls(adapter.parseResponse(done(script)))).toEqual([ + { callId: "call_1", name: "exec", args: { script } }, + ]); + + const { body } = build([ + user, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call_1", + name: "exec", + arguments: { script }, + }, + ], + timestamp: 1, + }, + { + role: "user", + content: [ + { + type: "tool_result", + callId: "call_1", + content: [{ type: "text", text: "unknown tool: exec" }], + isError: true, + }, + ], + timestamp: 2, + }, + ]); + expect(body.input).toContainEqual({ + type: "custom_tool_call", + call_id: "call_1", + name: "exec", + input: script, + }); + const output = body.input.find( + (item: { type: string }) => item.type === "custom_tool_call_output", + ); + expect(output.output).toContain("await tools.({...})"); +}); + +test("reads a script only when it is one call to a known tool with a literal argument", () => { + const names = new Map([ + ["read_artifact", "read_artifact"], + ["write_file", "write_file"], + ]); + expect( + scriptCall( + '/* tools.write_file({}) */\nconst result = await tools.read_artifact({kind: "a (b); c", // note\n part: 1});\ntext(result)', + names, + ), + ).toEqual({ + name: "read_artifact", + arguments: { kind: "a (b); c", part: 1 }, + }); + expect( + scriptCall( + "await tools.write_file({path: 'a.ts', content: `one\n\"two\" \\`three\\``});", + names, + ), + ).toEqual({ + name: "write_file", + arguments: { path: "a.ts", content: 'one\n"two" `three`' }, + }); + expect( + scriptCall( + "await tools.write_file({content: `const x = \\`hi \\${name}\\`; \\u{1F600}`});", + names, + ), + ).toEqual({ + name: "write_file", + arguments: { content: "const x = `hi ${name}`; \u{1F600}" }, + }); + expect( + scriptCall("await tools.write_file({content: `a\r\nb\rc\\\r\nd`});", names), + ).toEqual({ name: "write_file", arguments: { content: "a\nb\ncd" } }); + expect( + scriptCall( + "await tools.write_file({path: '\\u{1F600}\\u{27}\\\\u{41}', content: \"\\u{22}\"});", + names, + ), + ).toEqual({ + name: "write_file", + arguments: { path: "\u{1F600}'\\u{41}", content: '"' }, + }); + expect( + scriptCall("await tools.write_file({path: '\\u{110000}'});", names), + ).toBeUndefined(); + const accepted = [ + "const r = await tools.read_artifact({});\ntext(r.content);", + "const r = await tools.read_artifact({});\nconsole.log(r?.items[0]);", + "await tools.read_artifact({});\ntext('done; tools.write_file({})');", + "const r =\n await tools.read_artifact({});\ntext(JSON.stringify(r, null, 2));", + ]; + for (const script of accepted) + expect(scriptCall(script, names)).toEqual({ + name: "read_artifact", + arguments: {}, + }); + const refused = [ + "await tools.delete_everything({})", + "await tools.read_artifact({kind: x})", + "await tools.read_artifact({kind: (() => 1)()})", + "await tools.write_file({content: `${x}`})", + "await mytools.read_artifact({})", + "await globalThis.tools.read_artifact({})", + "await tools.read_artifact({}); await tools.write_file({});", + 'if (false) { await tools.write_file({path: "a"}); }', + "try { await tools.read_artifact({}); } catch { await tools.write_file({}); }", + "for (let i = 0; i < 3; i++) await tools.read_artifact({});", + 'const s = "x".replace(/\'/g, ""); await tools.read_artifact({});', + "text(await tools.read_artifact({}))", + 'await tools.read_artifact({});\ntext(await tools["write_file"]({path: "x"}));', + 'await tools.read_artifact({});\ntext(await write_file({path: "x"}));', + 'await tools.read_artifact({});\nconsole.log(await tools?.write_file({path: "x"}));', + "await tools.read_artifact({});\ntext(/'/.source); await tools .write_file({path: \"x\"}); text('')", + "await read_artifact({})", + "const r = await tools.read_artifact({});\ntext(r.map((x) => x));", + "await tools.read_artifact({});\ntext(`${await tools.write_file({})}`);", + "text(ALL_TOOLS)", + ]; + for (const script of refused) + expect(scriptCall(script, names)).toBeUndefined(); +}); + +test("reads a large script in linear time", () => { + const slide = "{title: 'Slide', points: [1, 2, 3], layout: {columns: 2}},\n"; + const script = `await tools.write_deck({slides: [\n${slide.repeat(8000)}]});`; + const started = performance.now(); + expect( + scriptCall(script, new Map([["write_deck", "write_deck"]])), + ).toBeDefined(); + expect(performance.now() - started).toBeLessThan(500); +}); + +test("turns code tool calls in a buffered response into tool calls", () => { + const { adapter } = build(); + const events = adapter.parseJSONResponse( + JSON.stringify({ + status: "completed", + output: [ + { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "exec", + input: "await tools.read_artifact({kind: 'problem_brief'});", + }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + ); + expect(toolCalls(events)).toEqual([ + { + callId: "call_1", + name: "read_artifact", + args: { kind: "problem_brief" }, + }, + ]); +}); + +test("offers tools with dotted names under an identifier alias", () => { + const search = { + name: "exa.web_search_exa", + description: "Search the web.", + inputSchema: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }; + const { adapter, body } = build([user], [readArtifact, search]); + expect(body.tools).toHaveLength(1); + expect(body.tools[0].description).toContain(" exa_web_search_exa(args: {"); + const events = adapter.parseResponse( + done('await tools.exa_web_search_exa({query: "bun"});'), + ); + expect(toolCalls(events)).toEqual([ + { callId: "call_1", name: "exa.web_search_exa", args: { query: "bun" } }, + ]); +}); + +test("gives colliding names the same alias whatever the tool order", () => { + const tool = (name: string) => ({ ...readArtifact, name }); + const dotted = tool("server.get_issue"); + const plain = tool("server_get_issue"); + for (const order of [ + [dotted, plain], + [plain, dotted], + ]) { + const { adapter } = build([user], order); + const events = adapter.parseResponse( + done("await tools.server_get_issue_2({kind: 'a'});"), + ); + expect(toolCalls(events)).toEqual([ + { callId: "call_1", name: "server_get_issue", args: { kind: "a" } }, + ]); + } +}); + +test("declares referenced definitions, nullable types, maps and constraints", () => { + const writeDeck = { + name: "write_deck", + description: "Write the deck.", + inputSchema: { + type: "object", + properties: { + deck: { $ref: "#/$defs/deck" }, + "content-type": { type: ["string", "null"], maxLength: 40 }, + origin: { + type: "array", + prefixItems: [{ $ref: "#/$defs/hex" }, { type: "number" }], + }, + target: { + type: "object", + properties: { id: { type: "string" }, path: { type: "string" } }, + anyOf: [{ required: ["id"] }, { required: ["path"] }], + }, + style: { + type: "object", + properties: { name: { type: "string" } }, + anyOf: [{ $ref: "#/$defs/hex" }], + }, + }, + required: ["deck"], + $defs: { + deck: { + type: "object", + properties: { + slides: { + type: "array", + items: { type: "string", maxLength: 80 }, + maxItems: 5, + }, + }, + additionalProperties: { type: "number" }, + }, + hex: { type: "string", pattern: "^#[0-9a-f]{6}$" }, + }, + }, + }; + const { body } = build([user], [writeDeck]); + const { description } = body.tools[0]; + expect(description).toContain( + "type write_deck_deck = {\n // maxItems 5 — each item: maxLength 80\n slides?: Array;\n [key: string]: number;\n};", + ); + expect(description).toContain(" deck: write_deck_deck;"); + expect(description).toContain( + ' // maxLength 40\n "content-type"?: string | null;', + ); + expect(description).toContain( + '// pattern "^#[0-9a-f]{6}$"\ntype write_deck_hex = string;', + ); + expect(description).toContain("origin?: [write_deck_hex, number];"); + expect(description).toContain( + "style?: ({\n name?: string;\n }) & (write_deck_hex);", + ); + expect(description).toContain( + "target?: {\n id: string;\n path?: string;\n } | {\n id?: string;\n path: string;\n };", + ); +}); diff --git a/apps/hub/src/inference/code-mode.ts b/apps/hub/src/inference/code-mode.ts new file mode 100644 index 00000000..8b232c32 --- /dev/null +++ b/apps/hub/src/inference/code-mode.ts @@ -0,0 +1,651 @@ +import { + decodeToolName, + ProtocolMismatchError, + type AdapterFactory, +} from "@intx/inference"; +import type { ConversationTurn } from "@intx/types/runtime"; + +// Codex models are trained to reach tools through one `exec` tool that runs JavaScript, and given +// plain function tools they repeat calls and write placeholder arguments. The model sees `exec`; +// the one `tools.(...)` call in its script returns to Interchange as an ordinary tool call, so +// the reactor still gates, audits and loop-checks it. Nothing the model writes is executed. + +const EXEC = "exec"; + +// The grammar Codex sends with its `exec` tool. +const EXEC_GRAMMAR = String.raw` +start: pragma_source | plain_source +pragma_source: PRAGMA_LINE NEWLINE SOURCE +plain_source: SOURCE + +PRAGMA_LINE: /[ \t]*\/\/ @exec:[^\r\n]*/ +NEWLINE: /\r?\n/ +SOURCE: /[\s\S]+/ +`; + +const UNSUPPORTED_SCRIPT = + "This script was not run. A script makes exactly one tool call, `const result = await tools.({...});`, optionally followed by `text(result)`. The argument is a literal object of strings (backtick strings without `${...}` interpolation), numbers, booleans, null, arrays and objects. Conditions, loops, try/catch and variables are not run. To make several independent calls, send several exec calls at once."; + +const IDENTIFIER = /^[A-Za-z_$][\w$]*$/; + +// Schema keywords TypeScript cannot express, shown to the model as comments. +const CONSTRAINTS = [ + "minItems", + "maxItems", + "uniqueItems", + "minLength", + "maxLength", + "pattern", + "format", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "default", +]; + +type ToolCall = { name: string; arguments: Record }; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +function parseJSON(text: string): unknown { + try { + return JSON.parse(text); + } catch (error) { + if (error instanceof SyntaxError) return undefined; + throw error; + } +} + +const identifier = (name: string) => { + const safe = name.replace(/[^\w$]/g, "_"); + return /^\d/.test(safe) ? `_${safe}` : safe; +}; + +// Definitions a schema's `$ref`s point at, rendered as type aliases named `_`. +type Definitions = { prefix: string; names: ReadonlySet }; + +// Every definition in the schema, including those of a schema embedded whole as a property. +function definitions( + schema: unknown, + found = new Map(), +): Map { + if (Array.isArray(schema)) + for (const member of schema) definitions(member, found); + if (!isRecord(schema)) return found; + for (const [key, value] of Object.entries(schema)) { + if ((key === "$defs" || key === "definitions") && isRecord(value)) + for (const [name, def] of Object.entries(value)) found.set(name, def); + definitions(value, found); + } + return found; +} + +// The description and constraints of a schema, and of the items of an array schema. +function note(schema: unknown): string { + if (!isRecord(schema)) return ""; + const limits = CONSTRAINTS.filter((key) => key in schema).map( + (key) => `${key} ${JSON.stringify(schema[key])}`, + ); + const description = + typeof schema["description"] === "string" + ? schema["description"].replace(/\s*\n\s*/g, " ") + : ""; + const items = note(schema["items"]); + return [description, limits.join(", "), items && `each item: ${items}`] + .filter(Boolean) + .join(" — "); +} + +const without = (schema: Record, keys: string[]) => + Object.fromEntries( + Object.entries(schema).filter(([key]) => !keys.includes(key)), + ); + +// A union member read together with the object schema around it, whose properties and required +// fields it extends. +function merged(base: Record, member: unknown): unknown { + if (!isRecord(member) || !isRecord(base["properties"])) return member; + if ("$ref" in member) return { allOf: [base, member] }; + return { + ...base, + ...member, + properties: { + ...base["properties"], + ...(isRecord(member["properties"]) ? member["properties"] : undefined), + }, + required: [base["required"], member["required"]] + .filter(Array.isArray) + .flat(), + }; +} + +function typeScript( + schema: unknown, + indent: string, + defs: Definitions, +): string { + if (!isRecord(schema)) return "unknown"; + const ref = schema["$ref"]; + if (typeof ref === "string") { + const name = /^#\/(?:\$defs|definitions)\/(.+)$/.exec(ref)?.[1]; + return name !== undefined && defs.names.has(name) + ? `${defs.prefix}_${identifier(name)}` + : "unknown"; + } + if ("const" in schema) return JSON.stringify(schema["const"]); + const { enum: values, anyOf, oneOf, allOf, items, properties } = schema; + if (Array.isArray(values)) + return values.map((value) => JSON.stringify(value)).join(" | "); + const union = Array.isArray(anyOf) ? anyOf : oneOf; + if (Array.isArray(union)) { + const base = without(schema, ["anyOf", "oneOf"]); + return union + .map((member) => typeScript(merged(base, member), indent, defs)) + .join(" | "); + } + if (Array.isArray(allOf)) { + const base = without(schema, ["allOf"]); + const parts = isRecord(properties) ? [base, ...allOf] : allOf; + return parts + .map((member) => `(${typeScript(member, indent, defs)})`) + .join(" & "); + } + const kinds = Array.isArray(schema["type"]) + ? schema["type"] + : [schema["type"]]; + if (kinds.length > 1) + return kinds + .map((kind) => typeScript({ ...schema, type: kind }, indent, defs)) + .join(" | "); + const [kind] = kinds; + const tuple = schema["prefixItems"]; + if (kind === "array" && Array.isArray(tuple)) + return `[${tuple.map((member) => typeScript(member, indent, defs)).join(", ")}]`; + if (kind === "array") return `Array<${typeScript(items, indent, defs)}>`; + if (kind === "integer" || kind === "number") return "number"; + if (kind === "string" || kind === "boolean" || kind === "null") return kind; + const rest = schema["additionalProperties"]; + if (kind !== "object" && !isRecord(properties)) return "unknown"; + if (!isRecord(properties)) + return `Record`; + const required = new Set( + Array.isArray(schema["required"]) ? schema["required"] : [], + ); + const inner = `${indent} `; + const fields = Object.entries(properties).map(([key, value]) => { + const comment = note(value); + const name = IDENTIFIER.test(key) ? key : JSON.stringify(key); + const optional = required.has(key) ? "" : "?"; + return `${comment ? `${inner}// ${comment}\n` : ""}${inner}${name}${optional}: ${typeScript(value, inner, defs)};`; + }); + if (isRecord(rest)) + fields.push(`${inner}[key: string]: ${typeScript(rest, inner, defs)};`); + return `{\n${fields.join("\n")}\n${indent}}`; +} + +// `alias` is the name the model calls; `wire` is the name the inner adapter knows the tool by. +type FunctionTool = { + alias: string; + wire: string; + description: string; + parameters: unknown; +}; + +function execTool(tools: FunctionTool[]) { + const aliases: string[] = []; + const methods: string[] = []; + for (const { alias, description, parameters } of tools) { + const defs = definitions(parameters); + const scope = { prefix: alias, names: new Set(defs.keys()) }; + for (const [name, schema] of defs) { + const comment = note(schema); + aliases.push( + `${comment ? `// ${comment}\n` : ""}type ${alias}_${identifier(name)} = ${typeScript(schema, "", scope)};`, + ); + } + const doc = description + .split("\n") + .map((line) => ` // ${line}`) + .join("\n"); + methods.push( + `${doc}\n ${alias}(args: ${typeScript(parameters, " ", scope)}): Promise;`, + ); + } + return { + type: "custom", + name: EXEC, + description: [ + "Run a script that calls one tool.", + "- Write `const result = await tools.({...});` then `text(result)`.", + "- A script makes exactly one tool call, with a literal object argument; conditions, loops, try/catch and variables are not run.", + "- To make several independent calls, send several exec calls at once.", + "", + "```ts", + ...aliases, + `declare const tools: {\n${methods.join("\n\n")}\n};`, + "```", + ].join("\n"), + format: { type: "grammar", syntax: "lark", definition: EXEC_GRAMMAR }, + }; +} + +function functionTools(tools: unknown): FunctionTool[] | undefined { + if (!Array.isArray(tools) || tools.length === 0) return undefined; + const found: Omit[] = []; + for (const tool of tools) { + if ( + !isRecord(tool) || + tool["type"] !== "function" || + typeof tool["name"] !== "string" || + tool["name"] === EXEC || + typeof tool["description"] !== "string" + ) + return undefined; + found.push({ + wire: tool["name"], + description: tool["description"], + parameters: tool["parameters"], + }); + } + // Aliases are given in name order, so a tool keeps its alias however the list is ordered. + const taken = new Set([EXEC]); + const aliases = new Map(); + for (const wire of found.map((tool) => tool.wire).sort()) { + const base = identifier(decodeToolName(wire)); + let alias = base; + for (let n = 2; taken.has(alias); n++) alias = `${base}_${n}`; + taken.add(alias); + aliases.set(wire, alias); + } + return found.map((tool) => ({ + ...tool, + alias: aliases.get(tool.wire) ?? tool.wire, + })); +} + +const callScript = (alias: string, args: string) => + `const result = await tools.${alias}(${args});\ntext(result);\n`; + +// The wire body carries tool results as plain text, so which calls failed is read from the turns. +const failedCalls = (messages: ConversationTurn[]) => + new Set( + messages.flatMap(({ content }) => + content.flatMap((block) => + block.type === "tool_result" && block.isError ? [block.callId] : [], + ), + ), + ); + +// Past calls are shown to the model as the script it would have written for each one. +function historyItems( + input: unknown[], + aliases: ReadonlyMap, + failed: ReadonlySet, +): unknown[] { + const unsupported = new Set(); + return input.map((item) => { + if (!isRecord(item) || typeof item["call_id"] !== "string") return item; + const callId = item["call_id"]; + if ( + item["type"] === "function_call" && + typeof item["name"] === "string" && + typeof item["arguments"] === "string" + ) { + const args = parseJSON(item["arguments"]); + if ( + item["name"] === EXEC && + isRecord(args) && + typeof args["script"] === "string" + ) { + unsupported.add(callId); + return { + type: "custom_tool_call", + call_id: callId, + name: EXEC, + input: args["script"], + }; + } + return { + type: "custom_tool_call", + call_id: callId, + name: EXEC, + input: callScript( + aliases.get(item["name"]) ?? identifier(decodeToolName(item["name"])), + item["arguments"], + ), + }; + } + if (item["type"] === "function_call_output") { + const result = + typeof item["output"] === "string" + ? item["output"] + : JSON.stringify(item["output"]); + let output = `Script completed\nOutput:\n${result}`; + if (failed.has(callId)) output = `Script failed\nError:\n${result}`; + if (unsupported.has(callId)) output = UNSUPPORTED_SCRIPT; + return { type: "custom_tool_call_output", call_id: callId, output }; + } + return item; + }); +} + +// Index just past the string, template or comment opening at `start`, or `start` when none does. +function skipLiteral(source: string, start: number): number { + const char = source.charAt(start); + if (char === '"' || char === "'" || char === "`") { + let i = start + 1; + while (i < source.length && source.charAt(i) !== char) + i += source.charAt(i) === "\\" ? 2 : 1; + return i + 1; + } + if (source.startsWith("//", start)) { + const end = source.indexOf("\n", start); + return end === -1 ? source.length : end + 1; + } + if (source.startsWith("/*", start)) { + const end = source.indexOf("*/", start + 2); + return end === -1 ? source.length : end + 2; + } + return start; +} + +// The top-level statements of a script, split at `;` and line breaks outside brackets, strings +// and comments. A regular expression literal can be misread as a string; the statement it lands +// in then matches no accepted shape, so the script is refused rather than misread. +function statements(source: string): string[] { + const found: string[] = []; + let depth = 0; + let start = 0; + // Index of the last character outside whitespace and comments, which decides whether a line + // break ends the statement. + let last = -1; + let i = 0; + while (i < source.length) { + const char = source.charAt(i); + const comment = source.startsWith("//", i) || source.startsWith("/*", i); + if (comment && depth === 0) found.push(source.slice(start, i)); + const after = skipLiteral(source, i); + if (after !== i) { + i = after; + if (comment && depth === 0) start = i; + if (!comment) last = i - 1; + continue; + } + if ("([{".includes(char)) depth++; + if (")]}".includes(char)) depth--; + const continues = + char === "\n" && + /(?:=|\bawait)$/.test(source.slice(Math.max(start, last - 6), last + 1)); + if (!/\s/.test(char)) last = i; + if (depth === 0 && (char === ";" || (char === "\n" && !continues))) { + found.push(source.slice(start, i)); + start = i + 1; + } + i++; + } + found.push(source.slice(start)); + return found.map((statement) => statement.trim()).filter(Boolean); +} + +// Index of the parenthesis closing the one before `open`, or -1. +function closingParen(source: string, open: number): number { + let depth = 1; + let i = open; + while (i < source.length) { + const after = skipLiteral(source, i); + if (after !== i) { + i = after; + continue; + } + if (source.charAt(i) === "(") depth++; + if (source.charAt(i) === ")" && --depth === 0) return i; + i++; + } + return -1; +} + +const CODE_POINT = /u\{([0-9a-fA-F]+)\}/y; +// A `\u{...}` escape in a quoted string, after an even run of backslashes. +const QUOTED_CODE_POINT = /(? + JSON.stringify(String.fromCodePoint(point)).slice(1, -1).replace("'", "\\'"); + +// `text` with each `\u{...}` escape, which JSON5 lacks, written out, or `undefined` when one is +// out of range. +function withCodePoints(text: string): string | undefined { + let valid = true; + const out = text.replace( + QUOTED_CODE_POINT, + (_match, slashes: string, hex: string) => { + const point = Number.parseInt(hex, 16); + valid &&= point <= 0x10ffff; + return valid ? slashes + codePoint(point) : ""; + }, + ); + return valid ? out : undefined; +} + +// `literal` as JSON5: each template literal becomes a JSON string and each `\u{...}` escape a +// character, or `undefined` when a template interpolates, since its value depends on code that is +// not run, or an escape is out of range. +function asJSON5(literal: string): string | undefined { + let out = ""; + let i = 0; + while (i < literal.length) { + const after = skipLiteral(literal, i); + if (after === i) { + out += literal.charAt(i++); + continue; + } + const text = literal.slice(i, after); + i = after; + if (!text.startsWith("`")) { + const quoted = withCodePoints(text); + if (quoted === undefined) return undefined; + out += quoted; + continue; + } + const body = text.slice(1, -1); + let string = '"'; + for (let k = 0; k < body.length; k++) { + const char = body.charAt(k); + if (char === "$" && body.charAt(k + 1) === "{") return undefined; + if (char === "\\") { + CODE_POINT.lastIndex = k + 1; + const hex = CODE_POINT.exec(body)?.[1]; + if (hex === undefined) { + const next = body.charAt(++k); + if (next === "\r") { + if (body.charAt(k + 1) === "\n") k++; + continue; + } + string += next === "`" || next === "$" ? next : char + next; + continue; + } + const point = Number.parseInt(hex, 16); + if (point > 0x10ffff) return undefined; + string += codePoint(point); + k = CODE_POINT.lastIndex - 1; + } else if (char === '"') string += '\\"'; + else if (char === "\n") string += "\\n"; + else if (char === "\r") { + string += "\\n"; + if (body.charAt(k + 1) === "\n") k++; + } else string += char; + } + out += `${string}"`; + } + return out; +} + +const CALL_STATEMENT = + /^(?:(?:const|let|var)\s+[A-Za-z_$][\w$]*\s*=\s*)?await\s+tools\.([A-Za-z_$][\w$]*)\s*\(/; +// Output adds nothing to the call, so only output of a variable, a property of one or a plain +// string is accepted; anything that evaluates more could hide a second call. +const PATH = String.raw`[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*|\[\d+\])*`; +const STRING = String.raw`"(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'|\`[^\`$\\]*\``; +const OUTPUT_STATEMENT = new RegExp( + String.raw`^(?:text|console\.log)\(\s*(?:${PATH}|${STRING}|JSON\.stringify\(\s*${PATH}(?:\s*,\s*null\s*,\s*\d+)?\s*\))\s*\)$`, +); + +/** + * The one tool call a model's script makes, named as `tools` maps the alias it calls, or `undefined` + * when the script is anything else: more than one call, a call to an unknown tool, an argument that + * is not a literal object, or any statement besides the call and `text()` output, such as + * a condition, loop or try/catch. The script is read, never run; the argument is parsed as JSON5, + * the data subset of JavaScript. + */ +export function scriptCall( + source: string, + tools: ReadonlyMap, +): ToolCall | undefined { + let call: ToolCall | undefined; + for (const statement of statements(source)) { + if (OUTPUT_STATEMENT.test(statement)) continue; + const match = CALL_STATEMENT.exec(statement); + const alias = match?.[1]; + const name = alias === undefined ? undefined : tools.get(alias); + if (match === null || name === undefined || call) return undefined; + const open = match[0].length; + if (closingParen(statement, open) !== statement.length - 1) + return undefined; + const literal = asJSON5(statement.slice(open, -1).trim()); + if (literal === undefined) return undefined; + let args: unknown = {}; + if (literal !== "") { + try { + args = Bun.JSON5.parse(literal); + } catch (error) { + if (error instanceof SyntaxError) return undefined; + throw error; + } + } + if (!isRecord(args)) return undefined; + call = { name, arguments: args }; + } + return call; +} + +type CustomToolCall = { id: string; call_id: string; input: string }; + +function customToolCall(item: unknown): CustomToolCall | undefined { + if ( + !isRecord(item) || + item["type"] !== "custom_tool_call" || + typeof item["id"] !== "string" || + typeof item["call_id"] !== "string" || + typeof item["input"] !== "string" + ) + return undefined; + return { id: item["id"], call_id: item["call_id"], input: item["input"] }; +} + +// A script that cannot be read becomes an `exec` call, which no tool answers, and is replayed to +// the model as UNSUPPORTED_SCRIPT. +function functionCall( + call: CustomToolCall, + tools: ReadonlyMap, +) { + const { name, arguments: args } = scriptCall(call.input, tools) ?? { + name: EXEC, + arguments: { script: call.input }, + }; + return { + type: "function_call", + id: call.id, + call_id: call.call_id, + name, + arguments: JSON.stringify(args), + }; +} + +/** + * Wraps a Responses adapter so the model reaches its tools through one `exec` code tool, the way + * Codex serves its models. Each tool is offered under a JavaScript-safe alias of its decoded name. + * Requests with a tool that is not a function tool pass through unchanged. + */ +export function withCodeMode(factory: AdapterFactory): AdapterFactory { + return (source, quirks) => { + const adapter = factory(source, quirks); + let wires: ReadonlyMap = new Map(); + return { + ...adapter, + buildRequest: (messages, model, options) => { + const request = adapter.buildRequest(messages, model, options); + const body = parseJSON(request.body); + if (!isRecord(body) || !Array.isArray(body["input"])) return request; + const tools = functionTools(body["tools"]); + wires = new Map(tools?.map(({ alias, wire }) => [alias, wire])); + if (tools === undefined) return request; + const aliases = new Map(tools.map(({ alias, wire }) => [wire, alias])); + return { + ...request, + body: JSON.stringify({ + ...body, + tools: [execTool(tools)], + input: historyItems(body["input"], aliases, failedCalls(messages)), + }), + }; + }, + parseResponse: (sseData) => { + const event = parseJSON(sseData); + if ( + !isRecord(event) || + !isRecord(event["item"]) || + event["item"]["type"] !== "custom_tool_call" + ) + return adapter.parseResponse(sseData); + if (event["type"] !== "response.output_item.done") return []; + const call = customToolCall(event["item"]); + if (call === undefined) + throw new ProtocolMismatchError( + "output_item.done custom_tool_call is missing id, call_id or input", + event, + ); + const { sequence_number, output_index } = event; + const { arguments: args, ...item } = functionCall(call, wires); + return [ + ...adapter.parseResponse( + JSON.stringify({ + type: "response.output_item.added", + sequence_number, + output_index, + item: { ...item, arguments: "" }, + }), + ), + ...adapter.parseResponse( + JSON.stringify({ + type: "response.function_call_arguments.delta", + sequence_number, + output_index, + item_id: item.id, + delta: args, + }), + ), + ]; + }, + parseJSONResponse: (responseBody) => { + const body = parseJSON(responseBody); + if (!isRecord(body) || !Array.isArray(body["output"])) + return adapter.parseJSONResponse(responseBody); + const output = body["output"].map((item: unknown) => { + if (!isRecord(item) || item["type"] !== "custom_tool_call") + return item; + const call = customToolCall(item); + if (call === undefined) + throw new ProtocolMismatchError( + "custom_tool_call output item is missing id, call_id or input", + item, + ); + return functionCall(call, wires); + }); + return adapter.parseJSONResponse(JSON.stringify({ ...body, output })); + }, + }; + }; +} diff --git a/apps/hub/src/inference/codex-adapter.ts b/apps/hub/src/inference/codex-adapter.ts index 4bef3bd4..fa958342 100644 --- a/apps/hub/src/inference/codex-adapter.ts +++ b/apps/hub/src/inference/codex-adapter.ts @@ -3,10 +3,11 @@ import { withCodexContentTypeRepair, } from "@corbits/codex-provider"; import type { AdapterFactory } from "@intx/inference"; +import { withCodeMode } from "./code-mode.ts"; // `@corbits/codex-provider` hardcodes `parallel_tool_calls: false`, which // forces the model to serialize independent tool calls across turns. -export const createCodexResponsesAdapter: AdapterFactory = (source, quirks) => { +const createParallelCodexAdapter: AdapterFactory = (source, quirks) => { const adapter = createProviderCodexResponsesAdapter(source, quirks); return { ...adapter, @@ -24,6 +25,10 @@ export const createCodexResponsesAdapter: AdapterFactory = (source, quirks) => { }; }; +export const createCodexResponsesAdapter = withCodeMode( + createParallelCodexAdapter, +); + // The Codex backend omits Content-Type on some response streams, and the inference harness rejects a response // without one. A sidecar builds its harness on the global fetch after importing this adapter, so the repair is // installed there; it only touches successful Codex responses that arrive without a Content-Type. diff --git a/apps/web/src/config.ts b/apps/web/src/config.ts index a6fc5eeb..310a7180 100644 --- a/apps/web/src/config.ts +++ b/apps/web/src/config.ts @@ -544,7 +544,12 @@ export const wording = { file_package: () => "filing the approval package", instruct_coder: () => "waiting for you to approve a change for the app builder", + exec: () => "rewriting a tool call", } as Record string>, + /** What a call returned, in plain words, for a tool whose raw result means nothing to a person. */ + toolResult: { + exec: "The assistant wrote this step in a form the app can't run, so it was asked to write it again.", + } as Record, otherTool: (name: string) => `using ${name.replaceAll("_", " ")}`, elapsed: (minutes: number, seconds: number) => minutes === 0 ? `${seconds}s` : `${minutes}m`, diff --git a/apps/web/src/pages/workspace/agent-activity.test.ts b/apps/web/src/pages/workspace/agent-activity.test.ts index 6eb36606..d19e5be9 100644 --- a/apps/web/src/pages/workspace/agent-activity.test.ts +++ b/apps/web/src/pages/workspace/agent-activity.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; -import { activityPhrase } from "./agent-activity.tsx"; +import { wording } from "../../config.ts"; +import { activityPhrase, toolResult } from "./agent-activity.tsx"; const coder = { runId: "run-coder", @@ -27,3 +28,10 @@ test("a long command's own word on its progress replaces the reading of the call }), ).toEqual({ phrase: "installing the app's packages", asking: false }); }); + +test("a call whose raw result means nothing to a person is described in plain words", () => { + expect(toolResult("exec", "unknown tool: exec")).toBe( + wording.activity.toolResult["exec"], + ); + expect(toolResult("read_file", '{"lines":3}')).toEqual({ lines: 3 }); +}); diff --git a/apps/web/src/pages/workspace/agent-activity.tsx b/apps/web/src/pages/workspace/agent-activity.tsx index 63bb503a..f2688353 100644 --- a/apps/web/src/pages/workspace/agent-activity.tsx +++ b/apps/web/src/pages/workspace/agent-activity.tsx @@ -84,6 +84,19 @@ export function toolPhrase(name: string, subject: string | undefined): string { return phrase.charAt(0).toUpperCase() + phrase.slice(1); } +/** What a call returned, in plain words when its raw result means nothing to a person. */ +export function toolResult(name: string, content: unknown): unknown { + const plain = text.toolResult[name]; + if (plain !== undefined) return plain; + if (typeof content !== "string") return content; + try { + return JSON.parse(content) as unknown; + } catch (error) { + if (error instanceof SyntaxError) return content; + throw error; + } +} + // One verb per file, so the line changes when the file does and not on every poll. function verbFor(verbs: readonly string[], subject: string): string { let hash = 0; diff --git a/apps/web/src/pages/workspace/transcript.tsx b/apps/web/src/pages/workspace/transcript.tsx index 4fe9d97c..2e79437b 100644 --- a/apps/web/src/pages/workspace/transcript.tsx +++ b/apps/web/src/pages/workspace/transcript.tsx @@ -48,7 +48,7 @@ import { type ToolApprovalDecision, type ToolApprovalStatus, } from "@corbits/react-ui"; -import { toolPhrase, toolSubject } from "./agent-activity.tsx"; +import { toolPhrase, toolResult, toolSubject } from "./agent-activity.tsx"; import { DOCUMENT_LINK, Markdown } from "../../markdown.jsx"; import { toast } from "sonner"; import { CONV_SCROLL_CLASS } from "./pane-classes.ts"; @@ -174,15 +174,6 @@ function instructionOf(input: unknown): string | undefined { : undefined; } -function resultValue(content: unknown): unknown { - if (typeof content !== "string") return content; - try { - return JSON.parse(content) as unknown; - } catch { - return content; - } -} - /** * A call named in plain words, so the activity's live line and steps read like * a person at work. A finished hand-off says whom it started. Opened, it shows @@ -206,7 +197,7 @@ function readableTool( ...(result === undefined ? {} : { - output: resultValue(result.content), + output: toolResult(part.name, result.content), ...(result.isError ? { status: "error" as const } : {}), }), };