From e0f015672728621221550f1257e49ffcc3fb1514 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Wed, 4 Nov 2026 12:05:32 -0800 Subject: [PATCH 1/2] fix(hub): read check verdicts from the tool result's detail --- apps/hub/src/builds/check-detail.ts | 12 +++ apps/hub/src/builds/sidecar-tools.test.ts | 49 ++++++++++ apps/hub/src/builds/sidecar-tools.ts | 31 +++++- apps/hub/src/workflows/director.test.ts | 111 ++++++++++++++++++++++ apps/hub/src/workflows/director.ts | 20 ++-- 5 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 apps/hub/src/builds/check-detail.ts create mode 100644 apps/hub/src/builds/sidecar-tools.test.ts diff --git a/apps/hub/src/builds/check-detail.ts b/apps/hub/src/builds/check-detail.ts new file mode 100644 index 00000000..6d5cd142 --- /dev/null +++ b/apps/hub/src/builds/check-detail.ts @@ -0,0 +1,12 @@ +import { type } from "arktype"; + +// A check's verdict is read once from its whole output and kept in the result's detail, which +// history trimming leaves alone: a trimmed result loses the "check passed" line the model read. +const CheckDetail = type({ checkPassed: "boolean", "filed?": "boolean" }); +export type CheckDetail = typeof CheckDetail.infer; + +/** The detail a check tool records, or undefined for any other result. */ +export function readCheckDetail(detail: unknown): CheckDetail | undefined { + const parsed = CheckDetail(detail); + return parsed instanceof type.errors ? undefined : parsed; +} diff --git a/apps/hub/src/builds/sidecar-tools.test.ts b/apps/hub/src/builds/sidecar-tools.test.ts new file mode 100644 index 00000000..8f399f80 --- /dev/null +++ b/apps/hub/src/builds/sidecar-tools.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { appCheckDetail } from "./sidecar-tools.ts"; + +const PASSED = + "check passed: built, started, and opened / at 1280 and 390 px with no errors"; + +const shell = (command: string) => ({ + id: "call", + name: "run_shell", + arguments: { command }, +}); +const output = (content: string) => ({ callId: "call", content }); + +test("a check-app run records its verdict, and only a real check-app run records one", () => { + expect(appCheckDetail(shell("check-app"), output(PASSED))).toEqual({ + checkPassed: true, + }); + expect(appCheckDetail(shell("cd app && check-app"), output(PASSED))).toEqual({ + checkPassed: true, + }); + expect( + appCheckDetail(shell("check-app"), output("check failed: 1 problem(s)")), + ).toEqual({ checkPassed: false }); + expect( + appCheckDetail(shell("check-app"), { + callId: "call", + content: PASSED, + isError: true, + }), + ).toEqual({ checkPassed: false }); + expect( + appCheckDetail( + shell("check-app && bun test"), + output(`exit code 1\n${PASSED}`), + ), + ).toEqual({ checkPassed: false }); + expect( + appCheckDetail(shell("grep -rn check-app ."), output(PASSED)), + ).toBeUndefined(); + expect( + appCheckDetail(shell("cat bin/check-app"), output(PASSED)), + ).toBeUndefined(); + expect( + appCheckDetail( + { id: "call", name: "read_file", arguments: { path: "check-app" } }, + output(PASSED), + ), + ).toBeUndefined(); +}); diff --git a/apps/hub/src/builds/sidecar-tools.ts b/apps/hub/src/builds/sidecar-tools.ts index 12047676..e02f4839 100644 --- a/apps/hub/src/builds/sidecar-tools.ts +++ b/apps/hub/src/builds/sidecar-tools.ts @@ -20,6 +20,7 @@ import { noteCommandProgress, } from "./posix-sandbox.ts"; import { browserTools } from "./browser.ts"; +import type { CheckDetail } from "./check-detail.ts"; import type { BrowserCatalog } from "./browser-catalog.ts"; import { CHECK_APP, @@ -149,12 +150,25 @@ const succeeded = (result: ToolResult) => const VERSIONED = new Set([...WRITES, "run_shell", PACKAGE_INSTALL_TOOL]); // Bounds teardown: the wait for the last version, then the revoke. const VERSION_CLOSE_MS = 15_000; +// check-app run as a command, not mentioned in one such as grep -rn check-app. +const RUNS_CHECK_APP = /(?:^|[;&|(]\s*)check-app(?=$|[\s;&|)])/m; +const ranAppCheck = (call: ToolCall) => + call.name === "run_shell" && + RUNS_CHECK_APP.test(String(call.arguments["command"])); // An app-mode check-app pass; a mockup check in the same workspace passes without building anything. const passedAppCheck = (call: ToolCall, result: ToolResult) => - call.name === "run_shell" && - /check-app/.test(String(call.arguments["command"])) && + ranAppCheck(call) && typeof result.content === "string" && /check passed: built, started,/.test(result.content); +/** A check-app run's verdict; check-app && npm test passes only when both do. */ +export function appCheckDetail( + call: ToolCall, + result: ToolResult, +): CheckDetail | undefined { + if (!ranAppCheck(call)) return undefined; + const passed = /^check passed:/m.test(String(result.content)); + return { checkPassed: succeeded(result) && passed }; +} // A cold install; under the executor's own ceiling. const SETUP_TIMEOUT_MS = 115_000; @@ -552,8 +566,13 @@ export function createSidecarCodingTools(policy: { await open(signal); if (!designCheck) throw new Error("design_check_missing"); await syncDesignScreensOnDisk(await realpath(env.toolCwd)); - const { checklist } = await designCheck.current(signal); - return { callId: call.id, content: checklist }; + const { checklist, passed, artifact } = + await designCheck.current(signal); + const detail: CheckDetail = { + checkPassed: passed, + filed: artifact !== undefined, + }; + return { callId: call.id, content: checklist, detail }; } if (!offered(call.name)) return { @@ -561,7 +580,9 @@ export function createSidecarCodingTools(policy: { content: `unknown_tool: ${call.name}`, isError: true, }; - const result = await native.run(call, signal); + const ran = await native.run(call, signal); + const detail = appCheckDetail(call, ran); + const result = detail ? { ...ran, detail } : ran; if (repository && VERSIONED.has(call.name)) repository.schedule( passedAppCheck(call, result) ? CHECK_PASSED : EDIT, diff --git a/apps/hub/src/workflows/director.test.ts b/apps/hub/src/workflows/director.test.ts index da279ffa..fee17801 100644 --- a/apps/hub/src/workflows/director.test.ts +++ b/apps/hub/src/workflows/director.test.ts @@ -23,6 +23,7 @@ import { createDesignCheck, } from "../builds/design-check.ts"; import { REPETITION_ABORT } from "../inference/repetition.ts"; +import { trimHistory } from "./compaction.ts"; import { compactingDirector } from "./director.ts"; const director = (replyOnFailure: boolean) => @@ -280,6 +281,7 @@ test("designer director holds replies until the latest edits pass check_design a text: 'check passed: served .builder/designs\nFiled design_option: {"artifactId":"design-1","kind":"design_option","version":1,"sha256":"a"}', }, ], + detail: { checkPassed: true, filed: true }, }, ], timestamp: 4, @@ -295,6 +297,115 @@ test("designer director holds replies until the latest edits pass check_design a expect(accepted.some((action) => action.type === "reply")).toBe(true); }); +test("coder director takes a check-app verdict from the result's detail after trimming drops its text", async () => { + const subject = compactingDirector("@corbits-builder/coder-test").factory( + { contextWindow: 200_000, compactAt: 0.8, checkBeforeDone: true }, + {} as never, + { systemPrompt: "", toolDefinitions: [], compactorNames: [] }, + ) as ReactorDirector; + const replyTurn = { + role: "assistant" as const, + model: "gpt-5", + content: [{ type: "text" as const, text: "Done." }], + timestamp: 5, + }; + const checked = ( + output: string, + checkPassed: boolean, + ): ConversationTurn[] => [ + { + role: "user", + content: [{ type: "text", text: "Build the app." }], + timestamp: 0, + }, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call-write", + name: "write_file", + arguments: { path: "src/App.tsx", content: "export {}" }, + }, + ], + timestamp: 1, + }, + { + role: "user", + content: [ + { + type: "tool_result", + callId: "call-write", + content: [{ type: "text", text: "wrote src/App.tsx" }], + }, + ], + timestamp: 2, + }, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call-check", + name: "run_shell", + arguments: { command: "check-app" }, + }, + ], + timestamp: 3, + }, + { + role: "user", + content: [ + { + type: "tool_result", + callId: "call-check", + content: [{ type: "text", text: output }], + detail: { checkPassed }, + }, + ], + timestamp: 4, + }, + ]; + const decideTrimmed = async (output: string, checkPassed: boolean) => { + const { output: turns } = await trimHistory(1).apply( + checked(output, checkPassed), + { state, trigger: "context" }, + ); + const result = turns + .flatMap((turn) => turn.content) + .find( + (block) => + block.type === "tool_result" && block.callId === "call-check", + ); + expect(JSON.stringify(result)).not.toContain(output.slice(0, 20)); + const actions = await subject.decide( + { + type: "inference.done", + turn: replyTurn, + usage: state.tokenUsage, + source: { sourceId: "openai", provider: "openai", model: "gpt-5" }, + }, + { ...state, turns: [...turns, replyTurn] }, + createCapabilities(), + ); + return (Array.isArray(actions) ? actions : [actions]) as ReactorAction[]; + }; + const checklist = "\n✓ every page opened with no errors".repeat(40); + + const passed = await decideTrimmed( + `check passed: built, started, and opened / at 1280 and 390 px with no errors${checklist}`, + true, + ); + expect(passed.some((action) => action.type === "reply")).toBe(true); + + const failed = await decideTrimmed( + `check failed: 1 problem(s) across 1 page(s)${checklist}`, + false, + ); + expect(failed.some((action) => action.type === "reply")).toBe(false); + expect(failed.some((action) => action.type === "compact")).toBe(true); +}); + test("check_design validates DESIGN.md and .dark in styles.css and auto-files the design option on pass", async () => { const starter = designStarterFiles(); const modernSleek = designThemes()["modern-sleek"]!; diff --git a/apps/hub/src/workflows/director.ts b/apps/hub/src/workflows/director.ts index 030915df..2c8b3d1c 100644 --- a/apps/hub/src/workflows/director.ts +++ b/apps/hub/src/workflows/director.ts @@ -10,7 +10,7 @@ import type { RetryPolicy, } from "@intx/types/runtime"; import { REPETITION_ABORT } from "../inference/repetition.ts"; -import { CHECK_DESIGN_TOOL } from "../builds/design-check.ts"; +import { readCheckDetail, type CheckDetail } from "../builds/check-detail.ts"; import { COMPACTOR, DROP_DEGENERATE } from "./compaction.ts"; // About 1 coder turn in 100 skipped reasoning and repeated one sentence ("Ok to write. Let me @@ -144,11 +144,11 @@ const SHELL_WRITE = const WRITE_TOOLS = new Set(["write_file", "edit_file"]); // The current task's calls, newest first: everything after the last mail the agent was handed. function taskCalls(turns: readonly ConversationTurn[]) { - const results = new Map(); + const results = new Map(); const calls: { name: string; command: unknown; - result: string | undefined; + check: CheckDetail | undefined; isError: boolean; }[] = []; for (let index = turns.length - 1; index >= 0; index--) { @@ -161,7 +161,7 @@ function taskCalls(turns: readonly ConversationTurn[]) { for (const block of [...turn.content].reverse()) { if (block.type === "tool_result") results.set(block.callId, { - text: textOf(block.content), + detail: block.detail, isError: block.isError === true, }); if (block.type === "tool_call") { @@ -169,7 +169,7 @@ function taskCalls(turns: readonly ConversationTurn[]) { calls.push({ name: block.name, command: (block.arguments as Record)["command"], - result: result?.text, + check: readCheckDetail(result?.detail), isError: result?.isError ?? true, }); } @@ -185,18 +185,12 @@ const ranShell = (call: TaskCall, pattern: RegExp) => typeof call.command === "string" && pattern.test(call.command); -const says = (call: TaskCall, pattern: RegExp) => - call.result !== undefined && pattern.test(call.result); - // Run 36's coder passed check-app, kept editing, broke the app and would have replied done; a reply // counts as done only when no file changed after the last passing check. check_design is the // designer's check-app; file_design_option files only a design it passed as it stands. function unchecked(turns: readonly ConversationTurn[]): boolean { for (const call of taskCalls(turns)) { - if (ranShell(call, /check-app/)) - return call.isError || !says(call, /check passed/); - if (call.name === CHECK_DESIGN_TOOL) - return call.isError || !says(call, /check passed/); + if (call.check) return call.isError || !call.check.checkPassed; if (call.name === "file_design_option") return call.isError; if (WRITE_TOOLS.has(call.name) || ranShell(call, SHELL_WRITE)) return true; } @@ -208,7 +202,7 @@ const filed = (call: TaskCall) => (call.name === "write_artifact" || call.name === "file_package" || call.name === "file_design_option" || - (call.name === CHECK_DESIGN_TOOL && says(call, /Filed design_option:/))); + call.check?.filed === true); // A writer on 2026-10-30 replied with the whole approaches document as text and never filed it; // the coordinator waited on a writer that had already answered. A designer's reply also waits for a From ff8ac328c085a07732ec691386aee6379afec730 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Wed, 4 Nov 2026 12:52:42 -0800 Subject: [PATCH 2/2] fix(hub): recognise check-app runs by their verdict line --- apps/hub/src/builds/sidecar-tools.test.ts | 41 ++++++++++++++++++----- apps/hub/src/builds/sidecar-tools.ts | 34 +++++++++++-------- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/apps/hub/src/builds/sidecar-tools.test.ts b/apps/hub/src/builds/sidecar-tools.test.ts index 8f399f80..dd7b934b 100644 --- a/apps/hub/src/builds/sidecar-tools.test.ts +++ b/apps/hub/src/builds/sidecar-tools.test.ts @@ -3,6 +3,8 @@ import { appCheckDetail } from "./sidecar-tools.ts"; const PASSED = "check passed: built, started, and opened / at 1280 and 390 px with no errors"; +const MOCKUP_PASSED = + "check passed: served .builder/designs and opened / at 1280 and 390 px with no errors"; const shell = (command: string) => ({ id: "call", @@ -11,16 +13,28 @@ const shell = (command: string) => ({ }); const output = (content: string) => ({ callId: "call", content }); -test("a check-app run records its verdict, and only a real check-app run records one", () => { - expect(appCheckDetail(shell("check-app"), output(PASSED))).toEqual({ - checkPassed: true, - }); - expect(appCheckDetail(shell("cd app && check-app"), output(PASSED))).toEqual({ - checkPassed: true, - }); +test("a check-app run records its verdict however the command runs it", () => { + for (const command of [ + "check-app", + "cd app && check-app", + "timeout 300 check-app", + "CI=1 check-app", + "./bin/check-app", + "bash -lc 'check-app'", + "timeout 600 check-app 2>&1 | tail -40", + ]) + expect(appCheckDetail(shell(command), output(PASSED))).toEqual({ + checkPassed: true, + }); +}); + +test("only an app check-app pass of a command that succeeded counts as passed", () => { expect( appCheckDetail(shell("check-app"), output("check failed: 1 problem(s)")), ).toEqual({ checkPassed: false }); + expect( + appCheckDetail(shell("check-app .builder/designs"), output(MOCKUP_PASSED)), + ).toEqual({ checkPassed: false }); expect( appCheckDetail(shell("check-app"), { callId: "call", @@ -34,11 +48,20 @@ test("a check-app run records its verdict, and only a real check-app run records output(`exit code 1\n${PASSED}`), ), ).toEqual({ checkPassed: false }); +}); + +test("a command that only names check-app records no verdict", () => { expect( - appCheckDetail(shell("grep -rn check-app ."), output(PASSED)), + appCheckDetail( + shell("grep -rn check-app ."), + output(`src/check.ts:12: ${PASSED}`), + ), ).toBeUndefined(); expect( - appCheckDetail(shell("cat bin/check-app"), output(PASSED)), + appCheckDetail( + shell("check-app --help"), + output("check-app check the app here"), + ), ).toBeUndefined(); expect( appCheckDetail( diff --git a/apps/hub/src/builds/sidecar-tools.ts b/apps/hub/src/builds/sidecar-tools.ts index e02f4839..cd55d2db 100644 --- a/apps/hub/src/builds/sidecar-tools.ts +++ b/apps/hub/src/builds/sidecar-tools.ts @@ -150,26 +150,32 @@ const succeeded = (result: ToolResult) => const VERSIONED = new Set([...WRITES, "run_shell", PACKAGE_INSTALL_TOOL]); // Bounds teardown: the wait for the last version, then the revoke. const VERSION_CLOSE_MS = 15_000; -// check-app run as a command, not mentioned in one such as grep -rn check-app. -const RUNS_CHECK_APP = /(?:^|[;&|(]\s*)check-app(?=$|[\s;&|)])/m; -const ranAppCheck = (call: ToolCall) => - call.name === "run_shell" && - RUNS_CHECK_APP.test(String(call.arguments["command"])); -// An app-mode check-app pass; a mockup check in the same workspace passes without building anything. -const passedAppCheck = (call: ToolCall, result: ToolResult) => - ranAppCheck(call) && - typeof result.content === "string" && - /check passed: built, started,/.test(result.content); -/** A check-app run's verdict; check-app && npm test passes only when both do. */ +// check-app's verdict heads a line of its output, however the command runs it (timeout, env, a +// wrapper or a pipe); a command that only names check-app, such as grep -rn check-app, prints none. +const CHECK_APP_VERDICT = /^check (?:passed|failed):/m; +// A mockup check passes without building anything; only this verdict says the app built and ran. +const APP_CHECK_PASSED = /^check passed: built, started,/m; + +/** A check-app run's verdict; check-app && bun test passes only when both do. */ export function appCheckDetail( call: ToolCall, result: ToolResult, ): CheckDetail | undefined { - if (!ranAppCheck(call)) return undefined; - const passed = /^check passed:/m.test(String(result.content)); - return { checkPassed: succeeded(result) && passed }; + const command = call.arguments["command"]; + if ( + call.name !== "run_shell" || + typeof command !== "string" || + !command.includes("check-app") + ) + return undefined; + const output = String(result.content); + if (!CHECK_APP_VERDICT.test(output)) return undefined; + return { checkPassed: succeeded(result) && APP_CHECK_PASSED.test(output) }; } +const passedAppCheck = (call: ToolCall, result: ToolResult) => + appCheckDetail(call, result)?.checkPassed === true; + // A cold install; under the executor's own ceiling. const SETUP_TIMEOUT_MS = 115_000; // What the person reads while the workspace installs its packages, which can take the whole timeout.