fix(hub): read check verdicts from the tool result's detail

Author: TheGreatAxios

apps/hub/src/builds/check-detail.ts

@@ -0,0 +1,12 @@
+import { type } from "arktype";
+
+// A check's verdict is read once from its whole output and kept in the result's detail, which
+// history trimming leaves alone: a trimmed result loses the "check passed" line the model read.
+const CheckDetail = type({ checkPassed: "boolean", "filed?": "boolean" });
+export type CheckDetail = typeof CheckDetail.infer;
+
+/** The detail a check tool records, or undefined for any other result. */
+export function readCheckDetail(detail: unknown): CheckDetail | undefined {
+  const parsed = CheckDetail(detail);
+  return parsed instanceof type.errors ? undefined : parsed;
+}

apps/hub/src/builds/sidecar-tools.test.ts

@@ -0,0 +1,49 @@
+import { expect, test } from "bun:test";
+import { appCheckDetail } from "./sidecar-tools.ts";
+
+const PASSED =
+  "check passed: built, started, and opened / at 1280 and 390 px with no errors";
+
+const shell = (command: string) => ({
+  id: "call",
+  name: "run_shell",
+  arguments: { command },
+});
+const output = (content: string) => ({ callId: "call", content });
+
+test("a check-app run records its verdict, and only a real check-app run records one", () => {
+  expect(appCheckDetail(shell("check-app"), output(PASSED))).toEqual({
+    checkPassed: true,
+  });
+  expect(appCheckDetail(shell("cd app && check-app"), output(PASSED))).toEqual({
+    checkPassed: true,
+  });
+  expect(
+    appCheckDetail(shell("check-app"), output("check failed: 1 problem(s)")),
+  ).toEqual({ checkPassed: false });
+  expect(
+    appCheckDetail(shell("check-app"), {
+      callId: "call",
+      content: PASSED,
+      isError: true,
+    }),
+  ).toEqual({ checkPassed: false });
+  expect(
+    appCheckDetail(
+      shell("check-app && bun test"),
+      output(`exit code 1\n${PASSED}`),
+    ),
+  ).toEqual({ checkPassed: false });
+  expect(
+    appCheckDetail(shell("grep -rn check-app ."), output(PASSED)),
+  ).toBeUndefined();
+  expect(
+    appCheckDetail(shell("cat bin/check-app"), output(PASSED)),
+  ).toBeUndefined();
+  expect(
+    appCheckDetail(
+      { id: "call", name: "read_file", arguments: { path: "check-app" } },
+      output(PASSED),
+    ),
+  ).toBeUndefined();
+});

apps/hub/src/builds/sidecar-tools.ts

@@ -20,6 +20,7 @@ import {
   noteCommandProgress,
 } from "./posix-sandbox.ts";
 import { browserTools } from "./browser.ts";
+import type { CheckDetail } from "./check-detail.ts";
 import type { BrowserCatalog } from "./browser-catalog.ts";
 import {
   CHECK_APP,
@@ -149,12 +150,25 @@ const succeeded = (result: ToolResult) =>
 const VERSIONED = new Set([...WRITES, "run_shell", PACKAGE_INSTALL_TOOL]);
 // Bounds teardown: the wait for the last version, then the revoke.
 const VERSION_CLOSE_MS = 15_000;
+// check-app run as a command, not mentioned in one such as grep -rn check-app.
+const RUNS_CHECK_APP = /(?:^|[;&|(]\s*)check-app(?=$|[\s;&|)])/m;
+const ranAppCheck = (call: ToolCall) =>
+  call.name === "run_shell" &&
+  RUNS_CHECK_APP.test(String(call.arguments["command"]));
 // An app-mode check-app pass; a mockup check in the same workspace passes without building anything.
 const passedAppCheck = (call: ToolCall, result: ToolResult) =>
-  call.name === "run_shell" &&
-  /check-app/.test(String(call.arguments["command"])) &&
+  ranAppCheck(call) &&
   typeof result.content === "string" &&
   /check passed: built, started,/.test(result.content);
+/** A check-app run's verdict; check-app && npm test passes only when both do. */
+export function appCheckDetail(
+  call: ToolCall,
+  result: ToolResult,
+): CheckDetail | undefined {
+  if (!ranAppCheck(call)) return undefined;
+  const passed = /^check passed:/m.test(String(result.content));
+  return { checkPassed: succeeded(result) && passed };
+}
 
 // A cold install; under the executor's own ceiling.
 const SETUP_TIMEOUT_MS = 115_000;
@@ -552,16 +566,23 @@ export function createSidecarCodingTools(policy: {
             await open(signal);
             if (!designCheck) throw new Error("design_check_missing");
             await syncDesignScreensOnDisk(await realpath(env.toolCwd));
-            const { checklist } = await designCheck.current(signal);
-            return { callId: call.id, content: checklist };
+            const { checklist, passed, artifact } =
+              await designCheck.current(signal);
+            const detail: CheckDetail = {
+              checkPassed: passed,
+              filed: artifact !== undefined,
+            };
+            return { callId: call.id, content: checklist, detail };
           }
           if (!offered(call.name))
             return {
               callId: call.id,
               content: `unknown_tool: ${call.name}`,
               isError: true,
             };
-          const result = await native.run(call, signal);
+          const ran = await native.run(call, signal);
+          const detail = appCheckDetail(call, ran);
+          const result = detail ? { ...ran, detail } : ran;
           if (repository && VERSIONED.has(call.name))
             repository.schedule(
               passedAppCheck(call, result) ? CHECK_PASSED : EDIT,

apps/hub/src/workflows/director.test.ts

@@ -23,6 +23,7 @@ import {
   createDesignCheck,
 } from "../builds/design-check.ts";
 import { REPETITION_ABORT } from "../inference/repetition.ts";
+import { trimHistory } from "./compaction.ts";
 import { compactingDirector } from "./director.ts";
 
 const director = (replyOnFailure: boolean) =>
@@ -280,6 +281,7 @@ test("designer director holds replies until the latest edits pass check_design a
               text: 'check passed: served .builder/designs\nFiled design_option: {"artifactId":"design-1","kind":"design_option","version":1,"sha256":"a"}',
             },
           ],
+          detail: { checkPassed: true, filed: true },
         },
       ],
       timestamp: 4,
@@ -295,6 +297,115 @@ test("designer director holds replies until the latest edits pass check_design a
   expect(accepted.some((action) => action.type === "reply")).toBe(true);
 });
 
+test("coder director takes a check-app verdict from the result's detail after trimming drops its text", async () => {
+  const subject = compactingDirector("@corbits-builder/coder-test").factory(
+    { contextWindow: 200_000, compactAt: 0.8, checkBeforeDone: true },
+    {} as never,
+    { systemPrompt: "", toolDefinitions: [], compactorNames: [] },
+  ) as ReactorDirector;
+  const replyTurn = {
+    role: "assistant" as const,
+    model: "gpt-5",
+    content: [{ type: "text" as const, text: "Done." }],
+    timestamp: 5,
+  };
+  const checked = (
+    output: string,
+    checkPassed: boolean,
+  ): ConversationTurn[] => [
+    {
+      role: "user",
+      content: [{ type: "text", text: "Build the app." }],
+      timestamp: 0,
+    },
+    {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_call",
+          id: "call-write",
+          name: "write_file",
+          arguments: { path: "src/App.tsx", content: "export {}" },
+        },
+      ],
+      timestamp: 1,
+    },
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          callId: "call-write",
+          content: [{ type: "text", text: "wrote src/App.tsx" }],
+        },
+      ],
+      timestamp: 2,
+    },
+    {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_call",
+          id: "call-check",
+          name: "run_shell",
+          arguments: { command: "check-app" },
+        },
+      ],
+      timestamp: 3,
+    },
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          callId: "call-check",
+          content: [{ type: "text", text: output }],
+          detail: { checkPassed },
+        },
+      ],
+      timestamp: 4,
+    },
+  ];
+  const decideTrimmed = async (output: string, checkPassed: boolean) => {
+    const { output: turns } = await trimHistory(1).apply(
+      checked(output, checkPassed),
+      { state, trigger: "context" },
+    );
+    const result = turns
+      .flatMap((turn) => turn.content)
+      .find(
+        (block) =>
+          block.type === "tool_result" && block.callId === "call-check",
+      );
+    expect(JSON.stringify(result)).not.toContain(output.slice(0, 20));
+    const actions = await subject.decide(
+      {
+        type: "inference.done",
+        turn: replyTurn,
+        usage: state.tokenUsage,
+        source: { sourceId: "openai", provider: "openai", model: "gpt-5" },
+      },
+      { ...state, turns: [...turns, replyTurn] },
+      createCapabilities(),
+    );
+    return (Array.isArray(actions) ? actions : [actions]) as ReactorAction[];
+  };
+  const checklist = "\n✓ every page opened with no errors".repeat(40);
+
+  const passed = await decideTrimmed(
+    `check passed: built, started, and opened / at 1280 and 390 px with no errors${checklist}`,
+    true,
+  );
+  expect(passed.some((action) => action.type === "reply")).toBe(true);
+
+  const failed = await decideTrimmed(
+    `check failed: 1 problem(s) across 1 page(s)${checklist}`,
+    false,
+  );
+  expect(failed.some((action) => action.type === "reply")).toBe(false);
+  expect(failed.some((action) => action.type === "compact")).toBe(true);
+});
+
 test("check_design validates DESIGN.md and .dark in styles.css and auto-files the design option on pass", async () => {
   const starter = designStarterFiles();
   const modernSleek = designThemes()["modern-sleek"]!;

apps/hub/src/workflows/director.ts

@@ -10,7 +10,7 @@ import type {
   RetryPolicy,
 } from "@intx/types/runtime";
 import { REPETITION_ABORT } from "../inference/repetition.ts";
-import { CHECK_DESIGN_TOOL } from "../builds/design-check.ts";
+import { readCheckDetail, type CheckDetail } from "../builds/check-detail.ts";
 import { COMPACTOR, DROP_DEGENERATE } from "./compaction.ts";
 
 // About 1 coder turn in 100 skipped reasoning and repeated one sentence ("Ok to write. Let me
@@ -144,11 +144,11 @@ const SHELL_WRITE =
 const WRITE_TOOLS = new Set(["write_file", "edit_file"]);
 // The current task's calls, newest first: everything after the last mail the agent was handed.
 function taskCalls(turns: readonly ConversationTurn[]) {
-  const results = new Map<string, { text: string; isError: boolean }>();
+  const results = new Map<string, { detail: unknown; isError: boolean }>();
   const calls: {
     name: string;
     command: unknown;
-    result: string | undefined;
+    check: CheckDetail | undefined;
     isError: boolean;
   }[] = [];
   for (let index = turns.length - 1; index >= 0; index--) {
@@ -161,15 +161,15 @@ function taskCalls(turns: readonly ConversationTurn[]) {
     for (const block of [...turn.content].reverse()) {
       if (block.type === "tool_result")
         results.set(block.callId, {
-          text: textOf(block.content),
+          detail: block.detail,
           isError: block.isError === true,
         });
       if (block.type === "tool_call") {
         const result = results.get(block.id);
         calls.push({
           name: block.name,
           command: (block.arguments as Record<string, unknown>)["command"],
-          result: result?.text,
+          check: readCheckDetail(result?.detail),
           isError: result?.isError ?? true,
         });
       }
@@ -185,18 +185,12 @@ const ranShell = (call: TaskCall, pattern: RegExp) =>
   typeof call.command === "string" &&
   pattern.test(call.command);
 
-const says = (call: TaskCall, pattern: RegExp) =>
-  call.result !== undefined && pattern.test(call.result);
-
 // Run 36's coder passed check-app, kept editing, broke the app and would have replied done; a reply
 // counts as done only when no file changed after the last passing check. check_design is the
 // designer's check-app; file_design_option files only a design it passed as it stands.
 function unchecked(turns: readonly ConversationTurn[]): boolean {
   for (const call of taskCalls(turns)) {
-    if (ranShell(call, /check-app/))
-      return call.isError || !says(call, /check passed/);
-    if (call.name === CHECK_DESIGN_TOOL)
-      return call.isError || !says(call, /check passed/);
+    if (call.check) return call.isError || !call.check.checkPassed;
     if (call.name === "file_design_option") return call.isError;
     if (WRITE_TOOLS.has(call.name) || ranShell(call, SHELL_WRITE)) return true;
   }
@@ -208,7 +202,7 @@ const filed = (call: TaskCall) =>
   (call.name === "write_artifact" ||
     call.name === "file_package" ||
     call.name === "file_design_option" ||
-    (call.name === CHECK_DESIGN_TOOL && says(call, /Filed design_option:/)));
+    call.check?.filed === true);
 
 // A writer on 2026-10-30 replied with the whole approaches document as text and never filed it;
 // the coordinator waited on a writer that had already answered. A designer's reply also waits for a