perf(tests): cut fixed wall-clock waits from the slowest files

Author: Gustav-Simonsson

scripts/ci-timings.json

@@ -15,12 +15,16 @@ function processAlive(pid: number): boolean {
 }
 
 // These probes are `bun -e` one-liners, so they never import project modules
-// and finish in seconds. Stall windows are ~1.5s: under the `--parallel`
-// load this runner exists to enable, a delayed output tick can look like a
-// stall against a tighter window, so the window keeps several ticks of
-// scheduling headroom while a broken never-resetting timer still fires
-// mid-run (the tick test's total runtime exceeds the window).
+// and finish in seconds. The tick test keeps a ~1.5s stall window: under the
+// `--parallel` load this runner exists to enable, a delayed output tick can
+// look like a stall against a tighter window, so the window keeps several
+// ticks of scheduling headroom while a broken never-resetting timer still
+// fires mid-run (the tick test's total runtime exceeds the window). The
+// remaining tests drive children that never emit output, so their stall is
+// declared deterministically on the watchdog's first tick; they use a short
+// window so the suite does not pay the tick test's headroom in wall clock.
 const TEST_STALL_MS = 1_500;
+const SILENT_CHILD_STALL_MS = 400;
 
 describe("runWithWatchdog", () => {
   test("passes through a successful run without retrying", async () => {
@@ -58,7 +62,7 @@ describe("runWithWatchdog", () => {
       command: process.execPath,
       args: [
         "-e",
-        "for (let i = 0; i < 10; i++) { console.log('tick', i); await new Promise(r => setTimeout(r, 250)); }",
+        "for (let i = 0; i < 10; i++) { console.log('tick', i); await new Promise(r => setTimeout(r, 200)); }",
       ],
       stallMs: TEST_STALL_MS,
     });
@@ -83,7 +87,7 @@ describe("runWithWatchdog", () => {
     const result = await runWithWatchdog({
       command: process.execPath,
       args: ["-e", code],
-      stallMs: TEST_STALL_MS,
+      stallMs: SILENT_CHILD_STALL_MS,
       onStall: (attempt, max) => stalls.push([attempt, max]),
     });
     expect(result.exitCode).toBe(0);
@@ -100,9 +104,9 @@ describe("runWithWatchdog", () => {
       command: process.execPath,
       args: [
         "-e",
-        'process.on("SIGTERM", () => {}); setTimeout(() => process.exit(3), 4_000);',
+        'process.on("SIGTERM", () => {}); setTimeout(() => process.exit(3), 1_500);',
       ],
-      stallMs: TEST_STALL_MS,
+      stallMs: SILENT_CHILD_STALL_MS,
       onStall: () => {
         throw new Error("must not retry a run that exited on its own");
       },
@@ -120,7 +124,7 @@ describe("runWithWatchdog", () => {
     const result = await runWithWatchdog({
       command: process.execPath,
       args: ["-e", "setTimeout(() => {}, 30_000)"],
-      stallMs: TEST_STALL_MS,
+      stallMs: SILENT_CHILD_STALL_MS,
       attempts: 2,
       onStall: (attempt, max) => stalls.push([attempt, max]),
     });
@@ -144,7 +148,7 @@ describe("runWithWatchdog", () => {
     const result = await runWithWatchdog({
       command: process.execPath,
       args: ["-e", code],
-      stallMs: TEST_STALL_MS,
+      stallMs: SILENT_CHILD_STALL_MS,
       attempts: 1,
     });
     expect(result.exitCode).toBe(1);

scripts/test-parallel.test.ts

@@ -2,6 +2,17 @@ import { defined } from "../../tests/helpers/defined.js";
 import { describe, expect, test } from "bun:test";
 import { spawnSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
+
+/** Poll until no process carries `token`; fail instead of asserting on a pid. */
+async function waitUntilGone(token: string): Promise<void> {
+  const started = Date.now();
+  while (Date.now() - started < 5_000) {
+    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
+    if ((probe.stdout?.trim() ?? "").length === 0) return;
+    await new Promise((r) => setTimeout(r, 50));
+  }
+  throw new Error(`tagged child still alive after 5s: ${token}`);
+}
 import { createPermissionGate } from "../permission/gate.js";
 import { shellCollectDefinition } from "./background-shell-tool.js";
 import { createAgentToolset } from "./tools.js";
@@ -120,10 +131,7 @@ describe("background shell through the agent toolset", () => {
       expect(JSON.parse(String(cancelled.content))).toMatchObject({
         status: "cancelling",
       });
-      await new Promise((r) => setTimeout(r, 300));
-      const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-      expect(probe.stdout?.trim() ?? "").toBe("");
-      expect(probe.status).not.toBe(0);
+      await waitUntilGone(token);
     } finally {
       await toolset.dispose();
     }
@@ -221,10 +229,7 @@ describe("background shell through the agent toolset", () => {
     );
     expect(started.isError).not.toBe(true);
     await toolset.dispose();
-    await new Promise((r) => setTimeout(r, 300));
-    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-    expect(probe.stdout?.trim() ?? "").toBe("");
-    expect(probe.status).not.toBe(0);
+    await waitUntilGone(token);
   });
 });
 

src/agent/background-shell-tool.test.ts

@@ -164,10 +164,13 @@ describe("createAuthStore", () => {
   test("gives queued same-process writes their own lock window", async () => {
     const home = await mkdtemp(join(tmpdir(), "oauth-store-queue-"));
     try {
+      // A short lock window keeps the queued write's handoff fast without
+      // changing what is asserted; production keeps the 1s default.
       const store = createAuthStore<TestTokens>({
         filename: "test-auth.json",
         settingsDirName: TEST_SETTINGS_DIR,
         isTokens: isTestTokens,
+        lockTimeoutMs: 100,
       });
       await store.saveProfile(
         {
@@ -294,10 +297,12 @@ describe("createAuthStore", () => {
   test("fails closed with manual recovery guidance when an orphan lock exists", async () => {
     const home = await mkdtemp(join(tmpdir(), "oauth-store-orphan-"));
     try {
+      // A short lock window keeps the timeout fast; production keeps 1s.
       const store = createAuthStore<TestTokens>({
         filename: "test-auth.json",
         settingsDirName: TEST_SETTINGS_DIR,
         isTokens: isTestTokens,
+        lockTimeoutMs: 100,
       });
       const lockPath = `${store.authPath(home)}.lock`;
       await mkdir(join(home, TEST_SETTINGS_DIR), { recursive: true });

src/auth/store.test.ts

@@ -40,6 +40,12 @@ export interface AuthStoreOptions<TTokens extends BaseTokens> {
   filename: string;
   settingsDirName: string;
   isTokens: (value: unknown) => value is TTokens;
+  /**
+   * Override the credential-lock wait. Tests pass a short window so they do
+   * not pay the production 1s in wall clock; production never sets it and
+   * keeps the default.
+   */
+  lockTimeoutMs?: number;
 }
 
 interface AuthFile<TTokens extends BaseTokens> {
@@ -137,7 +143,7 @@ export function createAuthStore<TTokens extends BaseTokens>(
     const path = authPath(home);
     const lockPath = `${path}.lock`;
     await mkdir(dirname(path), { recursive: true, mode: 0o700 });
-    const deadline = Date.now() + LOCK_TIMEOUT_MS;
+    const deadline = Date.now() + (options.lockTimeoutMs ?? LOCK_TIMEOUT_MS);
     let lock;
 
     while (true) {

src/auth/store.ts

@@ -27,6 +27,17 @@ import {
 
 const neverAbort = () => new AbortController().signal;
 
+/** Poll until no process carries `token`; fail instead of asserting on a pid. */
+async function waitUntilGone(token: string): Promise<void> {
+  const started = Date.now();
+  while (Date.now() - started < 5_000) {
+    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
+    if ((probe.stdout?.trim() ?? "").length === 0) return;
+    await new Promise((r) => setTimeout(r, 50));
+  }
+  throw new Error(`tagged child still alive after 5s: ${token}`);
+}
+
 function toolContentTrimmed(result: ToolResult): string {
   const content = result.content;
   return (typeof content === "string" ? content : String(content)).trim();
@@ -170,10 +181,7 @@ describe("runGuardedShell", () => {
     const token = `ic_guard_orphan_${randomUUID()}`;
     const cmd = `bash -c 'IC_GUARD_TAG=${token} sleep 600 & IC_GUARD_TAG=${token} exec sleep 600'`;
     await runGuardedShell({ command: cmd, timeout: 250 }, neverAbort());
-    await new Promise((r) => setTimeout(r, 300));
-    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-    expect(probe.stdout?.trim() ?? "").toBe("");
-    expect(probe.status).not.toBe(0);
+    await waitUntilGone(token);
   });
 
   test("abort kills grandchildren in a shell pipeline", async () => {
@@ -187,10 +195,7 @@ describe("runGuardedShell", () => {
     );
     setTimeout(() => controller.abort(), 80);
     await expect(promise).rejects.toThrow(/aborted/);
-    await new Promise((r) => setTimeout(r, 300));
-    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-    expect(probe.stdout?.trim() ?? "").toBe("");
-    expect(probe.status).not.toBe(0);
+    await waitUntilGone(token);
   });
 
   test("abort kills the process group", async () => {
@@ -1095,10 +1100,7 @@ describe("shellGuardPlugin", () => {
       ).not.toBe("");
       expect(plugin.dispose).toBeDefined();
       await defined(plugin.dispose)();
-      await new Promise((r) => setTimeout(r, 300));
-      const after = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-      expect(after.stdout?.trim() ?? "").toBe("");
-      expect(after.status).not.toBe(0);
+      await waitUntilGone(token);
       await defined(plugin.dispose)();
       await running;
     } finally {
@@ -1218,8 +1220,8 @@ describe("shellGuardPlugin", () => {
       signalCode: null,
       kill: () => true,
     }) as ChildProcess;
-    await expect(reapLiveChildren(new Set([child]))).rejects.toThrow(
-      /still live after 2000ms reap/,
+    await expect(reapLiveChildren(new Set([child]), 60)).rejects.toThrow(
+      /still live after 60ms reap/,
     );
   }, 10_000);
 });

src/plugins/shell-guard-plugin.test.ts

@@ -279,16 +279,19 @@ function childStillLive(child: ChildProcess): boolean {
 // Abort SIGKILLs the process group immediately (runGuardedShell onAbort). This
 // window is only a backstop for children still tracked at dispose. Leftovers
 // after it must fail teardown; do not stretch the process-exit 2s deadline.
+// Tests pass a short window so they do not pay the production 2s in wall
+// clock; production never passes it and keeps the default.
 export async function reapLiveChildren(
   liveChildren: Set<ChildProcess>,
+  reapWindowMs: number = SHELL_GUARD_DISPOSE_REAP_MS,
 ): Promise<void> {
   const remaining = [...liveChildren];
   for (const child of remaining) killProcessTree(child);
   if (remaining.length === 0) return;
   const closed = Promise.all(remaining.map(waitChildClose));
   let timer: ReturnType<typeof setTimeout> | undefined;
   const timedOut = new Promise<"timeout">((resolve) => {
-    timer = setTimeout(() => resolve("timeout"), SHELL_GUARD_DISPOSE_REAP_MS);
+    timer = setTimeout(() => resolve("timeout"), reapWindowMs);
   });
   try {
     const winner = await Promise.race([
@@ -299,7 +302,7 @@ export async function reapLiveChildren(
     const stillLive = remaining.filter(childStillLive);
     if (stillLive.length > 0) {
       throw new Error(
-        `${stillLive.length} shell child process${stillLive.length === 1 ? "" : "es"} still live after ${SHELL_GUARD_DISPOSE_REAP_MS}ms reap`,
+        `${stillLive.length} shell child process${stillLive.length === 1 ? "" : "es"} still live after ${reapWindowMs}ms reap`,
       );
     }
   } finally {

src/plugins/shell-guard-plugin.ts

@@ -103,7 +103,7 @@ describe("lifecycle hook payload delivery", () => {
     process.on("unhandledRejection", onUnhandled);
     try {
       await manager.dispatchPostRun(summary);
-      await new Promise((resolve) => setTimeout(resolve, 200));
+      await new Promise((resolve) => setTimeout(resolve, 100));
     } finally {
       process.off("unhandledRejection", onUnhandled);
     }

src/session/hooks.test.ts

@@ -10,11 +10,22 @@ import {
 
 const tmpCwd = process.cwd();
 
+/** Poll until no process carries `token`; fail instead of asserting on a pid. */
+async function waitUntilGone(token: string): Promise<void> {
+  const started = Date.now();
+  while (Date.now() - started < 5_000) {
+    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
+    if ((probe.stdout?.trim() ?? "").length === 0) return;
+    await new Promise((r) => setTimeout(r, 50));
+  }
+  throw new Error(`tagged child still alive after 5s: ${token}`);
+}
+
 describe("background shell registry", () => {
   test("start returns a handle immediately while the process runs", async () => {
     const registry = createBackgroundShellRegistry();
     const started = registry.start({
-      command: "sleep 1; echo done",
+      command: "sleep 0.5; echo done",
       cwd: tmpCwd,
     });
     if ("error" in started) throw new Error(started.error);
@@ -36,7 +47,10 @@ describe("background shell registry", () => {
     const started = registry.start({ command: "echo hi", cwd: tmpCwd });
     if ("error" in started) throw new Error(started.error);
     await registry.collect(started.id, 5_000);
-    await new Promise((r) => setTimeout(r, 50));
+    const deadline = Date.now() + 5_000;
+    while (exits.length === 0 && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 20));
+    }
     expect(exits).toHaveLength(1);
   });
 
@@ -68,10 +82,7 @@ describe("background shell registry", () => {
     expect(registry.cancel(started.id)).toBe(true);
     const exited = await registry.collect(started.id, 5_000);
     expect(exited.state).toBe("completed");
-    await new Promise((r) => setTimeout(r, 300));
-    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-    expect(probe.stdout?.trim() ?? "").toBe("");
-    expect(probe.status).not.toBe(0);
+    await waitUntilGone(token);
   });
 
   test("cancel on an unknown id returns false", () => {
@@ -115,10 +126,7 @@ describe("background shell registry", () => {
     });
     if ("error" in started) throw new Error(started.error);
     registry.disposeAll("session closed");
-    await new Promise((r) => setTimeout(r, 300));
-    const probe = spawnSync("pgrep", ["-f", token], { encoding: "utf8" });
-    expect(probe.stdout?.trim() ?? "").toBe("");
-    expect(probe.status).not.toBe(0);
+    await waitUntilGone(token);
     const after = await registry.collect(started.id, 0);
     expect(after.state).toBe("not-found");
   });

src/shell/background-shell.test.ts

@@ -34,7 +34,11 @@ import { join } from "node:path";
 import { tool } from "@intx/agent";
 import type { AgentTool } from "@intx/agent";
 import { type } from "arktype";
-import type { ToolDefinition, ToolResult } from "@intx/types/runtime";
+import type {
+  RetryPolicy,
+  ToolDefinition,
+  ToolResult,
+} from "@intx/types/runtime";
 import type { ReactorEmittedEvent } from "@intx/inference";
 import { getLogger } from "@intx/log";
 
@@ -643,6 +647,13 @@ export type AgentFleetDeps = SubAgentSandboxDeps & {
   telemetry?: Telemetry;
   /** Tests inject a private queue. Production omits and uses the process singleton. */
   admission?: AdmissionQueue;
+  /**
+   * Retry-timing overrides forwarded to each spawned run. Tests inject fast
+   * values so retry suites do not pay the production backoff; production
+   * omits them and keeps the defaults.
+   */
+  outerRetryDelayMs?: number;
+  retryPolicy?: RetryPolicy;
 };
 
 function resolveDep<T>(value: T | (() => T)): T {
@@ -1334,6 +1345,12 @@ export function createSpawnAgentTool(deps: AgentFleetDeps): AgentTool {
               : {}),
             signal: childCtl.signal,
             admission,
+            ...(deps.outerRetryDelayMs !== undefined
+              ? { outerRetryDelayMs: deps.outerRetryDelayMs }
+              : {}),
+            ...(deps.retryPolicy !== undefined
+              ? { retryPolicy: deps.retryPolicy }
+              : {}),
             onEvent,
             onRunSettled: (summary) => {
               settlement = summary;