fix(evals): contain completion-harness task paths and validate timeout

Confine custom task-set fixture/script/verify references to the harness directory, mark the version-controlled grading trust boundary, reject degenerate timeout values, and re-record the baseline against the current tree.

Author: TheGreatAxios

evals/completion/baseline-2026-09-14.json

@@ -1,9 +1,9 @@
 {
   "harness": "completion-baseline",
   "version": 1,
-  "startedAt": "2026-09-14T07:30:17.369Z",
-  "finishedAt": "2026-09-14T07:30:19.083Z",
-  "commitSha": "b509ed3edb0db5368eb190528d3beaee8895837b",
+  "startedAt": "2026-09-14T19:57:32.797Z",
+  "finishedAt": "2026-09-14T19:57:35.105Z",
+  "commitSha": "77b8c30b445655cba06cffd4ab8bbde1e8adee25",
   "provider": "stub-scripted",
   "model": "completion-baseline-v1",
   "repeats": 2,
@@ -25,8 +25,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 163,
-      "verifyDurationMs": 149,
+      "agentDurationMs": 252,
+      "verifyDurationMs": 157,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -45,8 +45,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 62,
-      "verifyDurationMs": 141,
+      "agentDurationMs": 95,
+      "verifyDurationMs": 257,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -65,8 +65,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 131,
-      "verifyDurationMs": 135,
+      "agentDurationMs": 92,
+      "verifyDurationMs": 154,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -85,8 +85,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 66,
-      "verifyDurationMs": 127,
+      "agentDurationMs": 69,
+      "verifyDurationMs": 152,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -105,8 +105,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 30,
-      "verifyDurationMs": 72,
+      "agentDurationMs": 86,
+      "verifyDurationMs": 93,
       "verifyExitCode": 1,
       "overBudget": false
     },
@@ -125,8 +125,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 22,
-      "verifyDurationMs": 65,
+      "agentDurationMs": 21,
+      "verifyDurationMs": 80,
       "verifyExitCode": 1,
       "overBudget": false
     },
@@ -145,8 +145,8 @@
       "doomLoopInterventions": 1,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 95,
-      "verifyDurationMs": 16,
+      "agentDurationMs": 66,
+      "verifyDurationMs": 19,
       "verifyExitCode": 2,
       "overBudget": false,
       "error": "reactor error: Doom loop detected: an identical tool batch (read_file) executed 3 times consecutively"
@@ -166,7 +166,7 @@
       "doomLoopInterventions": 1,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 65,
+      "agentDurationMs": 75,
       "verifyDurationMs": 17,
       "verifyExitCode": 2,
       "overBudget": false,
@@ -179,7 +179,7 @@
     "completedRuns": 4,
     "completionRate": 0.5,
     "meanTurnsToCompletion": 3,
-    "meanAgentDurationMs": 79.25,
+    "meanAgentDurationMs": 94.5,
     "totalRetries": 0,
     "totalCompactionEvents": 0,
     "totalDoomLoopInterventions": 2,

evals/completion/lib.test.ts

@@ -1,11 +1,13 @@
 import { describe, expect, test } from "bun:test";
 import {
   CompletionReport,
+  assertTaskSetContained,
   computeTotals,
   formatSummary,
   isCompletedRun,
   parseResponderScript,
   parseTaskSetFile,
+  resolveTaskRelativePath,
   TaskResult,
   type CompletionReport as CompletionReportType,
   type TaskResult as TaskResultType,
@@ -109,6 +111,59 @@ describe("boundary parsing", () => {
   });
 });
 
+describe("task path containment", () => {
+  const root = "/repo/evals/completion";
+
+  test("keeps relative fixture paths inside the harness directory", () => {
+    expect(resolveTaskRelativePath(root, "tasks/sum-fix/fixture")).toBe(
+      "/repo/evals/completion/tasks/sum-fix/fixture",
+    );
+  });
+
+  test("rejects absolute escapes", () => {
+    expect(() => resolveTaskRelativePath(root, "/etc/passwd")).toThrow(
+      /escapes the harness directory/,
+    );
+  });
+
+  test("rejects dot-dot escapes outside the harness directory", () => {
+    expect(() => resolveTaskRelativePath(root, "../capability/lib.ts")).toThrow(
+      /escapes the harness directory/,
+    );
+    expect(() =>
+      resolveTaskRelativePath(root, "tasks/../../package.json"),
+    ).toThrow(/escapes the harness directory/);
+  });
+
+  test("rejects an empty reference", () => {
+    expect(() => resolveTaskRelativePath(root, "  ")).toThrow(
+      /escapes the harness directory/,
+    );
+  });
+
+  test("assertTaskSetContained rejects a task set with an escaped grader", () => {
+    const taskSet = parseTaskSetFile({
+      version: 1,
+      note: "x",
+      tasks: [
+        {
+          id: "evil",
+          title: "evil",
+          profile: "solve",
+          prompt: "p",
+          fixture: "tasks/sum-fix/fixture",
+          script: "tasks/sum-fix/script.json",
+          verify: "/tmp/evil.sh",
+          maxTurns: 1,
+        },
+      ],
+    });
+    expect(() => assertTaskSetContained(taskSet, root)).toThrow(
+      /escapes the harness directory/,
+    );
+  });
+});
+
 describe("human summary", () => {
   test("names the harness, provenance, rate, and every run", () => {
     const results = [

evals/completion/lib.ts

@@ -3,8 +3,18 @@
  * No process I/O here: the runner (scripts/eval-completion.ts) owns the
  * filesystem, child processes, and the agent loop; this module owns the
  * shapes at the boundary (arktype) plus completion math and the summary.
+ *
+ * TRUST BOUNDARY: the version-controlled task and grading files under
+ * evals/completion/tasks/ (tasks.json, per-task script.json, verify.sh, and
+ * fixture/ copies) plus evals/completion/lib.ts and
+ * scripts/eval-completion.ts are the trusted grading boundary. Treat edits to
+ * those paths as grading changes requiring owner review; custom --tasks JSON
+ * passed on the CLI is untrusted input and its fixture/script/verify
+ * references stay confined to evals/completion/ via
+ * resolveTaskRelativePath (absolute paths and `..` escapes are rejected).
  */
 
+import { isAbsolute, relative, resolve } from "node:path";
 import { type } from "arktype";
 
 export const ResponderProfile = type("'solve' | 'decline' | 'stall'");
@@ -114,6 +124,36 @@ export function parseResponderScript(payload: unknown): ResponderScript {
   return ResponderScript.assert(payload);
 }
 
+/**
+ * Resolve a task-set fixture/script/verify reference against the harness
+ * task directory. Custom task sets are untrusted: absolute paths and `..`
+ * escapes outside the directory are rejected so fixture copies and grader
+ * spawns cannot reach outside evals/completion/.
+ */
+export function resolveTaskRelativePath(taskDir: string, ref: string): string {
+  if (ref.trim() === "" || isAbsolute(ref)) {
+    throw new Error(`Task path escapes the harness directory: ${ref}`);
+  }
+  const resolved = resolve(taskDir, ref);
+  const rel = relative(taskDir, resolved);
+  if (rel === "" || rel === ".." || rel.startsWith("../") || isAbsolute(rel)) {
+    throw new Error(`Task path escapes the harness directory: ${ref}`);
+  }
+  return resolved;
+}
+
+/** Assert every task-set path reference stays inside the harness directory. */
+export function assertTaskSetContained(
+  taskSet: CompletionTaskSet,
+  taskDir: string,
+): void {
+  for (const task of taskSet.tasks) {
+    resolveTaskRelativePath(taskDir, task.fixture);
+    resolveTaskRelativePath(taskDir, task.script);
+    resolveTaskRelativePath(taskDir, task.verify);
+  }
+}
+
 /**
  * Completion predicate shared by the runner and tests: a run counts as
  * complete only when the agent loop finished cleanly, the grader passed,

scripts/eval-completion.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { withTimeout } from "./eval-completion.js";
+import { parseArgs, withTimeout } from "./eval-completion.js";
 
 describe("withTimeout", () => {
   test("rejects a hung run after the timeout", async () => {
@@ -29,3 +29,15 @@ describe("withTimeout", () => {
     ).rejects.toThrow("boom");
   });
 });
+
+describe("parseArgs timeout validation", () => {
+  test("rejects NaN, zero, and negative timeouts", () => {
+    expect(() => parseArgs(["--timeout-ms", "NaN"])).toThrow(/--timeout-ms/);
+    expect(() => parseArgs(["--timeout-ms", "0"])).toThrow(/--timeout-ms/);
+    expect(() => parseArgs(["--timeout-ms", "-5"])).toThrow(/--timeout-ms/);
+  });
+
+  test("accepts a positive integer timeout", () => {
+    expect(parseArgs(["--timeout-ms", "1000"]).timeoutMs).toBe(1000);
+  });
+});

scripts/eval-completion.ts

@@ -14,6 +14,14 @@
  * purpose: solve/decline/stall profiles exercise finish, decline, and
  * guard-trip paths deterministically so the 0.4.x re-measure sees the
  * control layer move, not provider noise.
+ *
+ * TRUST BOUNDARY: the version-controlled files under evals/completion/tasks/
+ * (tasks.json, per-task script.json, verify.sh, fixture/) are the trusted
+ * grading boundary — edits there are grading changes requiring owner review.
+ * A custom --tasks JSON file is untrusted: its fixture/script/verify
+ * references are confined to evals/completion/ (see resolveTaskRelativePath;
+ * absolute paths and `..` escapes are rejected) before any fixture copy or
+ * grader spawn.
  */
 
 import { cpSync } from "node:fs";
@@ -29,11 +37,13 @@ import {
 } from "../tests/integration/harness.js";
 import {
   CompletionReport,
+  assertTaskSetContained,
   computeTotals,
   formatSummary,
   isCompletedRun,
   parseResponderScript,
   parseTaskSetFile,
+  resolveTaskRelativePath,
   TaskResult,
   type CompletionTask,
   type RunStatus,
@@ -64,7 +74,7 @@ function printUsage(): void {
   --help            Print this message`);
 }
 
-function parseArgs(argv: string[]): CliOptions {
+export function parseArgs(argv: string[]): CliOptions {
   const opts: CliOptions = {
     tasksPath: join(COMPLETION_ROOT, "tasks.json"),
     caseId: "all",
@@ -101,6 +111,11 @@ function parseArgs(argv: string[]): CliOptions {
       `--repeats must be a positive integer, got ${opts.repeats}`,
     );
   }
+  if (!Number.isInteger(opts.timeoutMs) || opts.timeoutMs <= 0) {
+    throw new Error(
+      `--timeout-ms must be a positive integer, got ${opts.timeoutMs}`,
+    );
+  }
   return opts;
 }
 
@@ -213,11 +228,15 @@ async function runTask(
   });
   const agentStart = Date.now();
   try {
-    cpSync(resolve(COMPLETION_ROOT, task.fixture), session.cwd, {
-      recursive: true,
-    });
+    cpSync(
+      resolveTaskRelativePath(COMPLETION_ROOT, task.fixture),
+      session.cwd,
+      {
+        recursive: true,
+      },
+    );
     const scriptRaw = await readFile(
-      resolve(COMPLETION_ROOT, task.script),
+      resolveTaskRelativePath(COMPLETION_ROOT, task.script),
       "utf8",
     );
     const script = parseResponderScript(JSON.parse(scriptRaw));
@@ -280,11 +299,15 @@ async function runTask(
     if (signals.runFailed) runStatus = "failed";
 
     const verifyStart = Date.now();
-    const verify = spawnSync("bash", [resolve(COMPLETION_ROOT, task.verify)], {
-      cwd: session.cwd,
-      encoding: "utf8",
-      timeout: 60_000,
-    });
+    const verify = spawnSync(
+      "bash",
+      [resolveTaskRelativePath(COMPLETION_ROOT, task.verify)],
+      {
+        cwd: session.cwd,
+        encoding: "utf8",
+        timeout: 60_000,
+      },
+    );
     const verifyDurationMs = Date.now() - verifyStart;
     const verifyExitCode = verify.status ?? 1;
 
@@ -330,6 +353,7 @@ async function main(): Promise<void> {
   const taskSet = parseTaskSetFile(
     JSON.parse(await readFile(opts.tasksPath, "utf8")),
   );
+  assertTaskSetContained(taskSet, COMPLETION_ROOT);
   const tasks =
     opts.caseId === "all"
       ? taskSet.tasks