Fix stale tier-xhard grader hash and guard locked fixtures (#936)

The tier-xhard grader pinned a hash its fixture no longer has, so every run failed at setup and blamed the agent for an edit it never made.

Author: TheGreatAxios

evals/capability/cases/tier-hard/verify.sh

@@ -13,6 +13,14 @@ TEST_SHA="a7ce0705273ad6b874b737d389db038745e5baec825fd757f42448cd1addb687"
 
 actual_sha=$(shasum -a 256 tests/report.test.ts | cut -d' ' -f1)
 if [[ "$actual_sha" != "$TEST_SHA" ]]; then
+  # A stale TEST_SHA is indistinguishable from an agent edit above, so check
+  # the fixture's own hash at setup before blaming the agent: the workdir is
+  # a git repo committed before the run, so HEAD holds the pristine file.
+  setup_sha=$(git show "HEAD:tests/report.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
+  if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
+    echo "FAIL: tests/report.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
+    exit 1
+  fi
   echo "FAIL: tests/report.test.ts was modified (contract file must be byte-unchanged)"
   exit 1
 fi

evals/capability/cases/tier-med/verify.sh

@@ -13,6 +13,14 @@ TEST_SHA="4b48fc362c6c3a72af405545ffa816f4a531bbed974d87b6f4d60cf5c96f06a0"
 
 actual_sha=$(shasum -a 256 tests/checkout.test.ts | cut -d' ' -f1)
 if [[ "$actual_sha" != "$TEST_SHA" ]]; then
+  # A stale TEST_SHA is indistinguishable from an agent edit above, so check
+  # the fixture's own hash at setup before blaming the agent: the workdir is
+  # a git repo committed before the run, so HEAD holds the pristine file.
+  setup_sha=$(git show "HEAD:tests/checkout.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
+  if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
+    echo "FAIL: tests/checkout.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
+    exit 1
+  fi
   echo "FAIL: tests/checkout.test.ts was modified (contract file must be byte-unchanged)"
   exit 1
 fi

evals/capability/cases/tier-xhard/verify.sh

@@ -5,13 +5,21 @@
 # at green fails every check below.
 set -euo pipefail
 
-TEST_SHA="e125ecc69453b923bb69b46e8411ca5b580e0afb81868a8e24f45a10d93e1d2a"
+TEST_SHA="bd97182202b283a721f2972f1fba79e2b552a95a64f1639d077022aa95a157d5"
 
 [[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; }
 [[ -f tests/notify.test.ts ]] || { echo "FAIL: tests/notify.test.ts is gone"; exit 1; }
 
 actual_sha=$(shasum -a 256 tests/notify.test.ts | cut -d' ' -f1)
 if [[ "$actual_sha" != "$TEST_SHA" ]]; then
+  # A stale TEST_SHA is indistinguishable from an agent edit above, so check
+  # the fixture's own hash at setup before blaming the agent: the workdir is
+  # a git repo committed before the run, so HEAD holds the pristine file.
+  setup_sha=$(git show "HEAD:tests/notify.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
+  if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
+    echo "FAIL: tests/notify.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
+    exit 1
+  fi
   echo "FAIL: tests/notify.test.ts was modified (contract file must be byte-unchanged)"
   exit 1
 fi

evals/capability/locked-fixtures.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, test } from "bun:test";
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadEvalCases, resolveFixturePath } from "./lib.js";
+
+const capabilityDir = dirname(fileURLToPath(import.meta.url));
+
+function parseLockedContract(
+  verifySrc: string,
+  caseId: string,
+): { sha: string; path: string } | null {
+  const sha = verifySrc.match(/^TEST_SHA="([0-9a-f]{64})"$/m)?.[1];
+  if (sha === undefined) return null;
+  const hashed = verifySrc.match(/shasum -a 256 (\S+)/)?.[1];
+  const guarded = [...verifySrc.matchAll(/\[\[ -f (\S+) \]\]/g)].map(
+    (m) => m[1],
+  );
+  if (hashed === undefined || !guarded.includes(hashed)) {
+    throw new Error(
+      `case ${caseId}: cannot determine the locked contract file from verify.sh`,
+    );
+  }
+  return { sha, path: hashed };
+}
+
+describe("locked fixture hashes", () => {
+  test("every TEST_SHA literal matches its fixture file", async () => {
+    const repoRoot = resolve(capabilityDir, "..", "..");
+    const cases = await loadEvalCases(join(capabilityDir, "cases"));
+    const failures: string[] = [];
+    let checked = 0;
+    for (const c of cases) {
+      const verifySrc = await readFile(join(c.caseDir, c.verify), "utf8");
+      const locked = parseLockedContract(verifySrc, c.id);
+      if (locked === null) continue;
+      checked += 1;
+      const bytes = await readFile(
+        join(resolveFixturePath(repoRoot, c.fixture), locked.path),
+      );
+      const actual = createHash("sha256").update(bytes).digest("hex");
+      if (actual !== locked.sha) {
+        failures.push(
+          `case ${c.id}: ${locked.path} hashes to ${actual} but verify.sh pins ${locked.sha}`,
+        );
+      }
+    }
+    expect(checked).toBeGreaterThan(0);
+    if (failures.length > 0) {
+      throw new Error(`stale locked hashes:\n${failures.join("\n")}`);
+    }
+  });
+});