Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 82 additions & 3 deletions bench/cdeb/orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
import { createHash } from "node:crypto";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

import {
executeAgentRun,
Expand All @@ -27,7 +28,11 @@ import {
import type { CapabilityGatePassed } from "./runtime/isolation.ts";
import { readExposureEvents, exposureLogSha256 } from "./runtime/exposure.ts";
import { readPersistedRawNdjson, readProviderLedger, type ProviderLedger } from "./runtime/provider-ledger.ts";
import { assertCaptureSurfaceAbsent, writeCdebArmConfig } from "./runtime/arm-settings.ts";
import {
assertCaptureSurfaceAbsent,
assertFrozenShippingProxy,
writeCdebArmConfig,
} from "./runtime/arm-settings.ts";
import { materializeBundle, type RepositoryBundleIdentity } from "./freeze/repository-bundle.ts";
import { freezeFinalTree, frozenTreeProvenance } from "./evaluator/freeze-tree.ts";
import { runEvaluatorOci } from "./evaluator/runner-oci.ts";
Expand Down Expand Up @@ -57,6 +62,10 @@ export type LifecycleState =
export const MAX_PRE_AGENT_ATTEMPTS = 3;
export const MAX_EVALUATOR_ATTEMPTS = 3;

const HERE = dirname(fileURLToPath(import.meta.url));
const SHIPPING_PROXY_PATH = join(HERE, "runtime", "shipping-proxy.ts");
const EXPOSURE_PARSER_PATH = join(HERE, "runtime", "exposure.ts");

const conditionSuffix = (condition: CdebCondition): "on" | "off" =>
condition === "commitlore-on" ? "on" : "off";

Expand Down Expand Up @@ -262,6 +271,8 @@ export interface LogicalRunPlan {
readonly condition: CdebCondition;
readonly repeat: 1 | 2 | 3;
readonly order: number;
/** Opaque analyzer input path committed by public-freeze.json. */
readonly analysis_row_file: string;
/** Required to re-parse retained CDEB-05 bytes during evaluator-only resume. */
readonly requested_model: string;
readonly prompt: string;
Expand Down Expand Up @@ -339,6 +350,11 @@ export const materializedWorkspacePreparer = (
export interface RunStudyOptions {
readonly storage: DurableStudyStorage;
readonly progress?: ProgressReporter;
/** Test-only path override for a copied, byte-mutated proxy fixture. */
readonly shipping_proxy_paths?: {
readonly proxy_path: string;
readonly parser_path: string;
};
}

export interface StudyRunResult {
Expand Down Expand Up @@ -378,6 +394,59 @@ const requiredRunId = (plan: LogicalRunPlan): void => {
}
};

interface FreezeWiring {
readonly hook_proxy_sha256: string;
readonly analysis_row_files: readonly string[];
}

const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);

/**
* Reads the two freeze commitments this coordinator must enforce itself.
* Keeping this narrow avoids turning the runner into a second freeze schema
* parser while making it impossible to omit either seam at execution time.
*/
const frozenWiring = (freeze: unknown): FreezeWiring => {
if (!isRecord(freeze)) throw new Error("CDEB public freeze is not an object with execution wiring");
const proxy = freeze["hook_proxy_sha256"];
if (typeof proxy !== "string" || !/^[0-9a-f]{64}$/u.test(proxy)) {
throw new Error("CDEB public freeze has no valid hook_proxy_sha256");
}
const analysis = freeze["analysis_inputs"];
if (!isRecord(analysis) || !Array.isArray(analysis["row_files"])) {
throw new Error("CDEB public freeze has no analysis_inputs.row_files");
}
const rowFiles = analysis["row_files"];
if (rowFiles.some((path) => typeof path !== "string" || !/^rows\/[a-z0-9][a-z0-9._-]*\.json$/u.test(path))) {
throw new Error("CDEB public freeze names an unsafe analysis row path");
}
if (new Set(rowFiles).size !== rowFiles.length) {
throw new Error("CDEB public freeze names an analysis row path more than once");
}
return { hook_proxy_sha256: proxy, analysis_row_files: rowFiles as readonly string[] };
};

/** The sealed schedule maps every logical observation to one frozen row path. */
const analysisRowsForPlan = (plan: CdebStudyPlan, wiring: FreezeWiring): ReadonlyMap<string, string> => {
const named = new Map<string, string>();
for (const logicalRun of plan.logical_runs) {
if (named.has(logicalRun.logical_run_id)) {
throw new Error(`CDEB logical run schedule has duplicate id ${logicalRun.logical_run_id}`);
}
if (!wiring.analysis_row_files.includes(logicalRun.analysis_row_file)) {
throw new Error(
`CDEB ${logicalRun.logical_run_id} writes ${logicalRun.analysis_row_file}, which public-freeze.json does not name`,
);
}
named.set(logicalRun.logical_run_id, logicalRun.analysis_row_file);
}
if (new Set(named.values()).size !== named.size || named.size !== wiring.analysis_row_files.length) {
throw new Error("CDEB sealed schedule and public freeze do not name the same one-to-one analysis row set");
}
return named;
};

/** Binds the sealed task mapping to the committed opaque block order. */
const assertBlockedSchedule = (plan: CdebStudyPlan): void => {
if (plan.randomization.algorithm !== "sha256-key-sort-v1" || plan.randomization.schema_version !== 1) {
Expand Down Expand Up @@ -505,7 +574,7 @@ const finalizeMeasuredRow = (
evaluator,
evaluator_attempts: evaluatorAttempts,
});
storage.writeRow(plan.logical_run_id, row);
storage.writeRow(plan.logical_run_id, plan.analysis_row_file, row);
};

const evaluateFrozenTree = async (
Expand Down Expand Up @@ -770,6 +839,15 @@ export const runStudy = async (
dependencies: OrchestratorDependencies,
options: RunStudyOptions,
): Promise<StudyRunResult> => {
const wiring = frozenWiring(plan.public_freeze);
const analysisRows = analysisRowsForPlan(plan, wiring);
const shippingPaths = options.shipping_proxy_paths ?? {
proxy_path: SHIPPING_PROXY_PATH,
parser_path: EXPOSURE_PARSER_PATH,
};
// The bytes the arm would execute are checked before an attempt checkpoint
// exists. A modified observer is a changed experiment, never a row.
assertFrozenShippingProxy(wiring.hook_proxy_sha256, shippingPaths.proxy_path, shippingPaths.parser_path);
if (plan.logical_runs.length === 0) throw new Error("CDEB study has no logical runs");
if (plan.randomization.block_count * 2 !== plan.logical_runs.length) {
throw new Error("CDEB randomization block count does not match its logical run schedule");
Expand All @@ -785,6 +863,7 @@ export const runStudy = async (
options.storage.repairBackupMirrors();
options.storage.ensureCommittedJson("public-freeze.json", plan.public_freeze);
options.storage.ensureCommittedJson("randomization.json", plan.randomization);
options.storage.reconcileNamedRows(analysisRows);
const completedBefore = options.storage.completedRows(expectedIds);
let completed = completedBefore.size;
for (const logicalRun of plan.logical_runs) {
Expand Down
76 changes: 62 additions & 14 deletions bench/cdeb/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,13 @@ const logicalIdPathSafe = (logicalRunId: string): void => {
}
};

/** The public freeze may name only opaque, flat analysis rows. */
const analysisRowPathSafe = (path: string): void => {
if (!/^rows\/[a-z0-9][a-z0-9._-]*\.json$/u.test(path)) {
throw new ImmutableArtifactError(`invalid freeze-named analysis row path ${JSON.stringify(path)}`);
}
};

const attemptIdPathSafe = (attemptId: string): void => {
if (!/^[a-z0-9-]+__[a-z0-9-]+__(on|off)__r[1-3]__a[1-9][0-9]*$/u.test(attemptId)) {
throw new ImmutableArtifactError(`invalid attempt id ${JSON.stringify(attemptId)}`);
Expand Down Expand Up @@ -247,6 +254,29 @@ export class DurableStudyStorage {
this.writeNew(relativePath, jsonBytes(value));
}

/**
* Finish a row publication interrupted between its two immutable views.
* Both views carry the same bytes: the per-run `row.json` binds the row to its
* evidence, while the freeze-named `rows/*.json` file is the analyzer's
* only permitted input. Either existing copy is evidence; divergent copies
* are an integrity failure, never a choice for recovery to make.
*/
private writeOrMatch(relativePath: string, bytes: Buffer): void {
const primary = this.absolute(relativePath);
const backup = this.absolute(relativePath, this.backupDir);
if (existsSync(primary)) {
if (!readFileSync(primary).equals(bytes)) {
throw new ImmutableArtifactError(`existing artifact differs from immutable row publication ${relativePath}`);
}
this.mirrorExisting(relativePath);
return;
}
if (existsSync(backup) && !readFileSync(backup).equals(bytes)) {
throw new ImmutableArtifactError(`backup artifact differs from immutable row publication ${relativePath}`);
}
this.writeNew(relativePath, bytes);
}

public readJson<T>(relativePath: string): T | null {
const path = this.absolute(relativePath);
return existsSync(path) ? parseJsonFile<T>(path, relativePath) : null;
Expand Down Expand Up @@ -438,19 +468,43 @@ export class DurableStudyStorage {
this.writeJsonNew(this.runRelative(logicalRunId, "evaluator.json"), value);
}

/** The row is the final commit record. It can never be replaced. */
public writeRow(logicalRunId: string, row: Record<string, unknown>): void {
/**
* The row is the final commit record. The per-run and analysis views are
* byte-identical immutable publications of one observation, not two rows.
*/
public writeRow(logicalRunId: string, analysisRowPath: string, row: Record<string, unknown>): void {
const named = row["logical_run_id"];
if (named !== logicalRunId) {
throw new ImmutableArtifactError(`row logical_run_id does not match its directory for ${logicalRunId}`);
}
const mirrorRow = this.absolute(join("rows", `${logicalRunId}.json`));
if (existsSync(mirrorRow)) {
// `rows/` is a legacy alternative verifier input, not a second output
// path. Writing both would look exactly like a duplicate observation.
throw new ImmutableArtifactError(`rows/${logicalRunId}.json already exists; refusing a duplicate logical row`);
analysisRowPathSafe(analysisRowPath);
const bytes = jsonBytes(row);
// The analyzer-facing name lands first. A kill before the run-local copy
// is recovered by `reconcileNamedRows` before resume decides what is done.
this.writeOrMatch(analysisRowPath, bytes);
this.writeOrMatch(this.runRelative(logicalRunId, "row.json"), bytes);
}

/** Repairs a killed row publication without re-running its observation. */
public reconcileNamedRows(namedRows: ReadonlyMap<string, string>): void {
const paths = new Set<string>();
for (const [logicalRunId, analysisRowPath] of namedRows) {
logicalIdPathSafe(logicalRunId);
analysisRowPathSafe(analysisRowPath);
if (paths.has(analysisRowPath)) {
throw new ImmutableArtifactError(`freeze names ${analysisRowPath} for more than one logical row`);
}
paths.add(analysisRowPath);
const runPath = this.runRelative(logicalRunId, "row.json");
const primaryRun = this.absolute(runPath);
const primaryAnalysis = this.absolute(analysisRowPath);
const hasRun = existsSync(primaryRun);
const hasAnalysis = existsSync(primaryAnalysis);
if (!hasRun && !hasAnalysis) continue;
const bytes = hasRun ? readFileSync(primaryRun) : readFileSync(primaryAnalysis);
this.writeOrMatch(runPath, bytes);
this.writeOrMatch(analysisRowPath, bytes);
}
this.writeJsonNew(this.runRelative(logicalRunId, "row.json"), row);
}

public readFinalTree(logicalRunId: string): FinalTreeArtifact | null {
Expand Down Expand Up @@ -531,12 +585,6 @@ export class DurableStudyStorage {
rows.set(id, row);
}
}
const legacyRows = this.absolute("rows");
if (existsSync(legacyRows) && readdirSync(legacyRows).length > 0) {
// The verifier treats `rows/` and `runs/*/row.json` as alternative input
// layouts; accepting both would create an unresolvable duplicate.
throw new ImmutableArtifactError("rows/ is populated alongside run directories; refusing duplicate row surfaces");
}
return rows;
}

Expand Down
33 changes: 28 additions & 5 deletions bench/cdeb/verify.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

import { createHash } from "node:crypto";
import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
import { join, dirname } from "node:path";
import { join, dirname, relative } from "node:path";
import { fileURLToPath } from "node:url";
import { zstdDecompressSync } from "node:zlib";

Expand Down Expand Up @@ -273,12 +273,14 @@ const verifyStudy = (root, studyName) => {
const freezePath = join(dir, "public-freeze.json");
let expectedRuns = null;
let freeze = null;
let freezeNamedRows = null;
if (!existsSync(freezePath)) {
fail(study, "public-freeze.json is missing — rows without a freeze manifest commit to nothing");
} else {
freeze = readJson(study, freezePath);
if (freeze !== null && validateAgainst(study, "study", freezePath, freeze)) {
expectedRuns = freeze.expected_logical_runs;
freezeNamedRows = new Set(freeze.analysis_inputs.row_files);
}
}
if (!existsSync(join(dir, "randomization.json"))) {
Expand All @@ -297,17 +299,38 @@ const verifyStudy = (root, studyName) => {
}
}

const seenIds = new Map(); // logical_run_id -> path
const seenIds = new Map(); // logical_run_id -> { path, row }

const isRunRow = (path) => {
const normalized = relative(dir, path).split("\\").join("/");
return /^runs\/[^/]+\/row\.json$/.test(normalized);
};

const isFreezeNamedRow = (path) => {
if (freezeNamedRows === null) return false;
const normalized = relative(dir, path).split("\\").join("/");
return freezeNamedRows.has(normalized);
};

const verifyRow = (path) => {
const row = readJson(study, path);
if (!validateAgainst(study, "result", path, row)) return null;
checkDerived(study, path, row);
checkExposure(study, path, row);
if (seenIds.has(row.logical_run_id)) {
fail(study, `duplicate logical_run_id ${row.logical_run_id} in ${path} and ${seenIds.get(row.logical_run_id)}`);
const existing = seenIds.get(row.logical_run_id);
if (existing !== undefined) {
// CDEB-09 publishes a byte-identical per-run audit copy and the opaque,
// freeze-named analyzer input. They are one logical observation, not a
// duplicate row. Any different bytes, two run rows, or an unregistered
// rows/ file remains a duplicate finding.
const pairedViews =
(isRunRow(path) && isFreezeNamedRow(existing.path)) ||
(isRunRow(existing.path) && isFreezeNamedRow(path));
if (!pairedViews || JSON.stringify(existing.row) !== JSON.stringify(row)) {
fail(study, `duplicate logical_run_id ${row.logical_run_id} in ${path} and ${existing.path}`);
}
} else {
seenIds.set(row.logical_run_id, path);
seenIds.set(row.logical_run_id, { path, row });
}
if (expectedIds !== null && !expectedIds.has(row.logical_run_id)) {
fail(study, `${path}: logical_run_id ${row.logical_run_id} is not in the randomization's expected set`);
Expand Down
35 changes: 24 additions & 11 deletions test/cdeb-orchestrator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { dirname, join, resolve } from "node:path";

import { afterAll, describe, expect, it } from "vitest";

Expand All @@ -30,6 +30,7 @@ import {
type PreparedWorkspace,
} from "../bench/cdeb/orchestrator.ts";
import { readProviderLedger } from "../bench/cdeb/runtime/provider-ledger.ts";
import { shippingProxySha256 } from "../bench/cdeb/runtime/arm-settings.ts";
import { DurableStudyStorage, SimulatedProcessKill } from "../bench/cdeb/storage.ts";
import { FIXTURE_ROOT, SEALED_DIR, TASK_ID, TEST_IMAGE_DIGEST } from "./cdeb-evaluator-helpers.ts";

Expand All @@ -51,6 +52,8 @@ const HEX = "a".repeat(64);
const OID = "b".repeat(40);
const RECORDED_STREAM = (): Buffer => readFileSync("test/fixtures/claude-stream/partial-messages.jsonl");
const RECORDED_MODEL = "claude-haiku-4-5-20251001";
const PROXY = resolve("bench/cdeb/runtime/shipping-proxy.ts");
const PARSER = resolve("bench/cdeb/runtime/exposure.ts");

interface Counters {
readonly agent: Map<string, number>;
Expand Down Expand Up @@ -146,6 +149,7 @@ const makePlan = (condition: CdebCondition, order: number): LogicalRunPlan => {
condition,
repeat: 1,
order,
analysis_row_file: `rows/${logical_run_id}.json`,
requested_model: RECORDED_MODEL,
prompt: "Fix calc without running external services.",
expected_record_ids: [],
Expand Down Expand Up @@ -210,16 +214,25 @@ const makePlan = (condition: CdebCondition, order: number): LogicalRunPlan => {
};
};

const studyPlan = (): CdebStudyPlan => ({
public_freeze: { benchmark: "cdeb-v1", study_id: "cdeb-orchestrator-test", freeze: "fixture" },
randomization: {
schema_version: 1,
algorithm: "sha256-key-sort-v1",
block_count: 1,
blocks: [{ block_index: "block-000", conditions: ["commitlore-on", "commitlore-off"] }],
},
logical_runs: [makePlan("commitlore-on", 1), makePlan("commitlore-off", 2)],
});
const studyPlan = (): CdebStudyPlan => {
const logical_runs = [makePlan("commitlore-on", 1), makePlan("commitlore-off", 2)];
return {
public_freeze: {
benchmark: "cdeb-v1",
study_id: "cdeb-orchestrator-test",
freeze: "fixture",
hook_proxy_sha256: shippingProxySha256(PROXY, PARSER),
analysis_inputs: { row_files: logical_runs.map((run) => run.analysis_row_file) },
},
randomization: {
schema_version: 1,
algorithm: "sha256-key-sort-v1",
block_count: 1,
blocks: [{ block_index: "block-000", conditions: ["commitlore-on", "commitlore-off"] }],
},
logical_runs,
};
};

const dependencies = (counts: Counters, options: { failEvaluatorFirst?: boolean; preTurnFailures?: number } = {}): OrchestratorDependencies => ({
prepare_workspace: async () => workspaceFor(),
Expand Down
Loading
Loading