diff --git a/bench/cdeb/evaluator/freeze-tree.ts b/bench/cdeb/evaluator/freeze-tree.ts index 3fb5952b..9df92824 100644 --- a/bench/cdeb/evaluator/freeze-tree.ts +++ b/bench/cdeb/evaluator/freeze-tree.ts @@ -21,6 +21,7 @@ * the OID rather than trusting the claim (ingest.ts). */ +import { spawnSync } from "node:child_process"; import { writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -36,6 +37,42 @@ export interface FrozenFinalTree { readonly staged_file_count: number; } +/** The extra §11.1 provenance CDEB-07 stores in `final-tree.json`. */ +export interface FrozenTreeProvenance { + readonly base_tree_oid: string; + readonly canonical_diff_sha256: string; + readonly workspace_status_digest: string; +} + +const FROZEN_GIT_ENV: Readonly> = { + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_ADVICE: "0", + GIT_OPTIONAL_LOCKS: "0", +}; + +const FROZEN_GIT_FLAGS = [ + "-c", "core.fsmonitor=false", + "-c", "core.autocrlf=false", + "-c", "core.symlinks=true", + "-c", "core.ignorecase=false", + "-c", "core.fileMode=true", +] as const; + +const gitOrThrow = (workdir: string, env: Record, args: readonly string[]): Buffer => { + const result = spawnSync("git", [...args], { + cwd: workdir, + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", ...env }, + encoding: "buffer", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`final tree provenance: git ${args.join(" ")} failed (${String(result.status)}): ${Buffer.from(result.stderr ?? Buffer.alloc(0)).toString("utf8").trim()}`); + } + return Buffer.from(result.stdout ?? Buffer.alloc(0)); +}; + /** * Freezes the agent's final working tree. `scratchDir` must be a fresh * directory the caller owns; nothing here writes inside `workdir`. @@ -55,6 +92,53 @@ export const freezeFinalTree = (workdir: string, scratchDir: string): FrozenFina }; }; +/** + * Captures the base→final binary diff and porcelain status from the actual + * materialized repository. It stages through a temporary index and asserts + * that its tree is the OID the hermetic freezer already produced; otherwise a + * "diff of the tree" would be a second, drifting implementation of §11.1. + * + * This is intentionally separate from `freezeFinalTree`: evaluator controls + * also freeze plain fixture directories with no Git history, while a measured + * CDEB workspace is always a materialized repository and therefore has a base. + */ +export const frozenTreeProvenance = ( + workdir: string, + scratchDir: string, + frozen: FrozenFinalTree, +): FrozenTreeProvenance => { + const indexPath = join(scratchDir, "cdeb-final-index"); + const env: Record = { + ...FROZEN_GIT_ENV, + GIT_INDEX_FILE: indexPath, + TMPDIR: scratchDir, + }; + const base_tree_oid = gitOrThrow(workdir, env, ["rev-parse", "HEAD^{tree}"]).toString("utf8").trim(); + gitOrThrow(workdir, env, [...FROZEN_GIT_FLAGS, "read-tree", "HEAD"]); + gitOrThrow(workdir, env, [...FROZEN_GIT_FLAGS, "add", "-A", "--", "."]); + const staged = gitOrThrow(workdir, env, [...FROZEN_GIT_FLAGS, "write-tree"]).toString("utf8").trim(); + if (staged !== frozen.final_tree_oid) { + throw new Error( + `final tree provenance: temporary-index OID ${staged} differs from hermetic freezer OID ${frozen.final_tree_oid}`, + ); + } + const canonicalDiff = gitOrThrow( + workdir, + env, + [...FROZEN_GIT_FLAGS, "diff", "--cached", "--binary", "--full-index", "--no-ext-diff", "--no-renames", "HEAD"], + ); + const workspaceStatus = gitOrThrow( + workdir, + env, + ["status", "--porcelain=v1", "-z", "--untracked-files=all"], + ); + return { + base_tree_oid, + canonical_diff_sha256: sha256Hex(canonicalDiff), + workspace_status_digest: sha256Hex(workspaceStatus), + }; +}; + /** Writes the §19.1 artifacts (`final-tree.tar.zst`) under a run directory. */ export const writeFrozenArtifacts = (runDir: string, frozen: FrozenFinalTree): string => { const archivePath = join(runDir, "final-tree.tar.zst"); diff --git a/bench/cdeb/orchestrator.ts b/bench/cdeb/orchestrator.ts new file mode 100644 index 00000000..2f8305ae --- /dev/null +++ b/bench/cdeb/orchestrator.ts @@ -0,0 +1,946 @@ +/** + * CDEB-07 run lifecycle coordinator (PRD §§10–11, §18.2/§18.4, §§19–20). + * + * The important policy is represented by control flow, not a comment: + * + * - an agent launch gets a durable checkpoint before the process can start; + * - only a typed pre-first-turn result reaches the agent retry loop; + * - a frozen tree routes directly to evaluation on resume; + * - evaluator calls receive only the persisted archive and its claimed OID; + * - progress values have no outcome-shaped field. + * + * CDEB-08 deliberately does not appear here. This module creates immutable + * rows; analysis receives the completed matrix later and is a separate ticket. + */ + +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + executeAgentRun, + type AgentRunOutcome, + type ContainerRuntimeCommands, + type RuntimePin, +} from "./runtime/agent-container.ts"; +import type { CapabilityGatePassed } from "./runtime/isolation.ts"; +import { readExposureEvents, exposureLogSha256 } from "./runtime/exposure.ts"; +import { readPersistedRawNdjson, readProviderLedger, type ProviderLedger } from "./runtime/provider-ledger.ts"; +import { assertCaptureSurfaceAbsent, writeCdebArmConfig } from "./runtime/arm-settings.ts"; +import { materializeBundle, type RepositoryBundleIdentity } from "./freeze/repository-bundle.ts"; +import { freezeFinalTree, frozenTreeProvenance } from "./evaluator/freeze-tree.ts"; +import { runEvaluatorOci } from "./evaluator/runner-oci.ts"; +import type { EvaluatorOutput } from "./evaluator/types.ts"; +import { + DurableStudyStorage, + type AgentLaunchCheckpoint, + type AgentStartedCheckpoint, + type FinalTreeArtifact, + type StoredAttempt, +} from "./storage.ts"; + +export type CdebCondition = "commitlore-on" | "commitlore-off"; +export type LifecycleState = + | "PLANNED" + | "PREFLIGHT" + | "AGENT_STARTING" + | "AGENT_STARTED" + | "FINAL_TREE_FROZEN" + | "EVALUATING" + | "MEASURED" + | "PRE_AGENT_INFRA_FAILURE" + | "MEASURED_AGENT_FAILURE" + | "EVALUATOR_INFRA_FAILURE" + | "MEASUREMENT_INTEGRITY_FAILURE"; + +export const MAX_PRE_AGENT_ATTEMPTS = 3; +export const MAX_EVALUATOR_ATTEMPTS = 3; + +const conditionSuffix = (condition: CdebCondition): "on" | "off" => + condition === "commitlore-on" ? "on" : "off"; + +const sha256 = (input: string): string => createHash("sha256").update(input, "utf8").digest("hex"); + +/** One sealed task/repeat pair. `sealed_key` never enters public JSON. */ +export interface SealedPairBlock { + readonly sealed_key: string; + readonly value: T; +} + +export interface OpaqueRandomizationBlock { + readonly block_index: string; + readonly conditions: readonly [CdebCondition, CdebCondition]; +} + +/** Safe for the public freeze: opaque indices and arm order only. */ +export interface OpaqueRandomizationManifest { + readonly schema_version: 1; + readonly algorithm: "sha256-key-sort-v1"; + readonly block_count: number; + readonly blocks: readonly OpaqueRandomizationBlock[]; +} + +export interface ScheduledSealedBlock { + readonly block_index: string; + readonly value: T; + readonly conditions: readonly [CdebCondition, CdebCondition]; +} + +export interface BlockedRandomization { + readonly public_manifest: OpaqueRandomizationManifest; + /** Kept in the sealed plan, never serialized into public-freeze artifacts. */ + readonly sealed_schedule: readonly ScheduledSealedBlock[]; +} + +/** + * Deterministic blocked randomization. Sort-by-hash avoids a stateful PRNG + * implementation and makes the chosen order reproducible from the frozen + * seed, while the public side contains no task/repository identifier. + */ +export const blockedRandomization = ( + pairs: readonly SealedPairBlock[], + freezeSeed: string, +): BlockedRandomization => { + if (freezeSeed === "") throw new Error("CDEB randomization seed must not be empty"); + const keys = new Set(); + for (const pair of pairs) { + if (pair.sealed_key === "") throw new Error("CDEB sealed block key must not be empty"); + if (keys.has(pair.sealed_key)) throw new Error(`CDEB sealed block key is duplicated: ${pair.sealed_key}`); + keys.add(pair.sealed_key); + } + const sorted = [...pairs].sort((left, right) => { + const leftHash = sha256(`${freezeSeed}\u0000block\u0000${left.sealed_key}`); + const rightHash = sha256(`${freezeSeed}\u0000block\u0000${right.sealed_key}`); + return leftHash === rightHash ? (left.sealed_key < right.sealed_key ? -1 : 1) : (leftHash < rightHash ? -1 : 1); + }); + const sealed_schedule = sorted.map((pair, index): ScheduledSealedBlock => { + const onFirst = sha256(`${freezeSeed}\u0000arm\u0000${pair.sealed_key}`) < "8".repeat(64); + const conditions: [CdebCondition, CdebCondition] = onFirst + ? ["commitlore-on", "commitlore-off"] + : ["commitlore-off", "commitlore-on"]; + return { + block_index: `block-${String(index).padStart(3, "0")}`, + value: pair.value, + conditions, + }; + }); + return { + public_manifest: { + schema_version: 1, + algorithm: "sha256-key-sort-v1", + block_count: sealed_schedule.length, + blocks: sealed_schedule.map(({ block_index, conditions }) => ({ block_index, conditions })), + }, + sealed_schedule, + }; +}; + +/** + * The only values a progress consumer can receive. There is deliberately no + * result, evaluator, usage, condition aggregate, or outcome field in this + * type. `Readonly` plus construction inside `emitProgress` keep an adapter + * from receiving the mutable row object by accident. + */ +export interface OutcomeFreeProgress { + readonly logical_run_id: string; + readonly state: LifecycleState; + readonly attempt_count: number; + readonly completed: number; + readonly remaining: number; +} + +export type ProgressReporter = (progress: Readonly) => void; + +export const formatOutcomeFreeProgress = (progress: OutcomeFreeProgress): string => + `cdeb: ${progress.logical_run_id} ${progress.state} attempt ${String(progress.attempt_count)} ${String(progress.completed)} completed ${String(progress.remaining)} remaining`; + +const emitProgress = ( + report: ProgressReporter | undefined, + logicalRunId: string, + state: LifecycleState, + attemptCount: number, + completed: number, + total: number, +): void => { + if (report === undefined) return; + report(Object.freeze({ + logical_run_id: logicalRunId, + state, + attempt_count: attemptCount, + completed, + remaining: total - completed, + })); +}; + +export interface ExposureSummary { + readonly instrumentation_complete: true; + readonly hook_opportunities: number; + readonly proxy_executions: number; + readonly expected_record_delivered: boolean; + readonly delivered_before_first_mutation: boolean; + readonly delivered_record_ids: readonly string[]; + readonly payload_sha256s: readonly string[]; + readonly product_failures: number; + readonly exposure_log_sha256: string; +} + +export interface PreparedWorkspace { + readonly workdir: string; + /** An empty file must exist for OFF too: zero is observed, never inferred. */ + readonly exposure_path: string; + /** Directory that contains the isolated settings/MCP files for CDEB-03. */ + readonly config_dir: string; + readonly cleanup: () => void; +} + +export interface AgentTerminalObservation { + readonly kind: "after-first-model-turn"; + readonly started_at: string; + readonly finished_at: string; + readonly stop_reason: "completed" | "timeout" | "agent_error" | "provider_error_after_start"; + readonly provider_ledger: ProviderLedger; + /** Exact uncompressed CDEB-05 source bytes. */ + readonly raw_provider_ndjson: Buffer; +} + +export interface AgentPreTurnFailure { + readonly kind: "before-first-model-turn"; + readonly failure_detail: string; +} + +/** A post-turn parse/identity failure is evidence of an incomplete study, not a retry. */ +export interface AgentMeasurementIntegrityFailure { + readonly kind: "measurement-integrity-failure"; + readonly failure_detail: string; +} + +export type AgentExecution = + | AgentTerminalObservation + | AgentPreTurnFailure + | AgentMeasurementIntegrityFailure; + +export interface AgentRunnerInput { + readonly plan: LogicalRunPlan; + readonly workspace: PreparedWorkspace; + /** Called at the byte-level first-turn boundary; must be invoked exactly once. */ + readonly on_first_model_turn: () => void; +} + +export interface AgentRunner { + readonly run: (input: AgentRunnerInput) => Promise; +} + +export interface FrozenTreeObservation { + readonly archive: Buffer; + readonly metadata: FinalTreeArtifact; +} + +export interface FinalTreeFreezer { + readonly freeze: (workspace: PreparedWorkspace) => FrozenTreeObservation; +} + +export interface EvaluatorInput { + readonly plan: LogicalRunPlan; + readonly archive_path: string; + readonly final_tree: FinalTreeArtifact; +} + +export type EvaluatorExecution = + | { readonly kind: "verdict"; readonly verdict: EvaluatorOutput } + | { readonly kind: "infrastructure-failure"; readonly failure_detail: string }; + +export interface EvaluatorRunner { + readonly evaluate: (input: EvaluatorInput) => Promise; +} + +export interface LogicalRunPlan { + readonly logical_run_id: string; + readonly repository_id: string; + readonly task_id: string; + readonly category: string; + readonly condition: CdebCondition; + readonly repeat: 1 | 2 | 3; + readonly order: number; + /** Required to re-parse retained CDEB-05 bytes during evaluator-only resume. */ + readonly requested_model: string; + readonly prompt: string; + readonly expected_record_ids: readonly string[]; + /** Builds the closed §19.2 row from immutable observations only. */ + readonly make_row: (input: { + readonly agent: AgentTerminalObservation; + readonly exposure: ExposureSummary; + readonly final_tree: FinalTreeArtifact; + readonly evaluator: EvaluatorOutput; + readonly evaluator_attempts: number; + }) => Record; +} + +export interface CdebStudyPlan { + /** Exact public commitment that must not change on resume. */ + readonly public_freeze: unknown; + readonly randomization: OpaqueRandomizationManifest; + /** Sealed mapping from opaque blocks to the actual task/repeat cells. */ + readonly logical_runs: readonly LogicalRunPlan[]; +} + +export interface OrchestratorDependencies { + readonly prepare_workspace: (plan: LogicalRunPlan) => Promise; + readonly agent: AgentRunner; + readonly freeze_tree: FinalTreeFreezer; + readonly collect_exposure: (workspace: PreparedWorkspace, plan: LogicalRunPlan) => ExposureSummary; + readonly evaluator: EvaluatorRunner; +} + +/** Frozen bundle source for one repository named by the sealed run plan. */ +export interface MaterializedRepositorySource { + readonly bundle_path: string; + readonly identity: RepositoryBundleIdentity; +} + +export interface MaterializedWorkspacePreparerOptions { + readonly repositories: Readonly>; + /** Defaults to the OS scratch directory; never used as authoritative storage. */ + readonly scratch_parent?: string; +} + +/** + * Production workspace preparation, composing CDEB-02 materialization with + * CDEB-04's frozen delivery-only arm config. The output belongs only to one + * agent attempt; all authoritative evidence is copied into DurableStudyStorage. + */ +export const materializedWorkspacePreparer = ( + options: MaterializedWorkspacePreparerOptions, +): OrchestratorDependencies["prepare_workspace"] => async (plan): Promise => { + const source = options.repositories[plan.repository_id]; + if (source === undefined) throw new Error(`CDEB run ${plan.logical_run_id} has no frozen bundle for ${plan.repository_id}`); + const root = mkdtempSync(join(options.scratch_parent ?? tmpdir(), "cdeb-workspace-")); + const workdir = join(root, "repository"); + const configDir = join(root, "config"); + try { + materializeBundle(source.identity, source.bundle_path, workdir); + const arm = plan.condition === "commitlore-on" ? "on" : "off"; + const config = writeCdebArmConfig(workdir, configDir, arm); + // `writeCdebArmConfig` already checks this; make it explicit at the + // orchestration boundary so a later config construction cannot bypass it. + assertCaptureSurfaceAbsent(workdir, config); + return { + workdir, + exposure_path: config.exposurePath, + config_dir: config.configDir, + cleanup: () => rmSync(root, { recursive: true, force: true }), + }; + } catch (error) { + rmSync(root, { recursive: true, force: true }); + throw error; + } +}; + +export interface RunStudyOptions { + readonly storage: DurableStudyStorage; + readonly progress?: ProgressReporter; +} + +export interface StudyRunResult { + readonly completed_logical_run_ids: readonly string[]; + readonly missing_logical_run_ids: readonly string[]; +} + +export class InterruptedAgentAttemptError extends Error { + public constructor(logicalRunId: string, attempts: readonly string[]) { + super( + `CDEB ${logicalRunId}: agent launch ${attempts.join(", ")} has no terminal pre-turn record or frozen tree; refusing an agent rerun`, + ); + this.name = "InterruptedAgentAttemptError"; + } +} + +export class MeasurementIntegrityError extends Error { + public constructor(logicalRunId: string, detail: string) { + super(`CDEB ${logicalRunId}: measurement integrity failure: ${detail}`); + this.name = "MeasurementIntegrityError"; + } +} + +export class RetryExhaustedError extends Error { + public constructor(logicalRunId: string, stage: "agent" | "evaluator", count: number) { + super(`CDEB ${logicalRunId}: ${stage} retry limit reached after ${String(count)} attempt(s)`); + this.name = "RetryExhaustedError"; + } +} + +const attemptId = (plan: LogicalRunPlan, ordinal: number): string => `${plan.logical_run_id}__a${String(ordinal)}`; + +const requiredRunId = (plan: LogicalRunPlan): void => { + const expected = `${plan.repository_id}__${plan.task_id}__${conditionSuffix(plan.condition)}__r${String(plan.repeat)}`; + if (plan.logical_run_id !== expected) { + throw new Error(`logical run id ${plan.logical_run_id} does not name its repository/task/condition/repeat cell`); + } +}; + +/** Binds the sealed task mapping to the committed opaque block order. */ +const assertBlockedSchedule = (plan: CdebStudyPlan): void => { + if (plan.randomization.algorithm !== "sha256-key-sort-v1" || plan.randomization.schema_version !== 1) { + throw new Error("CDEB randomization manifest is not the frozen blocked-randomization format"); + } + if (plan.randomization.blocks.length !== plan.randomization.block_count) { + throw new Error("CDEB randomization block list length differs from its committed count"); + } + for (const [blockNumber, block] of plan.randomization.blocks.entries()) { + const expectedIndex = `block-${String(blockNumber).padStart(3, "0")}`; + if (block.block_index !== expectedIndex) { + throw new Error(`CDEB randomization block ${String(blockNumber)} is not the expected opaque index ${expectedIndex}`); + } + const first = plan.logical_runs[blockNumber * 2]; + const second = plan.logical_runs[blockNumber * 2 + 1]; + if (first === undefined || second === undefined) throw new Error(`CDEB block ${block.block_index} has no sealed pair`); + if (first.order !== blockNumber * 2 + 1 || second.order !== blockNumber * 2 + 2) { + throw new Error(`CDEB block ${block.block_index} does not occupy its committed consecutive order slots`); + } + if (first.condition !== block.conditions[0] || second.condition !== block.conditions[1]) { + throw new Error(`CDEB block ${block.block_index} condition order differs from randomization.json`); + } + if (first.repository_id !== second.repository_id || first.task_id !== second.task_id || first.repeat !== second.repeat) { + throw new Error(`CDEB block ${block.block_index} does not contain one task/repeat ON/OFF pair`); + } + } +}; + +const agentAttemptRecord = ( + plan: LogicalRunPlan, + id: string, + observation: AgentTerminalObservation | AgentMeasurementIntegrityFailure, + startedAt: string, +): StoredAttempt => { + if (observation.kind === "measurement-integrity-failure") { + return { + schema_version: 1, + benchmark: "cdeb-v1", + attempt_id: id, + logical_run_id: plan.logical_run_id, + terminal_state: "MEASUREMENT_INTEGRITY_FAILURE", + started_at: startedAt, + finished_at: new Date().toISOString(), + first_model_turn_observed: true, + failure_detail: observation.failure_detail, + }; + } + return { + schema_version: 1, + benchmark: "cdeb-v1", + attempt_id: id, + logical_run_id: plan.logical_run_id, + terminal_state: observation.stop_reason === "completed" ? "MEASURED" : "MEASURED_AGENT_FAILURE", + started_at: observation.started_at, + finished_at: observation.finished_at, + first_model_turn_observed: true, + ...(observation.stop_reason === "completed" ? {} : { failure_detail: observation.stop_reason }), + }; +}; + +const preAgentAttemptRecord = (plan: LogicalRunPlan, id: string, startedAt: string, detail: string): StoredAttempt => ({ + schema_version: 1, + benchmark: "cdeb-v1", + attempt_id: id, + logical_run_id: plan.logical_run_id, + terminal_state: "PRE_AGENT_INFRA_FAILURE", + started_at: startedAt, + finished_at: new Date().toISOString(), + first_model_turn_observed: false, + failure_detail: detail, +}); + +const evaluatorAttemptRecord = ( + finalTree: FinalTreeArtifact, + attempt: number, + result: EvaluatorExecution, +): Record => result.kind === "verdict" + ? { + schema_version: 1, + attempt: attempt, + terminal_state: "VERDICT", + candidate_tree_oid: finalTree.final_tree_oid, + evaluator_tree_oid: result.verdict.candidate_tree_oid, + } + : { + schema_version: 1, + attempt: attempt, + terminal_state: "EVALUATOR_INFRA_FAILURE", + candidate_tree_oid: finalTree.final_tree_oid, + failure_detail: result.failure_detail, + }; + +const assertEvaluatorBinding = ( + plan: LogicalRunPlan, + finalTree: FinalTreeArtifact, + verdict: EvaluatorOutput, +): void => { + if (verdict.candidate_tree_oid !== finalTree.final_tree_oid) { + throw new MeasurementIntegrityError( + plan.logical_run_id, + `evaluator tree ${verdict.candidate_tree_oid} differs from frozen tree ${finalTree.final_tree_oid}`, + ); + } + if (verdict.task_id !== plan.task_id) { + throw new MeasurementIntegrityError( + plan.logical_run_id, + `evaluator task ${verdict.task_id} differs from frozen task ${plan.task_id}`, + ); + } +}; + +const finalizeMeasuredRow = ( + plan: LogicalRunPlan, + storage: DurableStudyStorage, + agent: AgentTerminalObservation, + exposure: ExposureSummary, + finalTree: FinalTreeArtifact, + evaluator: EvaluatorOutput, + evaluatorAttempts: number, +): void => { + const row = plan.make_row({ + agent, + exposure, + final_tree: finalTree, + evaluator, + evaluator_attempts: evaluatorAttempts, + }); + storage.writeRow(plan.logical_run_id, row); +}; + +const evaluateFrozenTree = async ( + plan: LogicalRunPlan, + storage: DurableStudyStorage, + dependencies: OrchestratorDependencies, + finalTree: FinalTreeArtifact, + agent: AgentTerminalObservation, + exposure: ExposureSummary, + initialEvaluatorAttemptCount: number, + maxEvaluatorAttempts: number, + report: ProgressReporter | undefined, + completed: number, + total: number, +): Promise => { + let evaluatorAttempts = initialEvaluatorAttemptCount; + while (evaluatorAttempts < maxEvaluatorAttempts) { + evaluatorAttempts += 1; + emitProgress(report, plan.logical_run_id, "EVALUATING", evaluatorAttempts, completed, total); + let evaluation: EvaluatorExecution; + try { + evaluation = await dependencies.evaluator.evaluate({ + plan, + archive_path: storage.runArtifactPath(plan.logical_run_id, "final-tree.tar.zst"), + final_tree: finalTree, + }); + } catch (error) { + evaluation = { + kind: "infrastructure-failure", + failure_detail: error instanceof Error ? error.message : String(error), + }; + } + storage.writeEvaluatorAttempt(plan.logical_run_id, `e${String(evaluatorAttempts)}`, evaluatorAttemptRecord(finalTree, evaluatorAttempts, evaluation)); + if (evaluation.kind === "infrastructure-failure") { + emitProgress(report, plan.logical_run_id, "EVALUATOR_INFRA_FAILURE", evaluatorAttempts, completed, total); + continue; + } + assertEvaluatorBinding(plan, finalTree, evaluation.verdict); + storage.writeEvaluatorResult(plan.logical_run_id, evaluation.verdict); + finalizeMeasuredRow(plan, storage, agent, exposure, finalTree, evaluation.verdict, evaluatorAttempts); + return; + } + throw new RetryExhaustedError(plan.logical_run_id, "evaluator", evaluatorAttempts); +}; + +const runLogical = async ( + plan: LogicalRunPlan, + storage: DurableStudyStorage, + dependencies: OrchestratorDependencies, + maxPreAgentAttempts: number, + maxEvaluatorAttempts: number, + report: ProgressReporter | undefined, + completed: number, + total: number, +): Promise => { + const initial = storage.readRunState(plan.logical_run_id); + if (initial.row !== null) return; + if (initial.final_tree !== null) { + // The agent is structurally unreachable on this branch. The durable + // observation sidecar is written before final-tree.json below. + const terminalAttempt = initial.agent_attempts.find((attempt) => attempt.first_model_turn_observed); + if (terminalAttempt === undefined) { + throw new MeasurementIntegrityError(plan.logical_run_id, "frozen tree has no terminal first-turn agent attempt"); + } + const observation = storage.readAgentObservation(plan.logical_run_id, terminalAttempt.attempt_id); + if (observation === null) { + throw new MeasurementIntegrityError(plan.logical_run_id, "frozen tree has no durable agent observation sidecar"); + } + const restored = restoreAgentObservation(plan, storage, observation); + const existingVerdict = storage.readJson(join("runs", plan.logical_run_id, "evaluator.json")); + if (existingVerdict !== null) { + // A crash after evaluator.json but before row.json must not turn into a + // new evaluation, much less a new agent. The persisted verdict is the + // evaluation attempt; finish only its missing row commit record. + assertEvaluatorBinding(plan, initial.final_tree, existingVerdict); + finalizeMeasuredRow( + plan, + storage, + restored, + observation.exposure, + initial.final_tree, + existingVerdict, + initial.evaluator_attempt_count, + ); + return; + } + await evaluateFrozenTree( + plan, + storage, + dependencies, + initial.final_tree, + restored, + observation.exposure, + initial.evaluator_attempt_count, + maxEvaluatorAttempts, + report, + completed, + total, + ); + return; + } + + const preAttempts = storage.preAgentAttempts(plan.logical_run_id); + const terminalPreAttemptIds = new Set(preAttempts.map((attempt) => attempt.attempt_id)); + const uncertainLaunches = initial.launched_attempt_ids.filter((id) => !terminalPreAttemptIds.has(id)); + if (uncertainLaunches.length > 0) { + // Between launch and a terminal pre-turn record the durable state cannot + // distinguish a killed process before its first turn from one after it. + // The safe answer is incomplete, never an agent rerun. + throw new InterruptedAgentAttemptError(plan.logical_run_id, uncertainLaunches); + } + if (preAttempts.length >= maxPreAgentAttempts) { + throw new RetryExhaustedError(plan.logical_run_id, "agent", preAttempts.length); + } + + for (let ordinal = preAttempts.length + 1; ordinal <= maxPreAgentAttempts; ordinal += 1) { + const id = attemptId(plan, ordinal); + emitProgress(report, plan.logical_run_id, "PREFLIGHT", ordinal, completed, total); + const workspace = await dependencies.prepare_workspace(plan); + const launchedAt = new Date().toISOString(); + const launch: AgentLaunchCheckpoint = { + schema_version: 1, + logical_run_id: plan.logical_run_id, + attempt_id: id, + launched_at: launchedAt, + }; + storage.beginAgentAttempt(launch); + let firstTurnMarked = false; + const markFirstTurn = (): void => { + if (firstTurnMarked) return; + const marker: AgentStartedCheckpoint = { + ...launch, + first_model_turn_observed: true, + }; + storage.markFirstModelTurn(marker); + firstTurnMarked = true; + }; + + emitProgress(report, plan.logical_run_id, "AGENT_STARTING", ordinal, completed, total); + let execution: AgentExecution; + try { + execution = await dependencies.agent.run({ plan, workspace, on_first_model_turn: markFirstTurn }); + } catch (error) { + // A thrown adapter error is deliberately not reclassified as retryable: + // a process may have produced a model turn before its host reported it. + workspace.cleanup(); + throw new InterruptedAgentAttemptError(plan.logical_run_id, [id]); + } + + if (execution.kind === "before-first-model-turn") { + if (firstTurnMarked) { + workspace.cleanup(); + throw new MeasurementIntegrityError( + plan.logical_run_id, + "agent adapter reported a pre-turn failure after the durable first-turn marker", + ); + } + storage.writePreAgentAttempt(preAgentAttemptRecord(plan, id, launchedAt, execution.failure_detail)); + workspace.cleanup(); + emitProgress(report, plan.logical_run_id, "PRE_AGENT_INFRA_FAILURE", ordinal, completed, total); + continue; + } + if (!firstTurnMarked) { + workspace.cleanup(); + throw new MeasurementIntegrityError(plan.logical_run_id, "post-turn agent result arrived without the durable first-turn marker"); + } + if (execution.kind === "measurement-integrity-failure") { + storage.writeAgentAttempt(agentAttemptRecord(plan, id, execution, launchedAt)); + workspace.cleanup(); + emitProgress(report, plan.logical_run_id, "MEASUREMENT_INTEGRITY_FAILURE", ordinal, completed, total); + throw new MeasurementIntegrityError(plan.logical_run_id, execution.failure_detail); + } + + storage.writeAgentAttempt(agentAttemptRecord(plan, id, execution, launchedAt)); + emitProgress(report, plan.logical_run_id, "AGENT_STARTED", ordinal, completed, total); + storage.writeProviderNdjson(plan.logical_run_id, execution.raw_provider_ndjson); + const exposure = dependencies.collect_exposure(workspace, plan); + const exposureBytes = readFileSync(workspace.exposure_path); + storage.writeExposure(plan.logical_run_id, exposureBytes); + const frozen = dependencies.freeze_tree.freeze(workspace); + const observation: DurableAgentObservation = { + schema_version: 1, + agent: serializableAgentObservation(execution), + exposure, + }; + // This sidecar precedes final-tree.json. A resume can therefore evaluate + // the frozen observation without ever revisiting the agent workspace. + storage.writeAgentObservation(plan.logical_run_id, id, observation); + storage.writeFinalTree(plan.logical_run_id, frozen.archive, frozen.metadata); + workspace.cleanup(); + emitProgress(report, plan.logical_run_id, "FINAL_TREE_FROZEN", ordinal, completed, total); + await evaluateFrozenTree( + plan, + storage, + dependencies, + frozen.metadata, + execution, + exposure, + 0, + maxEvaluatorAttempts, + report, + completed, + total, + ); + return; + } + throw new RetryExhaustedError(plan.logical_run_id, "agent", maxPreAgentAttempts); +}; + +interface SerializedAgentObservation { + readonly started_at: string; + readonly finished_at: string; + readonly stop_reason: AgentTerminalObservation["stop_reason"]; + readonly raw_provider_ndjson_sha256: string; + readonly provider_ledger: ProviderLedger; +} + +interface DurableAgentObservation { + readonly schema_version: 1; + readonly agent: SerializedAgentObservation; + readonly exposure: ExposureSummary; +} + +const serializableAgentObservation = (agent: AgentTerminalObservation): SerializedAgentObservation => ({ + started_at: agent.started_at, + finished_at: agent.finished_at, + stop_reason: agent.stop_reason, + raw_provider_ndjson_sha256: agent.provider_ledger.usage.raw_stream_sha256, + provider_ledger: agent.provider_ledger, +}); + +const restoreAgentObservation = ( + plan: LogicalRunPlan, + storage: DurableStudyStorage, + observation: DurableAgentObservation, +): AgentTerminalObservation => { + const raw = readPersistedRawNdjson(storage.runDirectoryPath(plan.logical_run_id)); + const provider_ledger = readProviderLedger({ requested_model: plan.requested_model, raw_ndjson: raw }); + if (provider_ledger.usage.raw_stream_sha256 !== observation.agent.raw_provider_ndjson_sha256) { + throw new MeasurementIntegrityError(plan.logical_run_id, "retained provider stream differs from agent observation sidecar"); + } + if (JSON.stringify(provider_ledger) !== JSON.stringify(observation.agent.provider_ledger)) { + throw new MeasurementIntegrityError(plan.logical_run_id, "retained provider ledger differs from agent observation sidecar"); + } + return { + kind: "after-first-model-turn", + started_at: observation.agent.started_at, + finished_at: observation.agent.finished_at, + stop_reason: observation.agent.stop_reason, + provider_ledger, + raw_provider_ndjson: raw, + }; +}; + +/** + * Public entrypoint for new runs and resumes. Missing-set computation uses + * the sealed schedule and immutable per-run `row.json` records — never a glob + * of prior outputs and never a row overwrite. + */ +export const runStudy = async ( + plan: CdebStudyPlan, + dependencies: OrchestratorDependencies, + options: RunStudyOptions, +): Promise => { + if (plan.logical_runs.length === 0) throw new Error("CDEB study has no logical runs"); + if (plan.randomization.block_count * 2 !== plan.logical_runs.length) { + throw new Error("CDEB randomization block count does not match its logical run schedule"); + } + assertBlockedSchedule(plan); + const maxPreAgentAttempts = MAX_PRE_AGENT_ATTEMPTS; + const maxEvaluatorAttempts = MAX_EVALUATOR_ATTEMPTS; + for (const logicalRun of plan.logical_runs) requiredRunId(logicalRun); + const expectedIds = plan.logical_runs.map((run) => run.logical_run_id); + if (new Set(expectedIds).size !== expectedIds.length) throw new Error("CDEB logical run schedule has duplicate ids"); + + options.storage.recoverUnpublishedPartials(); + options.storage.repairBackupMirrors(); + options.storage.ensureCommittedJson("public-freeze.json", plan.public_freeze); + options.storage.ensureCommittedJson("randomization.json", plan.randomization); + const completedBefore = options.storage.completedRows(expectedIds); + let completed = completedBefore.size; + for (const logicalRun of plan.logical_runs) { + if (completedBefore.has(logicalRun.logical_run_id)) continue; + emitProgress(options.progress, logicalRun.logical_run_id, "PLANNED", 0, completed, expectedIds.length); + await runLogical( + logicalRun, + options.storage, + dependencies, + maxPreAgentAttempts, + maxEvaluatorAttempts, + options.progress, + completed, + expectedIds.length, + ); + completed += 1; + emitProgress(options.progress, logicalRun.logical_run_id, "MEASURED", 0, completed, expectedIds.length); + } + const missing = options.storage.missingLogicalIds(expectedIds); + return { + completed_logical_run_ids: expectedIds.filter((id) => !missing.includes(id)), + missing_logical_run_ids: missing, + }; +}; + +/** The production bridge from the CDEB-03 runtime to this state machine. */ +export interface RuntimeAgentRunnerOptions { + readonly docker: ContainerRuntimeCommands; + readonly pin: RuntimePin; + readonly gate: CapabilityGatePassed; + readonly provider_env: Readonly>; +} + +const stopReasonOf = (outcome: AgentRunOutcome): AgentTerminalObservation["stop_reason"] => + outcome.timed_out ? "timeout" : outcome.exit_code === 0 ? "completed" : "agent_error"; + +export const runtimeAgentRunner = (options: RuntimeAgentRunnerOptions): AgentRunner => ({ + run: async ({ plan, workspace, on_first_model_turn }): Promise => { + const startedAt = new Date().toISOString(); + const outDir = mkdtempSync(join(tmpdir(), "cdeb-agent-stream-")); + let firstTurnObserved = false; + try { + const outcome = await executeAgentRun(options.docker, options.pin, options.gate, { + repositoryPath: workspace.workdir, + configDir: workspace.config_dir, + prompt: plan.prompt, + outDir, + providerEnv: options.provider_env, + onFirstModelTurn: () => { + firstTurnObserved = true; + on_first_model_turn(); + }, + }); + if (!firstTurnObserved) { + return { kind: "before-first-model-turn", failure_detail: "agent process ended without a provider model turn" }; + } + return { + kind: "after-first-model-turn", + started_at: startedAt, + finished_at: new Date().toISOString(), + stop_reason: stopReasonOf(outcome), + provider_ledger: outcome.ledger, + raw_provider_ndjson: readPersistedRawNdjson(outDir), + }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return firstTurnObserved + ? { kind: "measurement-integrity-failure", failure_detail: detail } + : { kind: "before-first-model-turn", failure_detail: detail }; + } finally { + rmSync(outDir, { recursive: true, force: true }); + } + }, +}); + +/** CDEB-06 freezer plus the PRD §11.1 provenance that CDEB-07 persists. */ +export const canonicalFinalTreeFreezer: FinalTreeFreezer = { + freeze: (workspace): FrozenTreeObservation => { + const scratch = mkdtempSync(join(tmpdir(), "cdeb-final-tree-")); + try { + const frozen = freezeFinalTree(workspace.workdir, scratch); + const provenance = frozenTreeProvenance(workspace.workdir, scratch, frozen); + return { + archive: frozen.archive_zst, + metadata: { + schema_version: 1, + ...provenance, + final_tree_oid: frozen.final_tree_oid, + archive_sha256: frozen.archive_zst_sha256, + }, + }; + } finally { + rmSync(scratch, { recursive: true, force: true }); + } + }, +}; + +/** + * Derives the non-outcome exposure facts from the CDEB-04 append-only log. + * Callers supply the first-mutation fact because it belongs to the frozen + * runtime observation, not to the shipping-output parser. + */ +export const summarizeExposure = ( + exposurePath: string, + expectedRecordIds: readonly string[], + deliveredBeforeFirstMutation: boolean, +): ExposureSummary => { + const events = readExposureEvents(exposurePath); + const delivered = [...new Set(events.flatMap((event) => event.parsed_record_ids ?? []))].sort(); + const payloads = [...new Set(events.flatMap((event) => event.payload_sha256 === null ? [] : [event.payload_sha256]))].sort(); + return { + instrumentation_complete: true, + hook_opportunities: events.length, + proxy_executions: events.length, + expected_record_delivered: expectedRecordIds.every((id) => delivered.includes(id)), + delivered_before_first_mutation: deliveredBeforeFirstMutation, + delivered_record_ids: delivered, + payload_sha256s: payloads, + product_failures: events.filter((event) => event.product_error !== null || event.child_exit_code !== 0).length, + exposure_log_sha256: exposureLogSha256(exposurePath), + }; +}; + +export interface OciEvaluatorRunnerOptions { + readonly image_ref: string; + readonly sealed_tasks_dir: string; + readonly image_digest: string; +} + +export const ociEvaluatorRunner = (options: OciEvaluatorRunnerOptions): EvaluatorRunner => ({ + evaluate: async ({ plan, archive_path, final_tree }): Promise => { + try { + const result = runEvaluatorOci({ + imageRef: options.image_ref, + archivePath: archive_path, + tasksDir: options.sealed_tasks_dir, + taskId: plan.task_id, + claimedOid: final_tree.final_tree_oid, + imageDigest: options.image_digest, + }); + if (result.exitCode !== 0) { + return { kind: "infrastructure-failure", failure_detail: result.stderr || `evaluator exited ${String(result.exitCode)}` }; + } + const text = result.stdout.toString("utf8"); + let verdict: unknown; + try { + verdict = JSON.parse(text); + } catch { + return { kind: "infrastructure-failure", failure_detail: "evaluator stdout was not JSON" }; + } + if (typeof verdict !== "object" || verdict === null || Array.isArray(verdict)) { + return { kind: "infrastructure-failure", failure_detail: "evaluator stdout was not an object" }; + } + return { kind: "verdict", verdict: verdict as EvaluatorOutput }; + } catch (error) { + return { kind: "infrastructure-failure", failure_detail: error instanceof Error ? error.message : String(error) }; + } + }, +}); diff --git a/bench/cdeb/run.ts b/bench/cdeb/run.ts new file mode 100644 index 00000000..ffe7d7a8 --- /dev/null +++ b/bench/cdeb/run.ts @@ -0,0 +1,64 @@ +/** + * Public CDEB run-lifecycle surface. + * + * Freeze/sealed-bundle loading belongs to the freeze tooling; this module + * intentionally exports only the immutable execution API so a caller cannot + * slip scientific CLI overrides between a frozen manifest and `runStudy`. + */ + +export { + MAX_EVALUATOR_ATTEMPTS, + MAX_PRE_AGENT_ATTEMPTS, + MeasurementIntegrityError, + InterruptedAgentAttemptError, + RetryExhaustedError, + blockedRandomization, + canonicalFinalTreeFreezer, + formatOutcomeFreeProgress, + materializedWorkspacePreparer, + ociEvaluatorRunner, + runStudy, + runtimeAgentRunner, + summarizeExposure, +} from "./orchestrator.ts"; + +export type { + AgentExecution, + AgentRunner, + AgentRunnerInput, + AgentTerminalObservation, + BlockedRandomization, + CdebCondition, + CdebStudyPlan, + EvaluatorExecution, + EvaluatorRunner, + ExposureSummary, + FinalTreeFreezer, + FrozenTreeObservation, + LifecycleState, + LogicalRunPlan, + MaterializedRepositorySource, + MaterializedWorkspacePreparerOptions, + OciEvaluatorRunnerOptions, + OpaqueRandomizationManifest, + OrchestratorDependencies, + OutcomeFreeProgress, + PreparedWorkspace, + ProgressReporter, + RunStudyOptions, + RuntimeAgentRunnerOptions, + ScheduledSealedBlock, + SealedPairBlock, + StudyRunResult, +} from "./orchestrator.ts"; + +export { DurableStudyStorage, ImmutableArtifactError, SimulatedProcessKill } from "./storage.ts"; +export type { + AgentLaunchCheckpoint, + AgentStartedCheckpoint, + DurableStudyStorageOptions, + FinalTreeArtifact, + StorageFaults, + StoredAttempt, + StoredRunState, +} from "./storage.ts"; diff --git a/bench/cdeb/runtime/agent-container.ts b/bench/cdeb/runtime/agent-container.ts index 6f8979d3..bae7a8c6 100644 --- a/bench/cdeb/runtime/agent-container.ts +++ b/bench/cdeb/runtime/agent-container.ts @@ -23,6 +23,7 @@ import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { createWriteStream, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { Writable } from "node:stream"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; @@ -1032,6 +1033,16 @@ export interface AgentRunParams { readonly outDir: string; readonly providerEnv: Readonly>; readonly timeoutMs?: number; + /** + * Called once, synchronously, as soon as the main agent's first provider + * `message_start` reaches the byte sink. CDEB-07 uses this to durably mark + * the logical cell non-rerunnable before the rest of the stream arrives. + * + * This is deliberately an observation callback rather than a lifecycle + * policy: the runtime preserves the bytes exactly as before, while the + * orchestrator owns the retry state machine that consumes the observation. + */ + readonly onFirstModelTurn?: () => void; } export interface AgentRunOutcome { @@ -1047,6 +1058,66 @@ export interface AgentRunOutcome { readonly ledger: ProviderLedger; } +/** + * Watches complete NDJSON lines without changing the byte stream. A main + * agent turn is the provider's `stream_event/message_start` with no parent + * tool-use id; delegated turns are never allowed to unlock a retry boundary. + */ +const firstModelTurnObserver = (onFirstModelTurn: () => void): ((chunk: Buffer) => void) => { + let pending = Buffer.alloc(0); + let observed = false; + return (chunk: Buffer): void => { + if (observed) return; + pending = pending.length === 0 ? Buffer.from(chunk) : Buffer.concat([pending, chunk]); + while (!observed) { + const newline = pending.indexOf(0x0a); + if (newline === -1) return; + const line = pending.subarray(0, newline); + pending = pending.subarray(newline + 1); + let event: unknown; + try { + event = JSON.parse(line.toString("utf8")); + } catch { + continue; + } + if (typeof event !== "object" || event === null || Array.isArray(event)) continue; + const envelope = event as Record; + const nested = envelope["event"]; + if (typeof nested !== "object" || nested === null || Array.isArray(nested)) continue; + const nestedRecord = nested as Record; + if ( + envelope["type"] === "stream_event" && + nestedRecord["type"] === "message_start" && + (envelope["parent_tool_use_id"] === null || envelope["parent_tool_use_id"] === undefined) + ) { + // Persist the state checkpoint before forwarding this chunk into the + // raw sink. If the process is killed next, CDEB-07 fails closed rather + // than treating a possible model answer as a retryable non-start. + onFirstModelTurn(); + observed = true; + } + } + }; +}; + +/** Wraps a writable sink solely to observe bytes; it never serializes them. */ +const observingSink = (sink: NodeJS.WritableStream, observe: (chunk: Buffer) => void): Writable => + new Writable({ + write(chunk, encoding, callback): void { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding); + try { + observe(bytes); + sink.write(bytes); + callback(); + } catch (error) { + callback(error as Error); + } + }, + final(callback): void { + sink.end(callback); + }, + }); + /** * One measured run inside the pinned runtime, or a refusal. The gate token is * checked against this pin's digest — a preflight that passed for another pin @@ -1112,7 +1183,10 @@ export const executeAgentRun = async ( mkdirSync(params.outDir, { recursive: true }); const streamPath = join(params.outDir, "provider.ndjson"); const sink = createWriteStream(streamPath); - const result = await docker.runToSink(dockerRunArgs(spec), sink, { + const streamSink = params.onFirstModelTurn === undefined + ? sink + : observingSink(sink, firstModelTurnObserver(params.onFirstModelTurn)); + const result = await docker.runToSink(dockerRunArgs(spec), streamSink, { timeoutMs: params.timeoutMs ?? 15 * 60 * 1000, }); await new Promise((resolve) => { diff --git a/bench/cdeb/storage.ts b/bench/cdeb/storage.ts new file mode 100644 index 00000000..2a3bd3c2 --- /dev/null +++ b/bench/cdeb/storage.ts @@ -0,0 +1,621 @@ +/** + * CDEB-07 durable study storage (PRD §§19–20). + * + * This module deliberately makes every authoritative write immutable. A + * result may be mirrored, inspected and analysed later; it must never be + * "updated" after the fact. Interrupted `.partial` names are outside the + * contract and are removed before resume — no caller can mistake one for a + * durable observation. + */ + +import { createHash, randomBytes } from "node:crypto"; +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + statSync, + unlinkSync, + writeSync, +} from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { zstdCompressSync } from "node:zlib"; + +export class ImmutableArtifactError extends Error { + public constructor(message: string) { + super(`CDEB durable storage: ${message}`); + this.name = "ImmutableArtifactError"; + } +} + +/** Test-only fault used to model a process death after fsync and before rename. */ +export class SimulatedProcessKill extends Error { + public constructor(message: string) { + super(message); + this.name = "SimulatedProcessKill"; + } +} + +export interface StorageFaults { + /** Called after a temporary file is fsynced, immediately before its rename. */ + readonly after_file_fsync_before_rename?: (relativePath: string) => void; +} + +export interface DurableStudyStorageOptions { + /** `bench/results/cdeb/` — never a temporary directory. */ + readonly studyDir: string; + /** Required independent mirror for every completed artifact (§20.3). */ + readonly backupDir: string; + readonly faults?: StorageFaults; +} + +export interface FinalTreeArtifact { + readonly schema_version: 1; + readonly base_tree_oid: string; + readonly final_tree_oid: string; + readonly canonical_diff_sha256: string; + readonly archive_sha256: string; + readonly workspace_status_digest: string; +} + +export interface AgentLaunchCheckpoint { + readonly schema_version: 1; + readonly logical_run_id: string; + readonly attempt_id: string; + readonly launched_at: string; +} + +export interface AgentStartedCheckpoint extends AgentLaunchCheckpoint { + readonly first_model_turn_observed: true; +} + +export interface StoredAttempt { + readonly schema_version: 1; + readonly benchmark: "cdeb-v1"; + readonly attempt_id: string; + readonly logical_run_id: string; + readonly terminal_state: + | "MEASURED" + | "PRE_AGENT_INFRA_FAILURE" + | "MEASURED_AGENT_FAILURE" + | "EVALUATOR_INFRA_FAILURE" + | "MEASUREMENT_INTEGRITY_FAILURE"; + readonly started_at: string; + readonly finished_at: string; + readonly first_model_turn_observed: boolean; + readonly failure_detail?: string; +} + +export interface StoredRunState { + readonly logical_run_id: string; + readonly row: Record | null; + readonly final_tree: FinalTreeArtifact | null; + readonly launched_attempt_ids: readonly string[]; + readonly started_attempt_ids: readonly string[]; + readonly agent_attempts: readonly StoredAttempt[]; + readonly evaluator_attempt_count: number; +} + +const sha256 = (bytes: Uint8Array | string): string => createHash("sha256").update(bytes).digest("hex"); + +const fsyncDirectory = (directory: string): void => { + const descriptor = openSync(directory, "r"); + try { + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } +}; + +const ensureDirectory = (directory: string): void => { + mkdirSync(directory, { recursive: true }); +}; + +const jsonBytes = (value: unknown): Buffer => Buffer.from(`${JSON.stringify(value, null, 2)}\n`, "utf8"); + +const parseJsonFile = (path: string, label: string): T => { + try { + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch (error) { + throw new ImmutableArtifactError(`${label} is not valid JSON (${(error as Error).message})`); + } +}; + +const isInside = (candidate: string, parent: string): boolean => { + const rel = relative(parent, candidate); + return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".."); +}; + +const assertRelative = (path: string): void => { + if (path === "" || isAbsolute(path) || path.split(/[\\/]/u).some((part) => part === ".." || part === "")) { + throw new ImmutableArtifactError(`unsafe relative artifact path ${JSON.stringify(path)}`); + } +}; + +const logicalIdPathSafe = (logicalRunId: string): void => { + if (!/^[a-z0-9-]+__[a-z0-9-]+__(on|off)__r[1-3]$/u.test(logicalRunId)) { + throw new ImmutableArtifactError(`invalid logical run id ${JSON.stringify(logicalRunId)}`); + } +}; + +const attemptIdPathSafe = (attemptId: string): void => { + if (!/^[a-z0-9-]+__[a-z0-9-]+__(on|off)__r[1-3]__a[1-9][0-9]*$/u.test(attemptId)) { + throw new ImmutableArtifactError(`invalid attempt id ${JSON.stringify(attemptId)}`); + } +}; + +/** + * Owns the two authoritative roots. The backup root deliberately has the + * same layout as the primary, allowing a reviewer to compare relative paths + * without trusting a database or a copy manifest. + */ +export class DurableStudyStorage { + public readonly studyDir: string; + public readonly backupDir: string; + private readonly faults: StorageFaults | undefined; + private temporarySequence = 0; + + public constructor(options: DurableStudyStorageOptions) { + if (options.studyDir === "") throw new ImmutableArtifactError("studyDir must not be empty"); + if (options.backupDir === "") throw new ImmutableArtifactError("CDEB_BACKUP_DIR is required"); + this.studyDir = resolve(options.studyDir); + this.backupDir = resolve(options.backupDir); + if (this.studyDir === this.backupDir) { + throw new ImmutableArtifactError("backup directory must differ from the authoritative study directory"); + } + this.faults = options.faults; + ensureDirectory(this.studyDir); + ensureDirectory(this.backupDir); + } + + private absolute(relativePath: string, root: string = this.studyDir): string { + assertRelative(relativePath); + const path = resolve(root, relativePath); + if (!isInside(path, root)) throw new ImmutableArtifactError(`artifact path escapes root: ${relativePath}`); + return path; + } + + private temporaryFor(destination: string): string { + this.temporarySequence += 1; + return `${destination}.${String(process.pid)}.${String(this.temporarySequence)}.${randomBytes(6).toString("hex")}.partial`; + } + + private writeNewAt(root: string, relativePath: string, bytes: Uint8Array): void { + const destination = this.absolute(relativePath, root); + ensureDirectory(dirname(destination)); + if (existsSync(destination)) { + throw new ImmutableArtifactError(`refusing to overwrite ${relativePath}`); + } + const temporary = this.temporaryFor(destination); + let descriptor: number | null = null; + try { + descriptor = openSync(temporary, "wx"); + let offset = 0; + while (offset < bytes.byteLength) { + offset += writeSync(descriptor, bytes, offset, bytes.byteLength - offset); + } + fsyncSync(descriptor); + closeSync(descriptor); + descriptor = null; + this.faults?.after_file_fsync_before_rename?.(relativePath); + if (existsSync(destination)) { + throw new ImmutableArtifactError(`refusing to overwrite ${relativePath}`); + } + renameSync(temporary, destination); + fsyncDirectory(dirname(destination)); + } catch (error) { + if (descriptor !== null) closeSync(descriptor); + // A real SIGKILL cannot execute cleanup. Retaining the partial in this + // synthetic equivalent lets resume exercise exactly that path. + if (!(error instanceof SimulatedProcessKill) && existsSync(temporary)) unlinkSync(temporary); + throw error; + } + } + + private mirrorExisting(relativePath: string): void { + const primary = this.absolute(relativePath); + const backup = this.absolute(relativePath, this.backupDir); + if (!existsSync(primary)) throw new ImmutableArtifactError(`cannot mirror missing primary ${relativePath}`); + const primaryBytes = readFileSync(primary); + if (existsSync(backup)) { + const backupBytes = readFileSync(backup); + if (sha256(primaryBytes) !== sha256(backupBytes)) { + throw new ImmutableArtifactError(`backup hash differs for ${relativePath}`); + } + return; + } + this.writeNewAt(this.backupDir, relativePath, primaryBytes); + const copied = readFileSync(backup); + if (sha256(primaryBytes) !== sha256(copied)) { + throw new ImmutableArtifactError(`backup hash differs after copy for ${relativePath}`); + } + } + + /** Immutable primary write followed by an independently fsynced mirror. */ + public writeNew(relativePath: string, bytes: Uint8Array | string): void { + const value = typeof bytes === "string" ? Buffer.from(bytes, "utf8") : Buffer.from(bytes); + this.writeNewAt(this.studyDir, relativePath, value); + this.mirrorExisting(relativePath); + } + + public writeJsonNew(relativePath: string, value: unknown): void { + this.writeNew(relativePath, jsonBytes(value)); + } + + public readJson(relativePath: string): T | null { + const path = this.absolute(relativePath); + return existsSync(path) ? parseJsonFile(path, relativePath) : null; + } + + public exists(relativePath: string): boolean { + return existsSync(this.absolute(relativePath)); + } + + /** + * Writes an initial immutable freeze document, or proves a resume is against + * byte-identical commitments. A changed freeze is a different study. + */ + public ensureCommittedJson(relativePath: string, value: unknown): void { + const expected = jsonBytes(value); + const primary = this.absolute(relativePath); + if (!existsSync(primary)) { + this.writeNew(relativePath, expected); + return; + } + const actual = readFileSync(primary); + if (!actual.equals(expected)) { + throw new ImmutableArtifactError(`${relativePath} differs from the already committed freeze`); + } + this.mirrorExisting(relativePath); + } + + private runRelative(logicalRunId: string, name: string): string { + logicalIdPathSafe(logicalRunId); + assertRelative(name); + return join("runs", logicalRunId, name); + } + + private runDirectory(logicalRunId: string): string { + logicalIdPathSafe(logicalRunId); + return this.absolute(join("runs", logicalRunId)); + } + + /** Creates an empty per-run directory atomically, before an agent can start. */ + private ensureRunDirectory(logicalRunId: string): void { + const destination = this.runDirectory(logicalRunId); + if (existsSync(destination)) { + if (!statSync(destination).isDirectory()) throw new ImmutableArtifactError(`run path is not a directory for ${logicalRunId}`); + return; + } + ensureDirectory(dirname(destination)); + const temporary = this.temporaryFor(destination); + try { + mkdirSync(temporary); + mkdirSync(join(temporary, "attempts")); + fsyncDirectory(join(temporary, "attempts")); + fsyncDirectory(temporary); + if (existsSync(destination)) throw new ImmutableArtifactError(`run directory already exists for ${logicalRunId}`); + renameSync(temporary, destination); + fsyncDirectory(dirname(destination)); + this.mirrorRunDirectory(logicalRunId); + } catch (error) { + if (!(error instanceof SimulatedProcessKill) && existsSync(temporary)) rmSync(temporary, { recursive: true, force: true }); + throw error; + } + } + + private mirrorRunDirectory(logicalRunId: string): void { + const primary = this.runDirectory(logicalRunId); + const backup = this.absolute(join("runs", logicalRunId), this.backupDir); + if (!existsSync(backup)) { + ensureDirectory(dirname(backup)); + mkdirSync(backup); + mkdirSync(join(backup, "attempts")); + fsyncDirectory(join(backup, "attempts")); + fsyncDirectory(backup); + fsyncDirectory(dirname(backup)); + } + if (!statSync(primary).isDirectory() || !statSync(backup).isDirectory()) { + throw new ImmutableArtifactError(`run mirror is not a directory for ${logicalRunId}`); + } + } + + /** + * Must be called immediately before spawning the agent. Its presence means + * a process may have reached a model turn; an interrupted launch is therefore + * never guessed to be safe to rerun. + */ + public beginAgentAttempt(checkpoint: AgentLaunchCheckpoint): void { + logicalIdPathSafe(checkpoint.logical_run_id); + attemptIdPathSafe(checkpoint.attempt_id); + this.ensureRunDirectory(checkpoint.logical_run_id); + const dir = this.runRelative(checkpoint.logical_run_id, join("attempts", checkpoint.attempt_id)); + const primary = this.absolute(dir); + if (existsSync(primary)) throw new ImmutableArtifactError(`attempt directory already exists for ${checkpoint.attempt_id}`); + ensureDirectory(primary); + ensureDirectory(this.absolute(dir, this.backupDir)); + fsyncDirectory(dirname(primary)); + fsyncDirectory(dirname(this.absolute(dir, this.backupDir))); + this.writeJsonNew(join(dir, "agent-launched.json"), checkpoint); + } + + /** Durable first-turn marker. It is a state checkpoint, never an outcome. */ + public markFirstModelTurn(checkpoint: AgentStartedCheckpoint): void { + const dir = this.runRelative(checkpoint.logical_run_id, join("attempts", checkpoint.attempt_id)); + if (!existsSync(this.absolute(dir))) { + throw new ImmutableArtifactError(`agent attempt was not launched for ${checkpoint.attempt_id}`); + } + this.writeJsonNew(join(dir, "agent-started.json"), checkpoint); + } + + /** Pre-turn failures have no logical outcome but are preserved for retry lineage. */ + public writePreAgentAttempt(attempt: StoredAttempt): void { + if (attempt.terminal_state !== "PRE_AGENT_INFRA_FAILURE" || attempt.first_model_turn_observed) { + throw new ImmutableArtifactError("pre-agent attempt must be PRE_AGENT_INFRA_FAILURE with no first model turn"); + } + attemptIdPathSafe(attempt.attempt_id); + this.writeJsonNew(join("attempts", `${attempt.attempt_id}.json`), attempt); + } + + /** The terminal agent attempt belongs beside the frozen observation. */ + public writeAgentAttempt(attempt: StoredAttempt): void { + if (!attempt.first_model_turn_observed) { + throw new ImmutableArtifactError("a measured agent attempt must have a durable first-model-turn marker"); + } + const dir = this.runRelative(attempt.logical_run_id, join("attempts", attempt.attempt_id)); + if (!this.exists(join(dir, "agent-started.json"))) { + throw new ImmutableArtifactError(`first model turn was not durably marked for ${attempt.attempt_id}`); + } + this.writeJsonNew(join(dir, "attempt.json"), attempt); + } + + /** Facts needed to resume evaluator-only work, stored beside that agent attempt. */ + public writeAgentObservation(logicalRunId: string, attemptId: string, value: unknown): void { + attemptIdPathSafe(attemptId); + const dir = this.runRelative(logicalRunId, join("attempts", attemptId)); + if (!this.exists(join(dir, "attempt.json"))) { + throw new ImmutableArtifactError(`cannot attach an observation before terminal agent attempt ${attemptId}`); + } + this.writeJsonNew(join(dir, "observation.json"), value); + } + + public readAgentObservation(logicalRunId: string, attemptId: string): T | null { + return this.readJson(this.runRelative(logicalRunId, join("attempts", attemptId, "observation.json"))); + } + + public writeExposure(logicalRunId: string, bytes: Uint8Array): void { + this.ensureRunDirectory(logicalRunId); + const artifact = this.runRelative(logicalRunId, "exposure.jsonl"); + this.writeNew(artifact, bytes); + this.writeNew( + this.runRelative(logicalRunId, "exposure.sha256"), + `${sha256(bytes)} exposure.jsonl\n`, + ); + } + + /** CDEB-05's evidence pair, mirrored as one immutable durable artifact. */ + public writeProviderNdjson(logicalRunId: string, rawNdjson: Uint8Array): void { + this.ensureRunDirectory(logicalRunId); + const raw = Buffer.from(rawNdjson); + const digest = sha256(raw); + this.writeNew(this.runRelative(logicalRunId, "provider.ndjson.zst"), zstdCompressSync(raw)); + this.writeNew(this.runRelative(logicalRunId, "provider.ndjson.sha256"), `${digest} provider.ndjson\n`); + } + + /** Absolute path for a durable run artifact, for the evaluator's read-only mount. */ + public runArtifactPath(logicalRunId: string, name: string): string { + return this.absolute(this.runRelative(logicalRunId, name)); + } + + /** Absolute durable run directory, for readers of multi-file artifacts. */ + public runDirectoryPath(logicalRunId: string): string { + return this.runDirectory(logicalRunId); + } + + /** Archive first, metadata commit record second: no partial tree can verify. */ + public writeFinalTree(logicalRunId: string, archive: Uint8Array, metadata: FinalTreeArtifact): void { + this.ensureRunDirectory(logicalRunId); + if (sha256(archive) !== metadata.archive_sha256) { + throw new ImmutableArtifactError(`final tree archive digest does not match metadata for ${logicalRunId}`); + } + this.writeNew(this.runRelative(logicalRunId, "final-tree.tar.zst"), archive); + this.writeJsonNew(this.runRelative(logicalRunId, "final-tree.json"), metadata); + } + + public writeEvaluatorAttempt(logicalRunId: string, attemptId: string, value: unknown): void { + this.ensureRunDirectory(logicalRunId); + if (!/^e[1-9][0-9]*$/u.test(attemptId)) throw new ImmutableArtifactError(`invalid evaluator attempt id ${attemptId}`); + const relativePath = this.runRelative(logicalRunId, join("evaluator-attempts", `${attemptId}.json`)); + this.writeJsonNew(relativePath, value); + } + + public writeEvaluatorResult(logicalRunId: string, value: unknown): void { + this.writeJsonNew(this.runRelative(logicalRunId, "evaluator.json"), value); + } + + /** The row is the final commit record. It can never be replaced. */ + public writeRow(logicalRunId: string, row: Record): void { + const named = row["logical_run_id"]; + if (named !== logicalRunId) { + throw new ImmutableArtifactError(`row logical_run_id does not match its directory for ${logicalRunId}`); + } + const mirrorRow = this.absolute(join("rows", `${logicalRunId}.json`)); + if (existsSync(mirrorRow)) { + // `rows/` is a legacy alternative verifier input, not a second output + // path. Writing both would look exactly like a duplicate observation. + throw new ImmutableArtifactError(`rows/${logicalRunId}.json already exists; refusing a duplicate logical row`); + } + this.writeJsonNew(this.runRelative(logicalRunId, "row.json"), row); + } + + public readFinalTree(logicalRunId: string): FinalTreeArtifact | null { + return this.readJson(this.runRelative(logicalRunId, "final-tree.json")); + } + + public readRunState(logicalRunId: string): StoredRunState { + logicalIdPathSafe(logicalRunId); + const runDir = this.runDirectory(logicalRunId); + const row = this.readJson>(this.runRelative(logicalRunId, "row.json")); + const finalTree = this.readFinalTree(logicalRunId); + const launched: string[] = []; + const started: string[] = []; + const agentAttempts: StoredAttempt[] = []; + const attemptsDir = join(runDir, "attempts"); + if (existsSync(attemptsDir)) { + for (const name of readdirSync(attemptsDir).sort()) { + const attemptDir = join(attemptsDir, name); + if (!statSync(attemptDir).isDirectory()) throw new ImmutableArtifactError(`run attempt entry is not a directory: ${name}`); + if (existsSync(join(attemptDir, "agent-launched.json"))) launched.push(name); + if (existsSync(join(attemptDir, "agent-started.json"))) started.push(name); + if (existsSync(join(attemptDir, "attempt.json"))) { + agentAttempts.push(parseJsonFile(join(attemptDir, "attempt.json"), `attempt ${name}`)); + } + } + } + const evaluatorDir = join(runDir, "evaluator-attempts"); + const evaluatorAttemptCount = existsSync(evaluatorDir) + ? readdirSync(evaluatorDir).filter((name) => name.endsWith(".json") && statSync(join(evaluatorDir, name)).isFile()).length + : 0; + return { + logical_run_id: logicalRunId, + row, + final_tree: finalTree, + launched_attempt_ids: launched, + started_attempt_ids: started, + agent_attempts: agentAttempts, + evaluator_attempt_count: evaluatorAttemptCount, + }; + } + + public preAgentAttempts(logicalRunId: string): StoredAttempt[] { + const directory = this.absolute("attempts"); + if (!existsSync(directory)) return []; + const records: StoredAttempt[] = []; + for (const name of readdirSync(directory).sort()) { + const path = join(directory, name); + if (!name.endsWith(".json") || !statSync(path).isFile()) { + throw new ImmutableArtifactError(`top-level attempts contains a non-JSON artifact: ${name}`); + } + const value = parseJsonFile(path, `attempt ${name}`); + if (value.logical_run_id === logicalRunId) records.push(value); + } + return records; + } + + /** + * Finds completed ids from durable row commit records, rejecting all the + * ambiguous cases before a resume can decide what to run. + */ + public completedRows(expectedLogicalIds: readonly string[]): ReadonlyMap> { + const expected = new Set(expectedLogicalIds); + if (expected.size !== expectedLogicalIds.length) throw new ImmutableArtifactError("expected logical ids are duplicated"); + const rows = new Map>(); + const runs = this.absolute("runs"); + if (existsSync(runs)) { + for (const name of readdirSync(runs).sort()) { + const runDir = join(runs, name); + if (!statSync(runDir).isDirectory()) throw new ImmutableArtifactError(`runs contains a non-directory artifact: ${name}`); + if (name.includes(".partial")) throw new ImmutableArtifactError(`unrecovered partial run directory: ${name}`); + const path = join(runDir, "row.json"); + if (!existsSync(path)) continue; + const row = parseJsonFile>(path, `row ${name}`); + const id = row["logical_run_id"]; + if (typeof id !== "string" || id !== name) throw new ImmutableArtifactError(`row directory/id mismatch in ${name}`); + if (!expected.has(id)) throw new ImmutableArtifactError(`durable row ${id} is not named by this randomization`); + if (rows.has(id)) throw new ImmutableArtifactError(`duplicate logical row ${id}`); + rows.set(id, row); + } + } + const legacyRows = this.absolute("rows"); + if (existsSync(legacyRows) && readdirSync(legacyRows).length > 0) { + // The verifier treats `rows/` and `runs/*/row.json` as alternative input + // layouts; accepting both would create an unresolvable duplicate. + throw new ImmutableArtifactError("rows/ is populated alongside run directories; refusing duplicate row surfaces"); + } + return rows; + } + + public missingLogicalIds(expectedLogicalIds: readonly string[]): string[] { + const complete = this.completedRows(expectedLogicalIds); + return expectedLogicalIds.filter((id) => !complete.has(id)); + } + + /** Removes only unpublished names generated by this module's atomic writer. */ + public recoverUnpublishedPartials(): number { + let removed = 0; + const removePartialsUnder = (directory: string): void => { + if (!existsSync(directory)) return; + for (const name of readdirSync(directory)) { + const path = join(directory, name); + const stat = statSync(path); + if (name.includes(".partial")) { + rmSync(path, { recursive: stat.isDirectory(), force: true }); + removed += 1; + } else if (stat.isDirectory()) { + removePartialsUnder(path); + } + } + }; + removePartialsUnder(this.studyDir); + removePartialsUnder(this.backupDir); + const removeUncommittedPairs = (root: string): void => { + const runs = join(root, "runs"); + if (!existsSync(runs)) return; + for (const name of readdirSync(runs)) { + const run = join(runs, name); + if (!statSync(run).isDirectory()) continue; + const removePairWhenIncomplete = (left: string, right: string): void => { + const leftPath = join(run, left); + const rightPath = join(run, right); + if (existsSync(leftPath) !== existsSync(rightPath)) { + // Neither half has its commit record. It was never a readable + // CDEB artifact and deleting it prevents a verifier from treating + // a byte blob as a frozen tree or provider stream. + rmSync(leftPath, { force: true }); + rmSync(rightPath, { force: true }); + removed += 1; + } + }; + removePairWhenIncomplete("provider.ndjson.zst", "provider.ndjson.sha256"); + removePairWhenIncomplete("exposure.jsonl", "exposure.sha256"); + removePairWhenIncomplete("final-tree.tar.zst", "final-tree.json"); + } + }; + removeUncommittedPairs(this.studyDir); + removeUncommittedPairs(this.backupDir); + return removed; + } + + /** + * Completes backup copies interrupted after the primary rename. It reads + * primary bytes only and refuses a divergent backup, so recovery never + * rewrites or selects between two observations. + */ + public repairBackupMirrors(): void { + const walk = (directory: string, prefix: string): void => { + if (!existsSync(directory)) return; + for (const name of readdirSync(directory).sort()) { + if (name.includes(".partial")) { + throw new ImmutableArtifactError(`cannot mirror unpublished partial ${join(prefix, name)}`); + } + const path = join(directory, name); + const rel = prefix === "" ? name : join(prefix, name); + const stat = statSync(path); + if (stat.isDirectory()) walk(path, rel); + else if (stat.isFile()) this.mirrorExisting(rel); + else throw new ImmutableArtifactError(`authoritative storage contains a non-file artifact ${rel}`); + } + }; + walk(this.studyDir, ""); + } + + /** Repairs a missing backup copy, but never changes primary evidence. */ + public ensureBackup(relativePath: string): void { + this.mirrorExisting(relativePath); + } +} diff --git a/bench/cdeb/verify.mjs b/bench/cdeb/verify.mjs index 5b626a38..8a4e7b01 100644 --- a/bench/cdeb/verify.mjs +++ b/bench/cdeb/verify.mjs @@ -186,6 +186,70 @@ const checkProviderArtifact = (study, runDir, row) => { } }; +/** + * CDEB-07: a final tree is an archive PLUS its metadata commit record. An + * archive without final-tree.json is deliberately not a tree that can verify; + * accepting it would turn a kill between the two writes into durable-looking + * evidence. The metadata's digest binds the bytes and the row binds both + * object identity and digests. + */ +const checkFinalTreeArtifact = (study, runDir, row) => { + const archivePath = join(runDir, "final-tree.tar.zst"); + const metadataPath = join(runDir, "final-tree.json"); + const hasArchive = existsSync(archivePath); + const hasMetadata = existsSync(metadataPath); + if (!hasArchive && !hasMetadata) { + if (row !== null) fail(study, `${runDir}: row.json has no final tree artifact`); + return; + } + if (!hasArchive || !hasMetadata) { + fail(study, `${runDir}: final tree archive and metadata must appear together`); + return; + } + const metadata = readJson(study, metadataPath); + if (metadata === null || typeof metadata !== "object" || Array.isArray(metadata)) return; + const expectedKeys = [ + "archive_sha256", "base_tree_oid", "canonical_diff_sha256", "final_tree_oid", "schema_version", "workspace_status_digest", + ].sort(); + const actualKeys = Object.keys(metadata).sort(); + if (actualKeys.length !== expectedKeys.length || actualKeys.some((key, index) => key !== expectedKeys[index])) { + fail(study, `${metadataPath}: final tree metadata has an unexpected shape`); + return; + } + if (metadata.schema_version !== 1) { + fail(study, `${metadataPath}: final tree metadata schema_version must be 1`); + return; + } + for (const key of ["base_tree_oid", "final_tree_oid"] ) { + if (typeof metadata[key] !== "string" || !/^[0-9a-f]{40}$/.test(metadata[key])) { + fail(study, `${metadataPath}: ${key} is not a git object id`); + } + } + for (const key of ["archive_sha256", "canonical_diff_sha256", "workspace_status_digest"]) { + if (typeof metadata[key] !== "string" || !/^[0-9a-f]{64}$/.test(metadata[key])) { + fail(study, `${metadataPath}: ${key} is not a sha256`); + } + } + const archiveDigest = createHash("sha256").update(readFileSync(archivePath)).digest("hex"); + if (archiveDigest !== metadata.archive_sha256) { + fail(study, `${runDir}: final tree archive digest does not match metadata`); + } + if (row !== null) { + if (row.final_tree.final_tree_oid !== metadata.final_tree_oid) { + fail(study, `${runDir}: row final tree oid does not match final-tree.json`); + } + if (row.final_tree.archive_sha256 !== metadata.archive_sha256) { + fail(study, `${runDir}: row final tree archive digest does not match final-tree.json`); + } + if (row.final_tree.canonical_diff_sha256 !== metadata.canonical_diff_sha256) { + fail(study, `${runDir}: row canonical diff digest does not match final-tree.json`); + } + if (row.final_tree.workspace_status_digest !== metadata.workspace_status_digest) { + fail(study, `${runDir}: row workspace status digest does not match final-tree.json`); + } + } +}; + const verifyStudy = (root, studyName) => { const study = studyName; const dir = join(root, studyName); @@ -306,6 +370,7 @@ const verifyStudy = (root, studyName) => { const rowPath = join(runDir, "row.json"); const row = existsSync(rowPath) ? verifyRow(rowPath) : null; checkProviderArtifact(study, runDir, row); + checkFinalTreeArtifact(study, runDir, row); const evalPath = join(runDir, "evaluator.json"); if (existsSync(evalPath)) { validateAgainst(study, "evaluator", evalPath, readJson(study, evalPath)); diff --git a/test/cdeb-orchestrator.test.ts b/test/cdeb-orchestrator.test.ts new file mode 100644 index 00000000..8f38a367 --- /dev/null +++ b/test/cdeb-orchestrator.test.ts @@ -0,0 +1,380 @@ +/** + * CDEB-07 acceptance: exercise the real coordinator over recorded provider + * bytes, the real final-tree freezer, the CDEB-05 ledger, and CDEB-06's + * evaluator entrypoint. The only substituted boundary is the provider/OCI + * process itself; its responses are recorded fixtures so no test can call a + * provider or require a container daemon. + */ + +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { afterAll, describe, expect, it } from "vitest"; + +import { normalizedResultSha256 } from "../bench/cdeb/evaluator/engine.ts"; +import { evaluateLocal } from "../bench/cdeb/evaluator/runner-local.ts"; +import { + blockedRandomization, + canonicalFinalTreeFreezer, + runStudy, + summarizeExposure, + type AgentRunner, + type CdebCondition, + type CdebStudyPlan, + type EvaluatorRunner, + type LogicalRunPlan, + type OrchestratorDependencies, + type OutcomeFreeProgress, + type PreparedWorkspace, +} from "../bench/cdeb/orchestrator.ts"; +import { readProviderLedger } from "../bench/cdeb/runtime/provider-ledger.ts"; +import { DurableStudyStorage, SimulatedProcessKill } from "../bench/cdeb/storage.ts"; +import { FIXTURE_ROOT, SEALED_DIR, TASK_ID, TEST_IMAGE_DIGEST } from "./cdeb-evaluator-helpers.ts"; + +const scratch: string[] = []; +afterAll(() => { + for (const directory of scratch) rmSync(directory, { recursive: true, force: true }); +}); + +const temp = (label: string): string => { + const directory = mkdtempSync(join(tmpdir(), `cdeb-orchestrator-${label}-`)); + scratch.push(directory); + return directory; +}; + +const git = (cwd: string, args: readonly string[]): string => + execFileSync("git", [...args], { cwd, encoding: "utf8" }).trim(); + +const HEX = "a".repeat(64); +const OID = "b".repeat(40); +const RECORDED_STREAM = (): Buffer => readFileSync("test/fixtures/claude-stream/partial-messages.jsonl"); +const RECORDED_MODEL = "claude-haiku-4-5-20251001"; + +interface Counters { + readonly agent: Map; + readonly evaluator: Map; + readonly evaluatorTreeOids: Map; +} + +const counter = (): Counters => ({ agent: new Map(), evaluator: new Map(), evaluatorTreeOids: new Map() }); +const increment = (map: Map, key: string): number => { + const next = (map.get(key) ?? 0) + 1; + map.set(key, next); + return next; +}; + +const workspaceFor = (): PreparedWorkspace => { + const root = temp("workspace"); + const workdir = join(root, "tree"); + const configDir = join(root, "config"); + cpSync(join(FIXTURE_ROOT, "base"), workdir, { recursive: true }); + mkdirSync(configDir, { recursive: true }); + writeFileSync(join(configDir, "settings.json"), "{\"hooks\":{}}\n"); + writeFileSync(join(configDir, "mcp.json"), "{\"mcpServers\":{}}\n"); + git(workdir, ["init", "--quiet"]); + git(workdir, ["config", "user.email", "cdeb@example.test"]); + git(workdir, ["config", "user.name", "CDEB test"]); + git(workdir, ["add", "-A"]); + git(workdir, ["commit", "--quiet", "-m", "base"]); + const exposurePath = join(workdir, ".git", "cdeb", "exposure.jsonl"); + mkdirSync(dirname(exposurePath), { recursive: true }); + writeFileSync(exposurePath, ""); + return { + workdir, + exposure_path: exposurePath, + config_dir: configDir, + cleanup: () => rmSync(root, { recursive: true, force: true }), + }; +}; + +const agentRunner = (counts: Counters, preTurnFailures = 0): AgentRunner => ({ + run: async ({ plan, workspace, on_first_model_turn }) => { + const ordinal = increment(counts.agent, plan.logical_run_id); + if (ordinal <= preTurnFailures) { + return { kind: "before-first-model-turn", failure_detail: "recorded preflight transport refusal" }; + } + on_first_model_turn(); + writeFileSync( + join(workspace.workdir, "src", "calc.js"), + readFileSync(join(FIXTURE_ROOT, "patches", "good", "calc.js"), "utf8"), + ); + const raw = RECORDED_STREAM(); + return { + kind: "after-first-model-turn", + started_at: "2026-08-11T00:00:00.000Z", + finished_at: "2026-08-11T00:00:01.000Z", + stop_reason: "completed", + provider_ledger: readProviderLedger({ requested_model: RECORDED_MODEL, raw_ndjson: raw }), + raw_provider_ndjson: raw, + }; + }, +}); + +const evaluatorRunner = (counts: Counters, failFirst: boolean): EvaluatorRunner => ({ + evaluate: async ({ plan, archive_path, final_tree }) => { + const ordinal = increment(counts.evaluator, plan.logical_run_id); + const trees = counts.evaluatorTreeOids.get(plan.logical_run_id) ?? []; + trees.push(final_tree.final_tree_oid); + counts.evaluatorTreeOids.set(plan.logical_run_id, trees); + if (failFirst && ordinal === 1) { + return { kind: "infrastructure-failure", failure_detail: "recorded evaluator transport interruption" }; + } + const local = evaluateLocal({ + tasksDir: SEALED_DIR, + taskId: TASK_ID, + archivePath: archive_path, + claimedOid: final_tree.final_tree_oid, + imageDigest: TEST_IMAGE_DIGEST, + }); + if (local.verdict === null) { + return { kind: "infrastructure-failure", failure_detail: local.stderr || "recorded evaluator did not return a verdict" }; + } + return { kind: "verdict", verdict: local.verdict }; + }, +}); + +const makePlan = (condition: CdebCondition, order: number): LogicalRunPlan => { + const suffix = condition === "commitlore-on" ? "on" : "off"; + const logical_run_id = `repo-a__${TASK_ID}__${suffix}__r1`; + return { + logical_run_id, + repository_id: "repo-a", + task_id: TASK_ID, + category: "rejected-architecture", + condition, + repeat: 1, + order, + requested_model: RECORDED_MODEL, + prompt: "Fix calc without running external services.", + expected_record_ids: [], + make_row: ({ agent, exposure, final_tree, evaluator, evaluator_attempts }) => ({ + schema_version: 1, + benchmark: "cdeb-v1", + protocol_version: "1.3.0", + study_id: "cdeb-orchestrator-test", + logical_run_id, + repository_id: "repo-a", + task_id: TASK_ID, + category: "rejected-architecture", + condition, + repeat: 1, + order, + freeze_manifest_sha256: HEX, + sealed_task_bundle_sha256: HEX, + repository_bundle_sha256: HEX, + repository_snapshot: OID, + base_tree_oid: final_tree.base_tree_oid, + refs_digest: HEX, + notes_ref_digest: HEX, + requested_model: RECORDED_MODEL, + observed_model_ids: agent.provider_ledger.observed_model_ids, + agent_cli_version: "2.1.220", + agent_executable_sha256: HEX, + node_version: process.version, + node_executable_sha256: HEX, + agent_runtime_image_digest: `sha256:${HEX}`, + tool_policy_digest: HEX, + network_policy_digest: HEX, + settings_digest: HEX, + mcp_config_digest: HEX, + harness_commit: OID, + product_commit: OID, + dist_digest: HEX, + hook_proxy_sha256: HEX, + started_at: agent.started_at, + finished_at: agent.finished_at, + stop_reason: agent.stop_reason, + first_model_turn_observed: true, + wall_ms: 1_000, + exposure, + usage: agent.provider_ledger.usage, + final_tree: { + final_tree_oid: final_tree.final_tree_oid, + canonical_diff_sha256: final_tree.canonical_diff_sha256, + archive_sha256: final_tree.archive_sha256, + workspace_status_digest: final_tree.workspace_status_digest, + }, + evaluation: { + evaluator_image_digest: evaluator.evaluator_image_digest, + evaluator_attempts, + functional_pass: evaluator.functional_pass, + rejected_decision_revived: evaluator.rejected_decision_revived, + normalized_result_sha256: normalizedResultSha256(evaluator), + }, + decision_safe_success: + agent.stop_reason === "completed" && evaluator.functional_pass && !evaluator.rejected_decision_revived, + simulated: false, + }), + }; +}; + +const studyPlan = (): CdebStudyPlan => ({ + public_freeze: { benchmark: "cdeb-v1", study_id: "cdeb-orchestrator-test", freeze: "fixture" }, + randomization: { + schema_version: 1, + algorithm: "sha256-key-sort-v1", + block_count: 1, + blocks: [{ block_index: "block-000", conditions: ["commitlore-on", "commitlore-off"] }], + }, + logical_runs: [makePlan("commitlore-on", 1), makePlan("commitlore-off", 2)], +}); + +const dependencies = (counts: Counters, options: { failEvaluatorFirst?: boolean; preTurnFailures?: number } = {}): OrchestratorDependencies => ({ + prepare_workspace: async () => workspaceFor(), + agent: agentRunner(counts, options.preTurnFailures ?? 0), + freeze_tree: canonicalFinalTreeFreezer, + collect_exposure: (workspace, plan) => summarizeExposure(workspace.exposure_path, plan.expected_record_ids, false), + evaluator: evaluatorRunner(counts, options.failEvaluatorFirst ?? false), +}); + +describe("CDEB-07 blocked randomization", () => { + it("publishes only opaque indices and randomized arm order", () => { + const randomized = blockedRandomization( + [ + { sealed_key: "repo-secret/task-secret/r1", value: { task_id: "task-secret" } }, + { sealed_key: "repo-secret/task-other/r1", value: { task_id: "task-other" } }, + ], + "frozen-seed", + ); + const publicText = JSON.stringify(randomized.public_manifest); + expect(publicText).not.toContain("task-secret"); + expect(publicText).not.toContain("repo-secret"); + expect(randomized.public_manifest.blocks.map((block) => block.block_index)).toEqual(["block-000", "block-001"]); + expect(randomized.sealed_schedule.map((block) => block.value.task_id).sort()).toEqual(["task-other", "task-secret"]); + for (const block of randomized.public_manifest.blocks) { + expect([...block.conditions].sort()).toEqual(["commitlore-off", "commitlore-on"]); + } + }); +}); + +describe("CDEB-07 state machine", () => { + it("never reruns an agent after its first turn; evaluator retries receive the same frozen tree", async () => { + const root = temp("evaluator-retry"); + const storage = new DurableStudyStorage({ studyDir: join(root, "study"), backupDir: join(root, "backup") }); + const counts = counter(); + const plan = studyPlan(); + await runStudy(plan, dependencies(counts, { failEvaluatorFirst: true }), { storage }); + + const onId = plan.logical_runs[0]!.logical_run_id; + expect(counts.agent.get(onId)).toBe(1); + expect(counts.evaluator.get(onId)).toBe(2); + expect(new Set(counts.evaluatorTreeOids.get(onId)).size).toBe(1); + + // A normal resume sees row.json and cannot reach either provider or evaluator. + await runStudy(plan, dependencies(counts, { failEvaluatorFirst: true }), { storage }); + expect(counts.agent.get(onId)).toBe(1); + expect(counts.evaluator.get(onId)).toBe(2); + }); + + it("retries only a typed pre-first-turn failure and preserves its attempt lineage", async () => { + const root = temp("pre-turn-retry"); + const storage = new DurableStudyStorage({ studyDir: join(root, "study"), backupDir: join(root, "backup") }); + const counts = counter(); + await runStudy(studyPlan(), dependencies(counts, { preTurnFailures: 1 }), { storage }); + const id = makePlan("commitlore-on", 1).logical_run_id; + expect(counts.agent.get(id)).toBe(2); + const attempts = storage.preAgentAttempts(id); + expect(attempts).toHaveLength(1); + expect(attempts[0]?.first_model_turn_observed).toBe(false); + }); + + it("resume launches only missing logical ids after an interruption between durable rows", async () => { + const root = temp("resume-missing"); + const storage = new DurableStudyStorage({ studyDir: join(root, "study"), backupDir: join(root, "backup") }); + const counts = counter(); + const plan = studyPlan(); + let interrupted = false; + await expect(runStudy(plan, dependencies(counts), { + storage, + progress: (progress) => { + if (!interrupted && progress.state === "MEASURED") { + interrupted = true; + throw new SimulatedProcessKill("process killed between logical rows"); + } + }, + })).rejects.toThrow(SimulatedProcessKill); + + const [first, second] = plan.logical_runs; + expect(counts.agent.get(first!.logical_run_id)).toBe(1); + expect(counts.agent.get(second!.logical_run_id) ?? 0).toBe(0); + + const resumed = await runStudy(plan, dependencies(counts), { storage }); + expect(resumed.missing_logical_run_ids).toEqual([]); + expect(counts.agent.get(first!.logical_run_id)).toBe(1); + expect(counts.agent.get(second!.logical_run_id)).toBe(1); + }); + + it("cleans an fsynced-but-unrenamed row partial and finishes that row without a second evaluation or agent", async () => { + const root = temp("atomic-row"); + let killed = false; + const storage = new DurableStudyStorage({ + studyDir: join(root, "study"), + backupDir: join(root, "backup"), + faults: { + after_file_fsync_before_rename: (relativePath) => { + if (!killed && relativePath.endsWith("row.json")) { + killed = true; + throw new SimulatedProcessKill("killed after row fsync before rename"); + } + }, + }, + }); + const counts = counter(); + const plan = studyPlan(); + await expect(runStudy(plan, dependencies(counts), { storage })).rejects.toThrow(SimulatedProcessKill); + const first = plan.logical_runs[0]!; + expect(counts.agent.get(first.logical_run_id)).toBe(1); + expect(counts.evaluator.get(first.logical_run_id)).toBe(1); + + const resumedStorage = new DurableStudyStorage({ studyDir: join(root, "study"), backupDir: join(root, "backup") }); + await runStudy(plan, dependencies(counts), { storage: resumedStorage }); + expect(counts.agent.get(first.logical_run_id)).toBe(1); + expect(counts.evaluator.get(first.logical_run_id)).toBe(1); + }); + + it("treats a kill between final-tree archive and metadata as incomplete, never as permission to rerun its agent", async () => { + const root = temp("atomic-tree"); + let killed = false; + const storage = new DurableStudyStorage({ + studyDir: join(root, "study"), + backupDir: join(root, "backup"), + faults: { + after_file_fsync_before_rename: (relativePath) => { + if (!killed && relativePath.endsWith("final-tree.json")) { + killed = true; + throw new SimulatedProcessKill("killed after final-tree metadata fsync before rename"); + } + }, + }, + }); + const counts = counter(); + const plan = studyPlan(); + await expect(runStudy(plan, dependencies(counts), { storage })).rejects.toThrow(SimulatedProcessKill); + const first = plan.logical_runs[0]!; + + const resumedStorage = new DurableStudyStorage({ studyDir: join(root, "study"), backupDir: join(root, "backup") }); + await expect(runStudy(plan, dependencies(counts), { storage: resumedStorage })).rejects.toThrow(/refusing an agent rerun/); + expect(counts.agent.get(first.logical_run_id)).toBe(1); + expect(resumedStorage.exists(join("runs", first.logical_run_id, "final-tree.tar.zst"))).toBe(false); + expect(resumedStorage.exists(join("runs", first.logical_run_id, "final-tree.json"))).toBe(false); + }); +}); + +describe("CDEB-07 outcome-free progress", () => { + it("exposes exactly lifecycle and count fields, never an outcome surface", async () => { + const root = temp("progress"); + const storage = new DurableStudyStorage({ studyDir: join(root, "study"), backupDir: join(root, "backup") }); + const observed: OutcomeFreeProgress[] = []; + await runStudy(studyPlan(), dependencies(counter()), { storage, progress: (event) => observed.push(event) }); + expect(observed.length).toBeGreaterThan(0); + for (const event of observed) { + expect(Object.keys(event).sort()).toEqual([ + "attempt_count", "completed", "logical_run_id", "remaining", "state", + ]); + expect(Object.isFrozen(event)).toBe(true); + expect(JSON.stringify(event)).not.toMatch(/functional|decision_safe|revived|token|usage|aggregate/i); + } + }); +}); diff --git a/test/cdeb-runtime-isolation.test.ts b/test/cdeb-runtime-isolation.test.ts index 767868c9..7d656aaf 100644 --- a/test/cdeb-runtime-isolation.test.ts +++ b/test/cdeb-runtime-isolation.test.ts @@ -610,10 +610,14 @@ describe('pin manifest and gate token', () => { it('executeAgentRun captures the raw stream byte-for-byte, persists it, and identity-checks it', async () => { const stream = validStream(); + const firstTurns: string[] = []; const streamingDocker: ContainerRuntimeCommands = { run: () => ({ stdout: '', stderr: '', exitCode: 0, timedOut: false }), runToSink: async (_args, sink) => { - sink.write(stream); + // Deliberately split a NDJSON event across chunks: CDEB-07's durable + // non-rerun marker must observe lines, not assume chunk boundaries. + sink.write(stream.slice(0, 37)); + sink.write(stream.slice(37)); sink.end(); return { exitCode: 0, stderr: '', timedOut: false }; }, @@ -626,6 +630,7 @@ describe('pin manifest and gate token', () => { prompt: 'task', outDir, providerEnv: {}, + onFirstModelTurn: () => firstTurns.push('observed-before-stream-completes'), }); const captured = readPersistedRawNdjson(outDir).toString('utf8'); expect(captured).toBe(stream); @@ -633,6 +638,7 @@ describe('pin manifest and gate token', () => { expect(outcome.ledger.usage.availability).toBe('measured'); expect(outcome.exit_code).toBe(0); expect(outcome.provider_stream_sha256).toMatch(/^[0-9a-f]{64}$/); + expect(firstTurns).toEqual(['observed-before-stream-completes']); }); it('executeAgentRun turns mid-run model drift into a hard stop after capture', async () => { diff --git a/test/cdeb-verify.test.ts b/test/cdeb-verify.test.ts index 88c72e48..c1e276b6 100644 --- a/test/cdeb-verify.test.ts +++ b/test/cdeb-verify.test.ts @@ -271,6 +271,18 @@ describe('#443 the CDEB recursive verifier', () => { expect(result.output).toMatch(/raw_stream_sha256 does not match provider NDJSON/); }); + it('fails an archive that was fsynced but never received final-tree.json', () => { + const root = study('half-final-tree', []); + const runDir = join(root, 'cdeb-test-01', 'runs', 'repo-a__task-a__on__r1'); + mkdirSync(runDir, { recursive: true }); + // This is the crash window CDEB-07 recovers before resume. It must not + // verify as a final tree merely because its archive bytes are complete. + writeFileSync(join(runDir, 'final-tree.tar.zst'), Buffer.from('not-a-committed-tree')); + const result = verify(root); + expect(result.code).toBe(1); + expect(result.output).toMatch(/final tree archive and metadata must appear together/); + }); + it('accepts an unavailable usage row without inventing a numeric total', () => { const row = validRow({ usage: {