Skip to content

Commit 6c7791e

Browse files
ALikesToCodeALikesToCode
committed
fix(runtime): guide bounded scientific repairs
1 parent 5c34919 commit 6c7791e

5 files changed

Lines changed: 449 additions & 98 deletions

File tree

‎packages/codex-client/src/app-server.ts‎

Lines changed: 125 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import {
1616
ExperimentIRPolicyError,
1717
ExperimentIRV5Schema,
1818
hashExperimentIR,
19-
type ExperimentIRPolicyFindingCode,
2019
} from "@counterlab/experiment-ir";
2120
import {
2221
LabSceneDraftV2Schema,
@@ -36,6 +35,14 @@ import {
3635
buildRepairHostedPatchPlanPrompt,
3736
buildRepairHostedScientificMethodPrompt,
3837
} from "./prompts.js";
38+
import {
39+
buildScientificPolicyRepairPrompt,
40+
mismatchedScientificFindingPaths,
41+
normalizeScientificFindingPaths,
42+
ScientificArtifactValidationError,
43+
type ScientificArtifactFindingCode,
44+
zodScientificFindingPaths,
45+
} from "./scientific-repair.js";
3946
import { redactSecrets, sanitizeAppServerMessage } from "./sanitizer.js";
4047
import {
4148
CompileLabInputSchema,
@@ -74,27 +81,6 @@ const DISCRIMINATION_CONTRACT_PATH = "discrimination-contract.json";
7481
const EXPERIMENT_IR_PATH = "experiment-ir.json";
7582
const LAB_SCENE_PATH = "lab-scene.json";
7683

77-
type ScientificArtifactFindingCode =
78-
| ExperimentIRPolicyFindingCode
79-
| "OUTPUT_SCHEMA_INVALID"
80-
| "IMMUTABLE_LINEAGE_MISMATCH"
81-
| "CANDIDATE_REFERENCE_INVALID"
82-
| "LAB_SCENE_INVALID";
83-
84-
class ScientificArtifactValidationError extends Error {
85-
readonly findingCode: ScientificArtifactFindingCode;
86-
87-
constructor(
88-
findingCode: ScientificArtifactFindingCode,
89-
message: string,
90-
options?: ErrorOptions,
91-
) {
92-
super(message, options);
93-
this.name = "ScientificArtifactValidationError";
94-
this.findingCode = findingCode;
95-
}
96-
}
97-
9884
const StructuredHostedOutputSchema = z
9985
.object({
10086
authoritativeArtifact: JsonValueSchema,
@@ -371,7 +357,7 @@ async function materializeStructuredScientificOutput(
371357
throw new ScientificArtifactValidationError(
372358
"OUTPUT_SCHEMA_INVALID",
373359
"Codex did not return schema-constrained scientific artifacts.",
374-
{ cause: error },
360+
{ cause: error, findingPaths: ["$"] },
375361
);
376362
}
377363

@@ -380,7 +366,10 @@ async function materializeStructuredScientificOutput(
380366
throw new ScientificArtifactValidationError(
381367
"OUTPUT_SCHEMA_INVALID",
382368
"Codex scientific output did not match the fixed response envelope.",
383-
{ cause: envelopeResult.error },
369+
{
370+
cause: envelopeResult.error,
371+
findingPaths: zodScientificFindingPaths(envelopeResult.error, "$"),
372+
},
384373
);
385374
}
386375
const artifactsResult = z
@@ -399,50 +388,85 @@ async function materializeStructuredScientificOutput(
399388
throw new ScientificArtifactValidationError(
400389
"OUTPUT_SCHEMA_INVALID",
401390
"Codex scientific artifacts did not match the fixed local schemas.",
402-
{ cause: artifactsResult.error },
391+
{
392+
cause: artifactsResult.error,
393+
findingPaths: zodScientificFindingPaths(
394+
artifactsResult.error,
395+
"authoritativeArtifact",
396+
),
397+
},
403398
);
404399
}
405400
const envelope = envelopeResult.data;
406401
const artifacts = artifactsResult.data;
407402
const beliefSpec = BeliefSpecV2Schema.parse(input.approvedBeliefSpec);
408-
if (
409-
artifacts.experimentIr.selection.status !== "UNSELECTED" ||
410-
artifacts.discriminationContract.sessionId !== input.sessionId ||
411-
artifacts.experimentIr.sessionId !== input.sessionId ||
412-
artifacts.labScene.sessionId !== input.sessionId ||
413-
artifacts.discriminationContract.artifactManifestHash !==
414-
input.artifactManifestHash ||
415-
artifacts.experimentIr.artifactManifestHash !==
416-
input.artifactManifestHash ||
417-
artifacts.discriminationContract.beliefSpecId !== beliefSpec.id ||
418-
artifacts.experimentIr.beliefSpecId !== beliefSpec.id ||
419-
artifacts.discriminationContract.beliefSpecHash !== input.beliefSpecHash ||
420-
artifacts.experimentIr.beliefSpecHash !== input.beliefSpecHash ||
421-
artifacts.discriminationContract.concept !== beliefSpec.concept ||
422-
artifacts.experimentIr.concept !== beliefSpec.concept ||
423-
artifacts.labScene.concept !== beliefSpec.concept ||
424-
artifacts.discriminationContract.conceptPackVersion !==
425-
input.conceptPack.version ||
426-
artifacts.experimentIr.conceptPackVersion !== input.conceptPack.version ||
427-
artifacts.experimentIr.provenance.kind !== "codex" ||
428-
artifacts.experimentIr.provenance.generatorId !==
429-
input.provenance.generatorId ||
430-
artifacts.experimentIr.provenance.promptHash !==
431-
input.provenance.promptHash ||
432-
JSON.stringify(artifacts.experimentIr.provenance.inputHashes) !==
433-
JSON.stringify(input.provenance.inputHashes) ||
434-
artifacts.discriminationContract.hypotheses[0].statement !==
435-
beliefSpec.hypotheses[0].statement ||
436-
artifacts.discriminationContract.hypotheses[1].statement !==
437-
beliefSpec.hypotheses[1].statement ||
438-
artifacts.experimentIr.hypotheses[0].statement !==
439-
beliefSpec.hypotheses[0].statement ||
440-
artifacts.experimentIr.hypotheses[1].statement !==
441-
beliefSpec.hypotheses[1].statement
442-
) {
403+
const immutableLineageFindingPaths = mismatchedScientificFindingPaths([
404+
{
405+
path: "experimentIr.selection.status",
406+
actual: artifacts.experimentIr.selection.status,
407+
expected: "UNSELECTED",
408+
},
409+
...(
410+
[
411+
["discriminationContract", artifacts.discriminationContract],
412+
["experimentIr", artifacts.experimentIr],
413+
["labScene", artifacts.labScene],
414+
] as const
415+
).flatMap(([path, artifact]) => [
416+
{
417+
path: `${path}.sessionId`,
418+
actual: artifact.sessionId,
419+
expected: input.sessionId,
420+
},
421+
{
422+
path: `${path}.concept`,
423+
actual: artifact.concept,
424+
expected: beliefSpec.concept,
425+
},
426+
]),
427+
...(
428+
[
429+
["discriminationContract", artifacts.discriminationContract],
430+
["experimentIr", artifacts.experimentIr],
431+
] as const
432+
).flatMap(([path, artifact]) => [
433+
{
434+
path: `${path}.artifactManifestHash`,
435+
actual: artifact.artifactManifestHash,
436+
expected: input.artifactManifestHash,
437+
},
438+
{
439+
path: `${path}.beliefSpecId`,
440+
actual: artifact.beliefSpecId,
441+
expected: beliefSpec.id,
442+
},
443+
{
444+
path: `${path}.beliefSpecHash`,
445+
actual: artifact.beliefSpecHash,
446+
expected: input.beliefSpecHash,
447+
},
448+
{
449+
path: `${path}.conceptPackVersion`,
450+
actual: artifact.conceptPackVersion,
451+
expected: input.conceptPack.version,
452+
},
453+
...beliefSpec.hypotheses.map((hypothesis, index) => ({
454+
path: `${path}.hypotheses[${index}].statement`,
455+
actual: artifact.hypotheses[index]?.statement,
456+
expected: hypothesis.statement,
457+
})),
458+
]),
459+
{
460+
path: "experimentIr.provenance",
461+
actual: artifacts.experimentIr.provenance,
462+
expected: { kind: "codex", ...input.provenance },
463+
},
464+
]);
465+
if (immutableLineageFindingPaths.length > 0) {
443466
throw new ScientificArtifactValidationError(
444467
"IMMUTABLE_LINEAGE_MISMATCH",
445468
"Codex scientific artifacts failed immutable lineage or selection policy.",
469+
{ findingPaths: immutableLineageFindingPaths },
446470
);
447471
}
448472

@@ -453,21 +477,33 @@ async function materializeStructuredScientificOutput(
453477
);
454478
const allowedCandidateIds = new Set(input.conceptPack.candidateExperimentIds);
455479
const allowedOperationIds = new Set(input.conceptPack.allowedOperations);
456-
if (
457-
artifacts.discriminationContract.candidateExperimentIds.some(
458-
(candidateId) => !candidateIds.has(candidateId),
459-
) ||
460-
artifacts.experimentIr.candidateExperiments.some(
461-
(candidate) =>
462-
!allowedCandidateIds.has(candidate.id) ||
463-
candidate.operationIds.some(
464-
(operationId) => !allowedOperationIds.has(operationId),
480+
const invalidCandidateFindingPaths = normalizeScientificFindingPaths([
481+
...artifacts.discriminationContract.candidateExperimentIds.flatMap(
482+
(candidateId, index) =>
483+
candidateIds.has(candidateId)
484+
? []
485+
: [`discriminationContract.candidateExperimentIds[${index}]`],
486+
),
487+
...artifacts.experimentIr.candidateExperiments.flatMap(
488+
(candidate, candidateIndex) => [
489+
...(allowedCandidateIds.has(candidate.id)
490+
? []
491+
: [`experimentIr.candidateExperiments[${candidateIndex}].id`]),
492+
...candidate.operationIds.flatMap((operationId, operationIndex) =>
493+
allowedOperationIds.has(operationId)
494+
? []
495+
: [
496+
`experimentIr.candidateExperiments[${candidateIndex}].operationIds[${operationIndex}]`,
497+
],
465498
),
466-
)
467-
) {
499+
],
500+
),
501+
]);
502+
if (invalidCandidateFindingPaths.length > 0) {
468503
throw new ScientificArtifactValidationError(
469504
"CANDIDATE_REFERENCE_INVALID",
470505
"The Discrimination Contract references an unresolved candidate experiment.",
506+
{ findingPaths: invalidCandidateFindingPaths },
471507
);
472508
}
473509

@@ -484,7 +520,10 @@ async function materializeStructuredScientificOutput(
484520
throw new ScientificArtifactValidationError(
485521
"LAB_SCENE_INVALID",
486522
"The generated lab scene did not match its fixed verified bindings.",
487-
{ cause: sceneResult.error },
523+
{
524+
cause: sceneResult.error,
525+
findingPaths: zodScientificFindingPaths(sceneResult.error, "labScene"),
526+
},
488527
);
489528
}
490529
const scene = sceneResult.data;
@@ -786,21 +825,6 @@ function scientificArtifactFailure(
786825
return undefined;
787826
}
788827

789-
function policyRepairPrompt(
790-
originalPrompt: string,
791-
attempt: 1 | 2,
792-
findingCodes: readonly ScientificArtifactFindingCode[],
793-
): string {
794-
return `${originalPrompt}
795-
796-
The previous candidate failed CounterLab's fixed scientific artifact validation before any file was materialized. This is bounded repair attempt ${attempt} of at most 2.
797-
Return a fresh complete structured candidate. Copy the immutable input lineage exactly, keep selection UNSELECTED, and use only the allowed candidate and operation IDs. Correct every sanitized rejection class below without weakening, omitting, or reinterpreting any fixed schema or policy. Use short plain-language comparisons only; do not use equations, assignment syntax, executable source, commands, SQL, URLs, or raw paths.
798-
799-
Fixed local validation classes:
800-
${JSON.stringify(findingCodes)}
801-
`;
802-
}
803-
804828
function parseInput<T>(schema: z.ZodType<T>, value: unknown): T {
805829
const result = schema.safeParse(value);
806830
if (!result.success) {
@@ -1333,17 +1357,28 @@ export class AppServerCodexCompiler implements CodexCompiler {
13331357
const findingCodes =
13341358
policyFailure === undefined
13351359
? [artifactFailure!.findingCode]
1336-
: ([
1360+
: [
13371361
...new Set(
13381362
policyFailure.findings.map((finding) => finding.code),
13391363
),
1340-
].sort() as ExperimentIRPolicyFindingCode[]);
1364+
].sort();
1365+
const findingPaths =
1366+
policyFailure === undefined
1367+
? artifactFailure!.findingPaths
1368+
: normalizeScientificFindingPaths(
1369+
policyFailure.findings.map((finding) => finding.path),
1370+
);
13411371
yield {
13421372
type: "policy_repair",
13431373
attempt,
13441374
findingCodes,
13451375
};
1346-
prompt = policyRepairPrompt(originalPrompt, attempt, findingCodes);
1376+
prompt = buildScientificPolicyRepairPrompt(
1377+
originalPrompt,
1378+
attempt,
1379+
findingCodes,
1380+
findingPaths,
1381+
);
13471382
phase = "repair";
13481383
}
13491384
}

‎packages/codex-client/src/scientific-method.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,6 +512,8 @@ describe("hosted scientific-method compiler", () => {
512512
"--expect-structured-turn",
513513
"--structured-scientific-output",
514514
"--structured-scientific-lineage-until-artifact-repair",
515+
"--expect-scientific-repair-path=experimentIr.sessionId",
516+
"--reject-scientific-repair-value=session_wrong_lineage",
515517
],
516518
timeoutMs: 2_000,
517519
...unisolatedTestProcess,
@@ -536,4 +538,51 @@ describe("hosted scientific-method compiler", () => {
536538
).sessionId,
537539
).toBe(scientificInput(work).sessionId);
538540
});
541+
542+
it("uses sanitized field paths to repair a fixed schema failure", async () => {
543+
const fakeServer = fileURLToPath(
544+
new URL("./test-fixtures/fake-app-server.mjs", import.meta.url),
545+
);
546+
const root = resolve(fileURLToPath(new URL("../../..", import.meta.url)));
547+
const work = join(
548+
root,
549+
`node_modules/.cache/counterlab-v6.1/test-work/codex-schema-repair-${process.pid}`,
550+
);
551+
await mkdir(work, { recursive: true });
552+
const compiler = new AppServerCodexCompiler({
553+
command: process.execPath,
554+
commandArgs: [
555+
fakeServer,
556+
"--expect-structured-turn",
557+
"--structured-scientific-output",
558+
"--structured-scientific-schema-until-artifact-repair",
559+
"--expect-scientific-repair-path=authoritativeArtifact.experimentIr.provenance.promptHash",
560+
"--reject-scientific-repair-value=private_invalid_hash",
561+
],
562+
timeoutMs: 2_000,
563+
...unisolatedTestProcess,
564+
});
565+
566+
const events = await collect(
567+
compiler.compileScientificMethod(scientificInput(work)),
568+
);
569+
570+
expect(events).toContainEqual({
571+
type: "policy_repair",
572+
attempt: 1,
573+
findingCodes: ["OUTPUT_SCHEMA_INVALID"],
574+
});
575+
expect(events.at(-1)).toMatchObject({
576+
type: "file_change",
577+
status: "completed",
578+
});
579+
expect(
580+
ExperimentIRV5Schema.parse(
581+
JSON.parse(await readFile(join(work, "experiment-ir.json"), "utf8")),
582+
).provenance,
583+
).toMatchObject({
584+
kind: "codex",
585+
promptHash: scientificInput(work).provenance.promptHash,
586+
});
587+
});
539588
});

0 commit comments

Comments
 (0)