Skip to content

Commit 9758a42

Browse files
feat(agent): add astra-only tool-evasion residual (#1195)
1 parent 1ae1a3e commit 9758a42

5 files changed

Lines changed: 313 additions & 11 deletions

File tree

‎packages/prompt-variance/src/index.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,16 @@ export {
1010
type PromptVarianceFamily,
1111
type PromptVarianceRow,
1212
} from "./rows.js";
13+
14+
// Astra tool-evasion residual (CL-9027): forensics on the repro trace showed
15+
// evasion, not waste — the gpt-6-astra leaf re-issues the same effective tool
16+
// call with trivial argument deltas (path prefix, default offset/limit,
17+
// padding whitespace), so no exact tool fingerprint repeats 3x and the shared
18+
// threshold guard stays silent. This forbids that specific variation
19+
// (muse-rule precedent, CL-7869); no signature normalization ships in
20+
// first-party code. Named export rather than a family row: it travels the
21+
// ModelFamilyPolicy.promptResidual seam, composed over the gpt narrate note.
22+
export const astraResidual: string =
23+
"Tool discipline (gpt-6-astra worker):\n" +
24+
"- Never re-issue a tool call that repeats a prior call with only trivial argument changes (path prefix, offset, limit, whitespace).\n" +
25+
"- Never re-read a file you have already read this session.";

‎src/agent/model-family-policy.test.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,4 +192,200 @@ describe("resolveModelFamilyPolicy", () => {
192192
expect(gpt.toolDisciplineRules).toBeUndefined();
193193
expect(gpt.advertisedToolDeny).toEqual([]);
194194
});
195+
196+
describe("astra repro trace (CL-9027)", () => {
197+
// Minimal failing session-trace fixture: 8 consecutive tool-only turns
198+
// from a gpt-6-astra leaf (tool names + args + result sizes per turn).
199+
// Fingerprint and repeat-count semantics mirror
200+
// scripts/tool-fingerprint-forensics.ts (stableJson exact signatures,
201+
// largest exact-repeat count per period 1-6): the shared threshold guard
202+
// fires only on exact repeats, so a loop that varies trivial argument
203+
// details escapes it. That is evasion, not threshold-tolerated waste —
204+
// and the Step-3 residual forbids exactly this variation. The
205+
// near-identical grouping below is test-only forensics; no signature
206+
// normalization ships in first-party code.
207+
interface ReproTurn {
208+
tool: string;
209+
args: Record<string, unknown>;
210+
resultTokens: number;
211+
}
212+
const ASTRA_REPRO_TRACE: readonly ReproTurn[] = [
213+
{
214+
tool: "read",
215+
args: { path: "src/agent/prompts.ts" },
216+
resultTokens: 3200,
217+
},
218+
{
219+
tool: "read",
220+
args: { path: "./src/agent/prompts.ts" },
221+
resultTokens: 3200,
222+
},
223+
{
224+
tool: "read",
225+
args: { path: "src/agent/prompts.ts", offset: 1 },
226+
resultTokens: 3180,
227+
},
228+
{
229+
tool: "grep",
230+
args: { pattern: "narrate", path: "src/agent" },
231+
resultTokens: 420,
232+
},
233+
{
234+
tool: "read",
235+
args: { path: "src/agent/prompts.ts" },
236+
resultTokens: 3200,
237+
},
238+
{
239+
tool: "grep",
240+
args: { pattern: "narrate ", path: "src/agent" },
241+
resultTokens: 420,
242+
},
243+
{
244+
tool: "read",
245+
args: { path: "./src/agent/prompts.ts", limit: 2000 },
246+
resultTokens: 3200,
247+
},
248+
{
249+
tool: "read",
250+
args: { path: "src/agent/prompts.ts", offset: 0 },
251+
resultTokens: 3200,
252+
},
253+
];
254+
255+
function stableJson(value: unknown): string {
256+
if (value === null || typeof value !== "object")
257+
return JSON.stringify(value);
258+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
259+
const obj = value as Record<string, unknown>;
260+
const keys = Object.keys(obj).sort();
261+
return `{${keys.map((k) => `${JSON.stringify(k)}:${stableJson(obj[k])}`).join(",")}}`;
262+
}
263+
264+
function fingerprint(turn: ReproTurn): string {
265+
return `${turn.tool}:${stableJson(turn.args)}`;
266+
}
267+
268+
function maxExactRepeats(fps: readonly string[]): number {
269+
let best = 1;
270+
for (let period = 1; period <= 6; period++) {
271+
for (let end = 1; end <= fps.length; end++) {
272+
const prefix = fps.slice(0, end);
273+
let i = prefix.length - 1;
274+
let j = i - period;
275+
let matched = 0;
276+
while (j >= 0 && prefix[i] === prefix[j]) {
277+
matched++;
278+
i--;
279+
j--;
280+
}
281+
const reps = Math.floor((matched + period) / period);
282+
if (reps > best) best = reps;
283+
}
284+
}
285+
return best;
286+
}
287+
288+
function evasionKey(turn: ReproTurn): string {
289+
const args = { ...turn.args };
290+
// The trivial deltas this trace varies: a leading ./ prefix, default
291+
// offset/limit values, and padding whitespace.
292+
if (typeof args.path === "string")
293+
args.path = args.path.replace(/^\.\//, "");
294+
if (args.offset === 0 || args.offset === 1) delete args.offset;
295+
if (typeof args.limit === "number") delete args.limit;
296+
if (typeof args.pattern === "string") args.pattern = args.pattern.trim();
297+
return `${turn.tool}:${stableJson(args)}`;
298+
}
299+
300+
function classifyAstraTrace(trace: readonly ReproTurn[]): string {
301+
if (maxExactRepeats(trace.map(fingerprint)) >= 3) return "waste";
302+
const groups = new Map<string, number>();
303+
for (const turn of trace) {
304+
const key = evasionKey(turn);
305+
groups.set(key, (groups.get(key) ?? 0) + 1);
306+
}
307+
return Math.max(...groups.values()) >= 3 ? "evasion" : "waste";
308+
}
309+
310+
test("classifies as evasion: no exact repeat trips the shared guard", () => {
311+
expect(ASTRA_REPRO_TRACE).toHaveLength(8);
312+
expect(maxExactRepeats(ASTRA_REPRO_TRACE.map(fingerprint))).toBeLessThan(
313+
3,
314+
);
315+
expect(classifyAstraTrace(ASTRA_REPRO_TRACE)).toBe("evasion");
316+
});
317+
318+
test("pins the offending pattern and the wasted turn/token delta", () => {
319+
const groups = new Map<string, number>();
320+
for (const turn of ASTRA_REPRO_TRACE) {
321+
const key = evasionKey(turn);
322+
groups.set(key, (groups.get(key) ?? 0) + 1);
323+
}
324+
// Six re-reads of one file plus two re-greps of one pattern, each run
325+
// differing only by a trivial argument delta.
326+
expect(groups.get('read:{"path":"src/agent/prompts.ts"}')).toBe(6);
327+
expect(groups.get('grep:{"path":"src/agent","pattern":"narrate"}')).toBe(
328+
2,
329+
);
330+
const wastedTokens = ASTRA_REPRO_TRACE.reduce(
331+
(sum, turn) => sum + turn.resultTokens,
332+
0,
333+
);
334+
expect(wastedTokens).toBe(20020);
335+
});
336+
});
337+
338+
test("astra resolves to astra with the composed residual; thresholds stay default (CL-9027)", () => {
339+
const base = resolveModelFamilyPolicy({
340+
providerName: "unknown-provider",
341+
model: "unknown-model",
342+
});
343+
const gpt = resolveModelFamilyPolicy({
344+
providerName: "openai",
345+
model: "gpt-5.6",
346+
});
347+
for (const orchestrator of [false, true]) {
348+
const astra = resolveModelFamilyPolicy({
349+
providerName: "codex/default",
350+
model: "gpt-6-astra",
351+
orchestrator,
352+
});
353+
expect(astra.family).toBe("astra");
354+
// Keeps the gpt narrate nudge and adds the evasion-specific rules.
355+
expect(astra.promptResidual).toContain(
356+
"Narrate before tools (GPT worker):",
357+
);
358+
expect(astra.promptResidual).toContain("trivial argument changes");
359+
expect(astra.promptResidual).not.toBe(gpt.promptResidual);
360+
// Evasion earns a forbidding residual, not tighter thresholds.
361+
expect(astra.toolOnlyTurnNudgeAt).toBe(base.toolOnlyTurnNudgeAt);
362+
expect(astra.subAgentStallTimeoutMs).toBe(base.subAgentStallTimeoutMs);
363+
expect(astra.applyGrokFinishBias).toBe(false);
364+
expect(astra.toolDisciplineRules).toBeUndefined();
365+
expect(astra.advertisedToolDeny).toEqual([]);
366+
}
367+
});
368+
369+
test("sol and generic gpt stay gpt with the byte-identical narrate residual", () => {
370+
const generic = resolveModelFamilyPolicy({
371+
providerName: "openai",
372+
model: "gpt-5.6",
373+
});
374+
for (const model of [
375+
"gpt-5.6-sol",
376+
"gpt-5.5",
377+
"gpt-5.6-luna",
378+
"gpt-5.6-terra",
379+
] as const) {
380+
for (const orchestrator of [false, true]) {
381+
const policy = resolveModelFamilyPolicy({
382+
providerName: "codex/default",
383+
model,
384+
orchestrator,
385+
});
386+
expect(policy.family).toBe("gpt");
387+
expect(policy.promptResidual).toBe(generic.promptResidual);
388+
}
389+
}
390+
});
195391
});

‎src/agent/model-family-policy.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
type ModelFamily,
44
} from "../subagent/provider-family.js";
55
import {
6+
astraResidual,
67
claudeRow,
78
gptRow,
89
grokRow,
@@ -164,9 +165,12 @@ const CLAUDE_POLICY: Omit<ModelFamilyPolicy, "family"> = {
164165
// Deliberately not manage_tasks ceremony — that is CL-7769, not this text.
165166
// The text lives here (policy owns data); buildGptNarrateBeforeToolsNote
166167
// (prompts.ts) returns it verbatim so the prompt carries exactly one copy.
167-
// Served cells (astra/sol/terra/…) are never named here — CL-8265
168-
// characterizes them later. Single-sourced from the versioned
169-
// prompt-variance package (CL-8269); the name stays for existing importers.
168+
// Served cells (sol/terra/…) are never named here — CL-8265 characterizes
169+
// them later. Astra is the one exception: forensics (CL-9027) showed the
170+
// gpt-6-astra cell evading the shared threshold guard via trivial argument
171+
// deltas, so it carries its own residual below. Single-sourced from the
172+
// versioned prompt-variance package (CL-8269); the name stays for existing
173+
// importers.
170174
export const GPT_NARRATE_BEFORE_TOOLS_NOTE = gptRow.residual;
171175

172176
// GPT (Codex / gpt-*) thresholds are provisional: we have no eval
@@ -181,6 +185,21 @@ const GPT_POLICY: Omit<ModelFamilyPolicy, "family"> = {
181185
promptResidual: GPT_NARRATE_BEFORE_TOOLS_NOTE,
182186
};
183187

188+
// Astra (served gpt-6-astra cell) is GPT plus the evasion residual. Forensics
189+
// on the repro trace (see model-family-policy.test.ts) classified the loop as
190+
// evasion — near-identical re-issued calls whose trivial argument deltas keep
191+
// every exact fingerprint under the shared threshold — so the residual forbids
192+
// that specific variation (muse-rule precedent, CL-7869) instead of tightening
193+
// thresholds: toolOnlyTurnNudgeAt stays at the permissive default. No
194+
// signature normalization ships in first-party code.
195+
export const ASTRA_PROMPT_RESIDUAL = `${GPT_NARRATE_BEFORE_TOOLS_NOTE}\n${astraResidual}`;
196+
197+
const ASTRA_POLICY: Omit<ModelFamilyPolicy, "family"> = {
198+
...GPT_POLICY,
199+
// Like gpt, primary and leaf alike: no orchestrator carve-out.
200+
promptResidual: ASTRA_PROMPT_RESIDUAL,
201+
};
202+
184203
export function resolveModelFamilyPolicy(input: {
185204
providerName: string;
186205
model?: string;
@@ -217,6 +236,10 @@ export function resolveModelFamilyPolicy(input: {
217236
case "gpt":
218237
// Primary and leaf alike: no orchestrator carve-out.
219238
return { family, ...GPT_POLICY };
239+
case "astra":
240+
// Primary and leaf alike, like gpt: no orchestrator carve-out. Generic
241+
// gpt prompts are byte-identical — only astra carries the residual.
242+
return { family, ...ASTRA_POLICY };
220243
default:
221244
return { family: "default", ...DEFAULT_POLICY };
222245
}

‎src/subagent/provider-family.test.ts‎

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { describe, expect, test } from "bun:test";
22
import {
33
detectModelFamily,
4+
isAstraLeafProvider,
45
isClaudeLeafProvider,
56
isGptProvider,
67
isKimiLeafProvider,
@@ -126,16 +127,69 @@ describe("isGptProvider (CL-8310)", () => {
126127
expect(isGptProvider(row)).toBe(expected);
127128
});
128129

129-
test("covers every in-tree codex catalog model without naming cells", () => {
130-
// No terra/sol/astra special-casing: every served codex id resolves via
131-
// the generic codex-provider / gpt-* match, so future cells ride along.
130+
test("astra branches to its own family; other cells ride the generic gpt match", () => {
131+
// CL-9027 reverses the no-special-casing rule for astra only: the
132+
// gpt-6-astra cell doom-loops, so it resolves to the astra family while
133+
// sol/terra/luna and generic gpt ids keep the generic gpt match.
132134
for (const model of CODEX_DEFAULT_MODELS) {
133135
expect(isGptProvider({ providerName: "codex/default", model })).toBe(
134136
true,
135137
);
136-
expect(detectModelFamily({ providerName: "codex/default", model })).toBe(
137-
"gpt",
138-
);
138+
const family = detectModelFamily({
139+
providerName: "codex/default",
140+
model,
141+
});
142+
if (model.toLowerCase().startsWith("gpt-6-astra")) {
143+
expect(family).toBe("astra");
144+
} else {
145+
expect(family).toBe("gpt");
146+
}
147+
}
148+
});
149+
});
150+
151+
describe("isAstraLeafProvider (CL-9027)", () => {
152+
test("matches the served gpt-6-astra cell id on any provider", () => {
153+
expect(
154+
isAstraLeafProvider({
155+
providerName: "codex/default",
156+
model: "gpt-6-astra",
157+
}),
158+
).toBe(true);
159+
expect(
160+
isAstraLeafProvider({
161+
providerName: "openai-compat",
162+
model: "GPT-6-ASTRA",
163+
}),
164+
).toBe(true);
165+
expect(
166+
detectModelFamily({
167+
providerName: "codex/default",
168+
model: "gpt-6-astra",
169+
}),
170+
).toBe("astra");
171+
});
172+
173+
test("rejects sol, generic gpt, and other families", () => {
174+
for (const input of [
175+
{ providerName: "codex/default", model: "gpt-5.6-sol" },
176+
{ providerName: "codex/default", model: "gpt-5.6-terra" },
177+
{ providerName: "codex/default", model: "gpt-5.6-luna" },
178+
{ providerName: "openai", model: "gpt-5.6" },
179+
{ providerName: "codex", model: "gpt-5.1" },
180+
{ providerName: "xai/default", model: "grok-4.6" },
181+
{ providerName: "anthropic", model: "claude-sonnet-4" },
182+
] as const) {
183+
expect(isAstraLeafProvider(input)).toBe(false);
139184
}
185+
expect(
186+
detectModelFamily({
187+
providerName: "codex/default",
188+
model: "gpt-5.6-sol",
189+
}),
190+
).toBe("gpt");
191+
expect(
192+
detectModelFamily({ providerName: "openai", model: "gpt-5.6" }),
193+
).toBe("gpt");
140194
});
141195
});

0 commit comments

Comments
 (0)