@@ -192,4 +192,200 @@ describe("resolveModelFamilyPolicy", () => {
192192 expect ( gpt . toolDisciplineRules ) . toBeUndefined ( ) ;
193193 expect ( gpt . advertisedToolDeny ) . toEqual ( [ ] ) ;
194194 } ) ;
195+
196+ describe ( "astra repro trace (CL-9027)" , ( ) => {
197+ // Minimal failing session-trace fixture: 8 consecutive tool-only turns
198+ // from a gpt-6-astra leaf (tool names + args + result sizes per turn).
199+ // Fingerprint and repeat-count semantics mirror
200+ // scripts/tool-fingerprint-forensics.ts (stableJson exact signatures,
201+ // largest exact-repeat count per period 1-6): the shared threshold guard
202+ // fires only on exact repeats, so a loop that varies trivial argument
203+ // details escapes it. That is evasion, not threshold-tolerated waste —
204+ // and the Step-3 residual forbids exactly this variation. The
205+ // near-identical grouping below is test-only forensics; no signature
206+ // normalization ships in first-party code.
207+ interface ReproTurn {
208+ tool : string ;
209+ args : Record < string , unknown > ;
210+ resultTokens : number ;
211+ }
212+ const ASTRA_REPRO_TRACE : readonly ReproTurn [ ] = [
213+ {
214+ tool : "read" ,
215+ args : { path : "src/agent/prompts.ts" } ,
216+ resultTokens : 3200 ,
217+ } ,
218+ {
219+ tool : "read" ,
220+ args : { path : "./src/agent/prompts.ts" } ,
221+ resultTokens : 3200 ,
222+ } ,
223+ {
224+ tool : "read" ,
225+ args : { path : "src/agent/prompts.ts" , offset : 1 } ,
226+ resultTokens : 3180 ,
227+ } ,
228+ {
229+ tool : "grep" ,
230+ args : { pattern : "narrate" , path : "src/agent" } ,
231+ resultTokens : 420 ,
232+ } ,
233+ {
234+ tool : "read" ,
235+ args : { path : "src/agent/prompts.ts" } ,
236+ resultTokens : 3200 ,
237+ } ,
238+ {
239+ tool : "grep" ,
240+ args : { pattern : "narrate " , path : "src/agent" } ,
241+ resultTokens : 420 ,
242+ } ,
243+ {
244+ tool : "read" ,
245+ args : { path : "./src/agent/prompts.ts" , limit : 2000 } ,
246+ resultTokens : 3200 ,
247+ } ,
248+ {
249+ tool : "read" ,
250+ args : { path : "src/agent/prompts.ts" , offset : 0 } ,
251+ resultTokens : 3200 ,
252+ } ,
253+ ] ;
254+
255+ function stableJson ( value : unknown ) : string {
256+ if ( value === null || typeof value !== "object" )
257+ return JSON . stringify ( value ) ;
258+ if ( Array . isArray ( value ) ) return `[${ value . map ( stableJson ) . join ( "," ) } ]` ;
259+ const obj = value as Record < string , unknown > ;
260+ const keys = Object . keys ( obj ) . sort ( ) ;
261+ return `{${ keys . map ( ( k ) => `${ JSON . stringify ( k ) } :${ stableJson ( obj [ k ] ) } ` ) . join ( "," ) } }` ;
262+ }
263+
264+ function fingerprint ( turn : ReproTurn ) : string {
265+ return `${ turn . tool } :${ stableJson ( turn . args ) } ` ;
266+ }
267+
268+ function maxExactRepeats ( fps : readonly string [ ] ) : number {
269+ let best = 1 ;
270+ for ( let period = 1 ; period <= 6 ; period ++ ) {
271+ for ( let end = 1 ; end <= fps . length ; end ++ ) {
272+ const prefix = fps . slice ( 0 , end ) ;
273+ let i = prefix . length - 1 ;
274+ let j = i - period ;
275+ let matched = 0 ;
276+ while ( j >= 0 && prefix [ i ] === prefix [ j ] ) {
277+ matched ++ ;
278+ i -- ;
279+ j -- ;
280+ }
281+ const reps = Math . floor ( ( matched + period ) / period ) ;
282+ if ( reps > best ) best = reps ;
283+ }
284+ }
285+ return best ;
286+ }
287+
288+ function evasionKey ( turn : ReproTurn ) : string {
289+ const args = { ...turn . args } ;
290+ // The trivial deltas this trace varies: a leading ./ prefix, default
291+ // offset/limit values, and padding whitespace.
292+ if ( typeof args . path === "string" )
293+ args . path = args . path . replace ( / ^ \. \/ / , "" ) ;
294+ if ( args . offset === 0 || args . offset === 1 ) delete args . offset ;
295+ if ( typeof args . limit === "number" ) delete args . limit ;
296+ if ( typeof args . pattern === "string" ) args . pattern = args . pattern . trim ( ) ;
297+ return `${ turn . tool } :${ stableJson ( args ) } ` ;
298+ }
299+
300+ function classifyAstraTrace ( trace : readonly ReproTurn [ ] ) : string {
301+ if ( maxExactRepeats ( trace . map ( fingerprint ) ) >= 3 ) return "waste" ;
302+ const groups = new Map < string , number > ( ) ;
303+ for ( const turn of trace ) {
304+ const key = evasionKey ( turn ) ;
305+ groups . set ( key , ( groups . get ( key ) ?? 0 ) + 1 ) ;
306+ }
307+ return Math . max ( ...groups . values ( ) ) >= 3 ? "evasion" : "waste" ;
308+ }
309+
310+ test ( "classifies as evasion: no exact repeat trips the shared guard" , ( ) => {
311+ expect ( ASTRA_REPRO_TRACE ) . toHaveLength ( 8 ) ;
312+ expect ( maxExactRepeats ( ASTRA_REPRO_TRACE . map ( fingerprint ) ) ) . toBeLessThan (
313+ 3 ,
314+ ) ;
315+ expect ( classifyAstraTrace ( ASTRA_REPRO_TRACE ) ) . toBe ( "evasion" ) ;
316+ } ) ;
317+
318+ test ( "pins the offending pattern and the wasted turn/token delta" , ( ) => {
319+ const groups = new Map < string , number > ( ) ;
320+ for ( const turn of ASTRA_REPRO_TRACE ) {
321+ const key = evasionKey ( turn ) ;
322+ groups . set ( key , ( groups . get ( key ) ?? 0 ) + 1 ) ;
323+ }
324+ // Six re-reads of one file plus two re-greps of one pattern, each run
325+ // differing only by a trivial argument delta.
326+ expect ( groups . get ( 'read:{"path":"src/agent/prompts.ts"}' ) ) . toBe ( 6 ) ;
327+ expect ( groups . get ( 'grep:{"path":"src/agent","pattern":"narrate"}' ) ) . toBe (
328+ 2 ,
329+ ) ;
330+ const wastedTokens = ASTRA_REPRO_TRACE . reduce (
331+ ( sum , turn ) => sum + turn . resultTokens ,
332+ 0 ,
333+ ) ;
334+ expect ( wastedTokens ) . toBe ( 20020 ) ;
335+ } ) ;
336+ } ) ;
337+
338+ test ( "astra resolves to astra with the composed residual; thresholds stay default (CL-9027)" , ( ) => {
339+ const base = resolveModelFamilyPolicy ( {
340+ providerName : "unknown-provider" ,
341+ model : "unknown-model" ,
342+ } ) ;
343+ const gpt = resolveModelFamilyPolicy ( {
344+ providerName : "openai" ,
345+ model : "gpt-5.6" ,
346+ } ) ;
347+ for ( const orchestrator of [ false , true ] ) {
348+ const astra = resolveModelFamilyPolicy ( {
349+ providerName : "codex/default" ,
350+ model : "gpt-6-astra" ,
351+ orchestrator,
352+ } ) ;
353+ expect ( astra . family ) . toBe ( "astra" ) ;
354+ // Keeps the gpt narrate nudge and adds the evasion-specific rules.
355+ expect ( astra . promptResidual ) . toContain (
356+ "Narrate before tools (GPT worker):" ,
357+ ) ;
358+ expect ( astra . promptResidual ) . toContain ( "trivial argument changes" ) ;
359+ expect ( astra . promptResidual ) . not . toBe ( gpt . promptResidual ) ;
360+ // Evasion earns a forbidding residual, not tighter thresholds.
361+ expect ( astra . toolOnlyTurnNudgeAt ) . toBe ( base . toolOnlyTurnNudgeAt ) ;
362+ expect ( astra . subAgentStallTimeoutMs ) . toBe ( base . subAgentStallTimeoutMs ) ;
363+ expect ( astra . applyGrokFinishBias ) . toBe ( false ) ;
364+ expect ( astra . toolDisciplineRules ) . toBeUndefined ( ) ;
365+ expect ( astra . advertisedToolDeny ) . toEqual ( [ ] ) ;
366+ }
367+ } ) ;
368+
369+ test ( "sol and generic gpt stay gpt with the byte-identical narrate residual" , ( ) => {
370+ const generic = resolveModelFamilyPolicy ( {
371+ providerName : "openai" ,
372+ model : "gpt-5.6" ,
373+ } ) ;
374+ for ( const model of [
375+ "gpt-5.6-sol" ,
376+ "gpt-5.5" ,
377+ "gpt-5.6-luna" ,
378+ "gpt-5.6-terra" ,
379+ ] as const ) {
380+ for ( const orchestrator of [ false , true ] ) {
381+ const policy = resolveModelFamilyPolicy ( {
382+ providerName : "codex/default" ,
383+ model,
384+ orchestrator,
385+ } ) ;
386+ expect ( policy . family ) . toBe ( "gpt" ) ;
387+ expect ( policy . promptResidual ) . toBe ( generic . promptResidual ) ;
388+ }
389+ }
390+ } ) ;
195391} ) ;
0 commit comments