From 87fb41261b44aff06bd18941ccdb273bd255b0ab Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 4 Oct 2026 14:19:41 -0700 Subject: [PATCH] fix(compaction): drop thinking blocks from the live tail Uncapped thinking on the newest pair was pinning post-fold occupancy near 45%. The live copy drops thinking and redacted_thinking; the archive keeps the full bodies. Thinking-only turns become the existing omitted marker so adapters do not 400 on empty content. --- docs/ARCHITECTURE.md | 4 +- docs/TELEMETRY.md | 5 +- src/context-compactor.test.ts | 132 ++++++++++++++++++++++++++++++++ src/session/compactor.ts | 74 ++++++++++++------ src/session/runtime-assembly.ts | 1 + src/telemetry/index.ts | 8 +- 6 files changed, 196 insertions(+), 28 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index dfa101708..74f55483e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -201,9 +201,9 @@ The agent maintains an optional **`manage_tasks`** list (create/update via the h #### Context compaction (the compaction governor) -When a cycle's input tokens cross a threshold, the director compacts the inference-facing history (the full run is always retained in the context store). The threshold is **model-aware** — roughly 60% of the active model's real context window — so small-window models compact early enough to avoid provider context-overflow while large-window models do not compact prematurely. The compacted prefix is **append-only across passes**: the existing compacted user turn stays byte-identical; new folds become later summary turns with a harness-inserted assistant spacer (identified by reserved `model: "harness"`, plus a visible sentinel; persisted `[compaction]` tokens without a producer still freeze) between them so the prompt head can remain in the provider KV cache. Model-emitted copies of the spacer are not frozen. Spacer-only model replies are incomplete: ChatDirector nudges, then falls through loop-protection, workflow-idle, and open-task rails rather than empty-settling with work still open. The governor covers five cases: +When a cycle's input tokens cross a threshold, the director compacts the inference-facing history (the full run is always retained in the context store). The threshold is **model-aware** — roughly 60% of the active model's real context window — so small-window models compact early enough to avoid provider context-overflow while large-window models do not compact prematurely. The compacted prefix is **append-only across passes**: the existing compacted user turn stays byte-identical; new folds become later summary turns with a harness-inserted assistant spacer (identified by reserved `model: "harness"`, plus a visible sentinel; persisted `[compaction]` tokens without a producer still freeze) between them so the prompt head can remain in the provider KV cache. Model-emitted copies of the spacer are not frozen. Spacer-only model replies are incomplete: ChatDirector nudges, then falls through loop-protection, workflow-idle, and open-task rails rather than empty-settling with work still open. The live tail drops `thinking` and `redacted_thinking` blocks so they are not billed against occupancy; full thinking stays in the archived transcript. The governor covers five cases: -- **Threshold at a tool pause** — Once over threshold, the follow-up `infer` after a tool batch is swapped for a `compact` cycle, and inference resumes via a host continuation message. After every compact (threshold, operator, overflow), the governor snapshots post-compact usage (`tokensAtLastCompact`) and keeps that growth latch. It does not re-arm the proactive path on growth alone; re-arm requires a wide resume gap (~20% of the window, a full warning band) past that snapshot. Usage back at or under the 60% threshold restores the consecutive-compact cap and the overflow-recovery budget; it does not drop the growth latch. A fold that landed well under (snapshot + 20% already past the 60% trigger) still re-arms at the next 60% crossing because the gap predicate is already satisfied. A fold that only just got under (typical live ~45% after a thin-tail fold) does not sawtooth on the next 60% crossing; it waits until post-compact + 20%. A consecutive-compact cap holds across tool-call occupancy as a second rail. +- **Threshold at a tool pause** — Once over threshold, the follow-up `infer` after a tool batch is swapped for a `compact` cycle, and inference resumes via a host continuation message. After every compact (threshold, operator, overflow), the governor snapshots post-compact usage (`tokensAtLastCompact`) and keeps that growth latch. It does not re-arm the proactive path on growth alone; re-arm requires a wide resume gap (~20% of the window, a full warning band) past that snapshot. Usage back at or under the 60% threshold restores the consecutive-compact cap and the overflow-recovery budget; it does not drop the growth latch. A fold that landed well under (snapshot + 20% already past the 60% trigger) still re-arms at the next 60% crossing because the gap predicate is already satisfied. A fold that only just got under (a marginal fold, not a deep one) does not sawtooth on the next 60% crossing; it waits until post-compact + 20%. A consecutive-compact cap holds across tool-call occupancy as a second rail. - **Idle (end-of-turn)** — An interactive turn can end with a reply and then sit idle with no tool batch to intercept; the governor requests a continuation at that pause and compacts when it arrives. An operator message that races the continuation still compacts first, then re-enters inference to answer it. - **Idle recompress past the provider TTL** — Cache expiry does not compact. Stored turns stay. When settings set `anthropicCachePrompt`, and the saved Anthropic-protocol cache write is at least 5 minutes old, the pre-inference context transform stubs old tool-result bodies on the outgoing prompt only. `turns.jsonl` is not rewritten. The setting defaults off. diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index f2276b5ae..5fe679e05 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -23,7 +23,7 @@ Each event carries a small set of properties: | `subagent_start` | A `spawn_agent` dispatch begins | `agent_name` | | `subagent_end` | A `spawn_agent` dispatch finishes | `agent_name`, `status`, `duration_ms`, `model`, `turn_count`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens`, `tool_call_count`, `tool_error_count`, `stop_reason`, `parent_trace_id` | | `permission_prompt` | An approval prompt is answered (or abandoned) | `decision`, `permission_kind` | -| `compaction` | The compactor actually folds turns away | `mode`, `duration_ms`, `turns_before`, `turns_after` | +| `compaction` | The compactor actually folds turns away | `mode`, `duration_ms`, `turns_before`, `turns_after`, `live_tokens` | | `summarizer_failure` | The compaction summary call fails after its retry budget is spent | `provider`, `model`, `error_kind`, `duration_ms` | | `crash` | A fatal error reaches the process-level handler | `kind`, `error_class` | | `auth_failure` | A provider rejects the stored credentials | `auth_provider` | @@ -34,7 +34,8 @@ Each event carries a small set of properties: `compaction` is deliberately silent on the runs where the compactor decides there is nothing to compact — an event that also fires on no-ops makes its own -duration and turn-count averages meaningless. +duration and turn-count averages meaningless. `live_tokens` is the post-fold +live-turn token estimate. `summarizer_failure` fires once per failed summary call, not per attempt. `error_kind` is a first-party enum (`auth`, `provider`, `timeout`, `aborted`, diff --git a/src/context-compactor.test.ts b/src/context-compactor.test.ts index e0790f944..a758b5a6b 100644 --- a/src/context-compactor.test.ts +++ b/src/context-compactor.test.ts @@ -14,6 +14,7 @@ import { import { HANDOFF_LATEST_KEY } from "./session/compaction-handoff.js"; import { compactionThresholdFor } from "./provider/context-window.js"; import { estimateContextTokens } from "./agent/context-estimate.js"; +import { THINKING_ONLY_OMITTED } from "./provider/replay-sanitizer.js"; import { createModelSummarizer } from "./session/summarizer.js"; import { createCompactionGovernor, @@ -1185,4 +1186,135 @@ describe("CL-9489 zero-verbatim fold", () => { compactionThresholdFor(undefined), ); }); + + test("a newest-pair thinking block is dropped from the live tail", async () => { + const thinking = "t".repeat(160_000); + const filler = "x".repeat(1_500); + const turns: ConversationTurn[] = []; + for (let i = 0; i < 6; i++) { + turns.push( + makeTurn({ + role: "user", + content: [{ type: "text", text: `dummy ask ${i} ${filler}` }], + }), + ); + turns.push( + makeTurn({ + role: "assistant", + content: [{ type: "text", text: `dummy reply ${i} ${filler}` }], + }), + ); + } + turns.push( + makeTurn({ + role: "assistant", + content: [ + { + type: "thinking", + thinking, + signature: "sig-thinking-1", + }, + { + type: "tool_call", + id: "c-newest", + name: "read_file", + arguments: { path: "src/app.ts" }, + }, + ], + }), + ); + turns.push( + makeTurn({ + role: "user", + content: [ + { + type: "tool_result", + callId: "c-newest", + content: [{ type: "text", text: "export const app = 1;\n" }], + }, + ], + }), + ); + + const result = await createPruningCompactor({ + summarize: async () => + "Folded dummy pairs. Next: resume the newest tool.", + }).apply(turns, mockStrategyCtx); + + const liveTypes = result.output.flatMap((t) => + t.content.map((b) => b.type), + ); + expect(liveTypes).not.toContain("thinking"); + expect(liveTypes).not.toContain("redacted_thinking"); + expect(estimateContextTokens(result.output)).toBeLessThan(12_800); + expect( + result.output.some((t) => + t.content.some((b) => b.type === "tool_call" && b.id === "c-newest"), + ), + ).toBe(true); + expect( + result.output.some((t) => + t.content.some( + (b) => b.type === "tool_result" && b.callId === "c-newest", + ), + ), + ).toBe(true); + expect(result.record.decisions.liveTokenEstimate).toBe( + estimateContextTokens(result.output), + ); + }); + + test("a thinking-only live turn is replaced with THINKING_ONLY_OMITTED", async () => { + const thinking = "t".repeat(20_000); + const filler = "x".repeat(1_500); + const turns: ConversationTurn[] = []; + for (let i = 0; i < 6; i++) { + turns.push( + makeTurn({ + role: "user", + content: [{ type: "text", text: `dummy ask ${i} ${filler}` }], + }), + ); + turns.push( + makeTurn({ + role: "assistant", + content: [{ type: "text", text: `dummy reply ${i} ${filler}` }], + }), + ); + } + turns.push( + makeTurn({ + role: "assistant", + content: [ + { + type: "thinking", + thinking, + signature: "sig-thinking-only", + }, + ], + }), + ); + + const result = await createPruningCompactor({ + summarize: async () => + "Folded dummy pairs. Next: resume after thinking-only.", + }).apply(turns, mockStrategyCtx); + + const liveTypes = result.output.flatMap((t) => + t.content.map((b) => b.type), + ); + expect(liveTypes).not.toContain("thinking"); + expect(liveTypes).not.toContain("redacted_thinking"); + expect(result.output.every((t) => t.content.length > 0)).toBe(true); + expect( + result.output.some( + (t) => + t.role === "assistant" && + t.content.some( + (b) => b.type === "text" && b.text === THINKING_ONLY_OMITTED, + ), + ), + ).toBe(true); + expect(estimateContextTokens(result.output)).toBeLessThan(12_800); + }); }); diff --git a/src/session/compactor.ts b/src/session/compactor.ts index c235be4ce..ecfea59b1 100644 --- a/src/session/compactor.ts +++ b/src/session/compactor.ts @@ -30,7 +30,11 @@ import { PATH_KEYED_READ_TOOLS, SEARCH_QUERY_TOOLS, } from "../agent/tool-classification.js"; -import { estimateContentBlockTokens } from "../agent/context-estimate.js"; +import { + estimateContentBlockTokens, + estimateContextTokens, +} from "../agent/context-estimate.js"; +import { THINKING_ONLY_OMITTED } from "../provider/replay-sanitizer.js"; // --------------------------------------------------------------------------- // Compactor @@ -710,31 +714,49 @@ function excerptTailText( }; } -// Excerpted live copy of a tail turn: large tool_result text parts shrink to -// head+tail excerpts, everything else (user text, attachments, tool calls, -// error results stay whole — errors are resume state, not bulk) passes -// through untouched. +// Excerpted live copy of a tail turn: thinking/redacted_thinking are dropped +// from the live copy (full bodies stay in the archive) because uncapped +// thinking on the newest pair was pinning occupancy near 45%. Drop whole +// blocks so signatures are not sent with truncated text. Large tool_result +// text parts shrink to head+tail excerpts; everything else (user text, +// attachments, tool calls, error results stay whole — errors are resume +// state, not bulk) passes through untouched. function excerptTailTurn( turn: ConversationTurn, shape: CompactionShape, -): { turn: ConversationTurn; shortenedOutputs: number } { +): { turn: ConversationTurn; shortenedOutputs: number; changed: boolean } { let shortenedOutputs = 0; let changed = false; - const content = turn.content.map( - (block): ConversationTurn["content"][number] => { - if (block.type !== "tool_result" || block.isError === true) return block; - const parts = block.content.map((c) => { - if (c.type !== "text") return c; - const excerpted = excerptTailText(c.text, shape); - if (!excerpted.shortened) return c; - shortenedOutputs += 1; - changed = true; - return { ...c, text: excerpted.text }; - }); - return changed ? { ...block, content: parts } : block; - }, - ); - return { turn: changed ? { ...turn, content } : turn, shortenedOutputs }; + const content = turn.content.flatMap((block): ConversationTurn["content"] => { + if (block.type === "thinking" || block.type === "redacted_thinking") { + changed = true; + return []; + } + if (block.type !== "tool_result" || block.isError === true) return [block]; + let blockChanged = false; + const parts = block.content.map((c) => { + if (c.type !== "text") return c; + const excerpted = excerptTailText(c.text, shape); + if (!excerpted.shortened) return c; + shortenedOutputs += 1; + blockChanged = true; + changed = true; + return { ...c, text: excerpted.text }; + }); + return [blockChanged ? { ...block, content: parts } : block]; + }); + // Dropping thinking/redacted_thinking can leave a thinking-only assistant + // with empty content. Adapters 400 on that; keep the turn (role + // alternation) and reuse the replay-sanitizer marker. + const liveContent = + content.length === 0 + ? [{ type: "text" as const, text: THINKING_ONLY_OMITTED }] + : content; + return { + turn: changed ? { ...turn, content: liveContent } : turn, + shortenedOutputs, + changed, + }; } interface TailSelection { @@ -804,8 +826,12 @@ function selectTail( const turnCost = (idx: number): { tokens: number; shortened: number } => { const turn = turns[idx]; if (turn === undefined) return { tokens: 0, shortened: 0 }; - const { turn: live, shortenedOutputs } = excerptTailTurn(turn, shape); - if (shortenedOutputs > 0) excerpted.set(idx, live); + const { + turn: live, + shortenedOutputs, + changed, + } = excerptTailTurn(turn, shape); + if (changed) excerpted.set(idx, live); return { tokens: excerptedTurnTokens(turn, shape), shortened: shortenedOutputs, @@ -1002,6 +1028,7 @@ export function createPruningCompactor( shortenedToolOutputs: tail.shortenedToolOutputs, agedImageCount: aged.agedImageCount, supersededReadCount: supersededReads.size, + liveTokenEstimate: estimateContextTokens(output), }, }, ...(aged.blobs.length > 0 ? { blobs: aged.blobs } : {}), @@ -1194,6 +1221,7 @@ export function createPruningCompactor( agedImageCount: aged.agedImageCount, supersededReadCount: supersededReads.size, repeatedErrorCount: repeatedErrors.size, + liveTokenEstimate: estimateContextTokens(liveOutput), ...(summarizeFallback !== undefined ? { summarizeFailed: 1, diff --git a/src/session/runtime-assembly.ts b/src/session/runtime-assembly.ts index 01a0b6d32..d6c9e3805 100644 --- a/src/session/runtime-assembly.ts +++ b/src/session/runtime-assembly.ts @@ -579,6 +579,7 @@ export function createSessionPruningCompactor( duration_ms: Date.now() - startedAt, turns_before: turnsBefore, turns_after: result.output.length, + live_tokens: result.record.decisions.liveTokenEstimate, }); return result; }, diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 78cb17e26..efc722f59 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -194,7 +194,13 @@ const EVENT_PROPERTY_ALLOWLIST: Record = { ], permission_prompt: ["decision", "permission_kind"], - compaction: ["mode", "duration_ms", "turns_before", "turns_after"], + compaction: [ + "mode", + "duration_ms", + "turns_before", + "turns_after", + "live_tokens", + ], // provider/model are the canonical runtime ids (same trust class as // $ai_provider/$ai_model); error_kind is the summarizer's first-party // failure enum, never the provider's error text.