diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 97bacb163..139c12e9a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -187,12 +187,27 @@ The agent maintains an optional **`manage_tasks`** list (create/update via the h #### Context compaction (the compaction governor) -When a cycle's input tokens cross a threshold, the director compacts the inference-facing history (the full run is always retained in the context store). The threshold is **model-aware** — roughly 60% of the active model's real context window — so small-window models compact early enough to avoid provider context-overflow while large-window models do not compact prematurely. The compacted prefix is **append-only across passes**: the existing compacted user turn stays byte-identical; new folds become later summary turns with a harness-inserted assistant spacer (identified by reserved `model: "harness"`, plus a visible sentinel; persisted `[compaction]` tokens without a producer still freeze) between them so the prompt head can remain in the provider KV cache. Model-emitted copies of the spacer are not frozen. Spacer-only model replies are incomplete: ChatDirector nudges, then falls through loop-protection, workflow-idle, and open-task rails rather than empty-settling with work still open. The governor covers three cases: +When a cycle's input tokens cross a threshold, the director compacts the inference-facing history (the full run is always retained in the context store). The threshold is **model-aware** — roughly 60% of the active model's real context window — so small-window models compact early enough to avoid provider context-overflow while large-window models do not compact prematurely. The compacted prefix is **append-only across passes**: the existing compacted user turn stays byte-identical; new folds become later summary turns with a harness-inserted assistant spacer (identified by reserved `model: "harness"`, plus a visible sentinel; persisted `[compaction]` tokens without a producer still freeze) between them so the prompt head can remain in the provider KV cache. Model-emitted copies of the spacer are not frozen. Spacer-only model replies are incomplete: ChatDirector nudges, then falls through loop-protection, workflow-idle, and open-task rails rather than empty-settling with work still open. The governor covers four cases: - **Threshold at a tool pause** — Once over threshold, the follow-up `infer` after a tool batch is swapped for a `compact` cycle, and inference resumes via a host continuation message. After a compact that remains over the high watermark, the governor uses **growth hysteresis** (wait for usage to grow by ~10% of the window) instead of re-arming on every cycle; dropping under 60% is not required. - **Idle (end-of-turn)** — An interactive turn can end with a reply and then sit idle with no tool batch to intercept; the governor requests a continuation at that pause and compacts when it arrives. An operator message that races the continuation still compacts first, then re-enters inference to answer it. +- **Idle recompress past the provider TTL** — The fold is a re-compress, not a cache play: provider KV caches expire on their own schedule, so a session compresses _after_ that expiry (the next turn is a cheaper write and later reads compound on the shrunk context) instead of only at 60% tokens. It is not limited to under-threshold sessions: once growth hysteresis has cleared `pending`, an over-threshold session in that gap can fire the same fold. Any live re-entry past the window fires it — an empty stall ping re-enters meter-only, a raced operator message re-infers — under reason `cache-ttl-recompress` through the same compactor, so the fresh tail stays raw exactly as in the threshold path. The interval follows provider cache economics (`src/provider/cache-ttl.ts`), not a global N minutes: + +| Provider segment | Idle recompress allowed after | Why | +| ----------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------ | +| `anthropic`, `zen-messages`, `opencode-go-messages` | 5 min | Default ephemeral cache TTL; hour-long breakpoints are never set | +| `openai-responses`, `codex-responses`, `openai-compatible`, `xai` | 10 min | In-memory prefixes typically evicted after 5–10 min idle; xAI assumed same economics | +| `gemini` | 15 min | No published implicit-cache eviction window; conservative | +| `deepseek` | 60 min | On-disk context cache persists for hours-to-days | +| `ollama` | never | Local inference has no remote cache to expire | +| anything else | 10 min | Assumed OpenAI-style in-memory economics; tune per upstream | + +The `ollama` table key is the bare provider segment; the harness stamps slash-form on `sourceId` (`ollama/default` / `ollama/`), with `provider: openai-compatible` (`buildOpenAISource`) and a bare model (`llama3` / `qwen3`). Idle-recompress disable matches via `isOllamaProviderId` on that `sourceId` (`src/provider/ollama.ts`), not by looking up slash-form as a table key. + - **Overflow recovery** — A `context_overflow` inference error would otherwise become a terminal error reply; the governor compacts and retries instead, bounded so a history the compactor cannot shrink does not loop forever. Overflow ignores hysteresis for the compact itself. +The TTL window is measured from the last `inference.done` (every inference rewrites the provider's prefix cache) with one fire per window per compact of any kind, and it never fires with a tool batch outstanding — stall pings can arrive mid-work. The consecutive-compact cap shared with the threshold path still bounds compact→infer→compact. Prompt-caching the summary call itself is an explicit non-goal: the fold carries no cache options. + The compaction control flow is shaped by a reactor invariant: a `compact` action runs in its own cycle (it cannot be paired with `infer`), and **the reactor delivers no event after a compact cycle**. A director that simply emitted `compact` in place of the follow-up `infer` would leave the loop idle forever — the cause of an earlier stall. Instead the governor pairs `compact` with an emit action (`custom.compaction.continue`), and the host answers that emission by delivering a content-less inbound message (`buildCompactionContinuationMessage()`) through the serial operation queue, generation-guarded like every other deliver. A hop superseded because interrupt already bumped generation and enqueued a rebuild is re-queued onto the replacement agent rather than consumed on the outgoing liveAgent. That message adds no turn (`createInboundTurn` returns `null` for empty content) but re-enters the loop, where the director issues the follow-up `infer` against the freshly truncated history. Each emission is answered at most once (a replayed duplicate of an already-answered emission is ignored), and an unsolicited continuation with neither resume flag set answers `wait` rather than burning a billable inference. The legacy `requestContinuation` closure survives only on the sub-agent path, which delivers directly with no host emit hop; idle arming keeps the two channels exclusive (closure fires and the arming flag stays false, or no closure and the flag reports the arming) so a caller honoring both cannot double-deliver. Compaction replaces older turns with a structured, workflow-aware summary rather than a stats blob: sections for **What Happened / What We're Doing / Relevant Links / Action Items / Next Steps**, with the active workflow and step woven in so compacting mid-`/build` or mid-`/plan` preserves the contract. The summary is produced by a one-shot model call; on any failure it falls back to a deterministic summary so a compaction cycle never breaks the session. diff --git a/src/agent/compaction.test.ts b/src/agent/compaction.test.ts index 44e9a8808..1d86af73d 100644 --- a/src/agent/compaction.test.ts +++ b/src/agent/compaction.test.ts @@ -786,3 +786,387 @@ describe("compaction governor", () => { expect(continuations).toBe(1); }); }); + +describe("provider-aware idle recompress (CL-8745)", () => { + const MINUTE_MS = 60_000; + + function ttlInferenceDone( + modelOrSource: + | string + | { sourceId?: string; provider?: string; model?: string }, + withTools: boolean, + ): Extract { + const source = + typeof modelOrSource === "string" + ? { sourceId: "s", provider: "p", model: modelOrSource } + : { + sourceId: modelOrSource.sourceId ?? "s", + provider: modelOrSource.provider ?? "p", + model: modelOrSource.model ?? "m", + }; + return { + type: "inference.done", + turn: { + role: "assistant", + content: withTools + ? [ + { + type: "tool_call", + id: "c1", + name: "read_file", + arguments: { path: "a.ts" }, + }, + ] + : [{ type: "text", text: "ok" }], + }, + usage: usage(1000), + source, + } as unknown as Extract; + } + + function racedMessage(): ReactorInboundEvent { + return { + type: "message.received", + message: { content: "next question" }, + } as ReactorInboundEvent; + } + + const ttlCompact = [ + { + type: "compact", + compactor: "pruning-compactor", + reason: "cache-ttl-recompress", + }, + ] as ReactorAction[]; + + test("fires past the provider TTL while under threshold, meter-only on empty", () => { + let continuations = 0; + let nowMs = 10_000_000; + const governor = createCompactionGovernor( + () => continuations++, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone( + ttlInferenceDone( + { provider: "anthropic", model: "claude-opus-4-6" }, + false, + ), + tenTurns, + ); + + // Inside the 5-minute Anthropic window: no fire. + nowMs += 4 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + + // Past the window: the same fold as the threshold path (same compactor, + // so the fresh tail stays raw) with an attributable reason. + nowMs += MINUTE_MS + 1; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect(continuations).toBe(1); + // Empty continuation adopts the shrunk turns without a new inference. + expect(governor.resumeAfterCompact(emptyMessage())).toBe("meter"); + }); + + test("production LastCycleSource: anthropic fires at 5m, codex stays quiet until 10m", () => { + // Harness stamps { sourceId, provider, model } with a bare model, not + // slash-form "anthropic/claude-opus-4-6". Anthropic's 5-minute window + // must come from provider, not a dummy model string; Codex must not + // inherit that 5-minute fire from sourceId "codex/work". + let nowMs = 15_000_000; + const clock = () => nowMs; + const anthropic = createCompactionGovernor(() => undefined, "", [], clock); + const codex = createCompactionGovernor(() => undefined, "", [], clock); + anthropic.noteInferenceDone( + ttlInferenceDone( + { provider: "anthropic", model: "claude-opus-4-6" }, + false, + ), + tenTurns, + ); + codex.noteInferenceDone( + ttlInferenceDone( + { + sourceId: "codex/work", + provider: "codex-responses", + model: "gpt-5.6-luna", + }, + false, + ), + tenTurns, + ); + + nowMs += 5 * MINUTE_MS + 1; + expect( + anthropic.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect( + codex.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + + nowMs += 5 * MINUTE_MS; + expect( + codex.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + }); + + test("follows provider economics: deepseek waits out its long window, ollama never fires", () => { + let nowMs = 20_000_000; + const clock = () => nowMs; + const deepseek = createCompactionGovernor(() => undefined, "", [], clock); + const local = createCompactionGovernor(() => undefined, "", [], clock); + deepseek.noteInferenceDone( + ttlInferenceDone("deepseek/deepseek-chat", false), + tenTurns, + ); + local.noteInferenceDone( + ttlInferenceDone("ollama/llama3.1", false), + tenTurns, + ); + + // Past Anthropic/OpenAI windows but inside DeepSeek's hour: neither fires. + nowMs += 30 * MINUTE_MS; + expect( + deepseek.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + expect( + local.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + + // Past DeepSeek's hour: recompress fires; local inference still never + // does — no remote cache means no cache benefit. + nowMs += 31 * MINUTE_MS; + expect( + deepseek.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect( + local.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + }); + + test("production ollama LastCycleSource never fires cache-ttl-recompress", () => { + // Harness stamps { sourceId, provider, model } with a bare model. Ollama is + // buildOpenAISource: sourceId "ollama/default", provider openai-compatible, + // model llama3. Keying TTL only off model would take the 10-minute default. + let nowMs = 25_000_000; + const governor = createCompactionGovernor( + () => undefined, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone( + ttlInferenceDone( + { + sourceId: "ollama/default", + provider: "openai-compatible", + model: "llama3", + }, + false, + ), + tenTurns, + ); + + nowMs += 11 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + }); + + test("stays inert with no observed cache write", () => { + const governor = createCompactionGovernor(() => undefined); + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + }); + + test("defers to the armed threshold path while over threshold", () => { + let nowMs = 30_000_000; + const governor = createCompactionGovernor( + () => undefined, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone(inferenceDone(overThreshold), tenTurns); + // The "m" fixture model carries the 10-minute default TTL; advance past it. + nowMs += 11 * MINUTE_MS; + // Threshold arming owns the over-threshold session: no TTL double-fold. + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + expect( + governor.interceptIdleContinuation(racedMessage(), capabilities), + ).toBeNull(); + }); + + test("fires cache-ttl-recompress in the hysteresis gap (over-threshold, no growth)", () => { + // Characterization, not a bug: after a threshold compact, a post-compact + // infer at the same usage clears `pending` via growth hysteresis, so the + // threshold path no longer owns the session. Idle past the provider TTL + // still folds — same window- and cap-bounded path as under-threshold. + let nowMs = 70_000_000; + const governor = createCompactionGovernor( + () => undefined, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone(inferenceDone(overThreshold), tenTurns); + expect( + governor.interceptActions(toolDone(), inferAction, capabilities), + ).not.toBeNull(); + expect(governor.resumeAfterCompact(emptyMessage())).toBe("infer"); + + governor.noteInferenceDone(inferenceDone(overThreshold), tenTurns); + expect( + governor.interceptActions(toolDone(), inferAction, capabilities), + ).toBeNull(); + + nowMs += 11 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + }); + + test("after threshold compact and gap TTL, growth-armed compact stays blocked until a tool_call", () => { + // Threshold compact (consecutive=1) plus TTL fire in the hysteresis gap + // (consecutive=2) fills the shared cap. Later growth that would re-arm + // the threshold path stays blocked until a tool_call occupancy resets it. + let nowMs = 80_000_000; + const governor = createCompactionGovernor( + () => undefined, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone(inferenceDone(overThreshold), tenTurns); + expect( + governor.interceptActions(toolDone(), inferAction, capabilities), + ).not.toBeNull(); + expect(governor.resumeAfterCompact(emptyMessage())).toBe("infer"); + + governor.noteInferenceDone(inferenceDone(overThreshold), tenTurns); + nowMs += 11 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect(governor.resumeAfterCompact(emptyMessage())).toBe("meter"); + + // Post-TTL snapshot, then growth past resumeDelta — pending re-arms, + // but the cap is full so interceptActions stays null. + governor.noteInferenceDone(inferenceDone(overThreshold), tenTurns); + governor.noteInferenceDone( + inferenceDone(overThreshold + resumeDelta), + tenTurns, + ); + expect( + governor.interceptActions(toolDone(), inferAction, capabilities), + ).toBeNull(); + + governor.noteInferenceDone( + inferenceDoneWithTools(overThreshold + 2 * resumeDelta), + tenTurns, + ); + expect( + governor.interceptActions(toolDone(), inferAction, capabilities), + ).not.toBeNull(); + }); + + test("does not fold under an outstanding tool batch, fires once it settles", () => { + let continuations = 0; + let nowMs = 40_000_000; + const governor = createCompactionGovernor( + () => continuations++, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone( + ttlInferenceDone("anthropic/claude-opus-4-6", true), + tenTurns, + ); + nowMs += 6 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + // The batch settles (threshold path uninvolved: under threshold). + expect( + governor.interceptActions(toolDone(), inferAction, capabilities), + ).toBeNull(); + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect(continuations).toBe(1); + }); + + test("one fire per window, then the shared consecutive-compact cap stops the spiral", () => { + let continuations = 0; + let nowMs = 50_000_000; + const governor = createCompactionGovernor( + () => continuations++, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone( + ttlInferenceDone("anthropic/claude-opus-4-6", false), + tenTurns, + ); + + nowMs += 5 * MINUTE_MS + 1; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect(governor.resumeAfterCompact(emptyMessage())).toBe("meter"); + governor.notePostCompact(tenTurns); + + // Inside the next window: no second fire. + nowMs += 4 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + + nowMs += MINUTE_MS + 1; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toEqual(ttlCompact); + expect(governor.resumeAfterCompact(emptyMessage())).toBe("meter"); + governor.notePostCompact(tenTurns); + + // Cap reached: the third window stays quiet. + nowMs += 5 * MINUTE_MS + 1; + expect( + governor.interceptIdleContinuation(emptyMessage(), capabilities), + ).toBeNull(); + expect(continuations).toBe(2); + }); + + test("a raced operator message past the TTL still folds, then re-infers", () => { + let continuations = 0; + let nowMs = 60_000_000; + const governor = createCompactionGovernor( + () => continuations++, + "", + [], + () => nowMs, + ); + governor.noteInferenceDone( + ttlInferenceDone("anthropic/claude-opus-4-6", false), + tenTurns, + ); + nowMs += 6 * MINUTE_MS; + expect( + governor.interceptIdleContinuation(racedMessage(), capabilities), + ).toEqual(ttlCompact); + expect(continuations).toBe(1); + // The follow-up empty continuation carries the infer that answers the + // raced question (mirrors the threshold raced path). + expect(governor.resumeAfterCompact(emptyMessage())).toBe("infer"); + }); +}); diff --git a/src/agent/compaction.ts b/src/agent/compaction.ts index 9f3fab454..13e378fb9 100644 --- a/src/agent/compaction.ts +++ b/src/agent/compaction.ts @@ -1,5 +1,6 @@ import type { ConversationTurn, + LastCycleSource, ReactorAction, ReactorCapabilities, ReactorInboundEvent, @@ -10,6 +11,7 @@ import { compactionThresholdFor, contextTokensFromUsage, } from "../provider/context-window.js"; +import { cacheTtlMsFor } from "../provider/cache-ttl.js"; import { COMPACTOR_KEEP_RECENT_TURNS, assistantTextIsCompactSpacerEcho, @@ -64,6 +66,7 @@ export function createCompactionGovernor( requestContinuation?: () => void, systemPrompt = "", toolDefinitions: readonly ToolDefinition[] = [], + now: () => number = Date.now, ) { let pending = false; let idlePending = false; @@ -79,10 +82,30 @@ export function createCompactionGovernor( // can flag the number as approximate instead of implying provider-grade // precision. let usingEstimate = false; - // Model of the last inference.done turn, kept for live re-checks between - // inference cycles (see interceptActions) where the event carries no model. + // Last-cycle source of the last inference.done, kept for live re-checks + // between inference cycles (see interceptActions) where the event carries + // no source. Threshold sizing still keys off `model`; TTL identity needs + // `sourceId` / `provider` as well — production LastCycleSource stamps a + // bare model, and Ollama is `openai-compatible` with id `ollama/…`. let lastModel: string | undefined; + let lastCycleSource: LastCycleSource | undefined; let turnCount = 0; + // Wall-clock of the last inference.done: the provider (re)wrote its prefix + // cache for this session on that turn, so the provider TTL window in + // provider/cache-ttl.ts is measured from here. Stamped on every + // inference.done — estimated-usage providers wrote a cache entry too. + let lastCacheWriteAt: number | undefined; + // Wall-clock of the last issued compact of any kind. A fresh fold rewrites + // the session prefix, so the TTL recompress must not fire again inside the + // same window even when its own cache write has not been observed yet (the + // summary call bypasses this governor). + let lastCompactAt: number | undefined; + // Tool calls issued by the last inference.done and not yet settled. A TTL + // recompress must never fold while a batch is outstanding — the stall ping + // that triggers it can arrive mid-work, and folding under it would rewrite + // turns the pending results still belong to. Assigned (not incremented) on + // every inference.done so the serial loop self-heals a miscount. + let outstandingToolCalls = 0; // Growth hysteresis after a compact that remained over the high watermark: // snapshot the post-compact infer's usage, then do not re-arm until usage // grows by resumeDelta. Cleared once usage drops back to or under high. @@ -122,6 +145,7 @@ export function createCompactionGovernor( function noteCompactIssued(): void { awaitingPostCompactMeasurement = true; + lastCompactAt = now(); } function atThresholdCompactCap(): boolean { @@ -170,7 +194,18 @@ export function createCompactionGovernor( consecutiveThresholdCompacts = 0; } syncFromTurns(turns); + lastCycleSource = event.source; lastModel = event.source?.model; + lastCacheWriteAt = now(); + // The terminal reply ends the previous tool batch (its results are + // already in the turns) and opens the batch the reply just issued. A TTL + // recompress must never fold while a batch is outstanding — the stall + // ping that triggers it can arrive mid-work, and folding under it would + // rewrite turns the pending results still belong to. Assigned, not + // incremented, so the serial loop self-heals a miscount. + outstandingToolCalls = event.turn.content.filter( + (block) => block.type === "tool_call", + ).length; const reportedTokens = contextTokensFromUsage(event.usage); usingEstimate = reportedTokens <= 0; const contextTokens = usingEstimate ? estimate.tokens : reportedTokens; @@ -210,6 +245,7 @@ export function createCompactionGovernor( capabilities: ReactorCapabilities, ): ReactorAction[] | null { if (event.type !== "tool.done") return null; + if (outstandingToolCalls > 0) outstandingToolCalls -= 1; if (!pending && !(usingEstimate && isOverThreshold(estimate.tokens))) return null; if (!actions.some((a) => a.type === "infer")) return null; @@ -256,33 +292,92 @@ export function createCompactionGovernor( return true; } + // Provider-aware idle recompress (CL-8745): the fold is a re-compress, not + // a cache play. Provider KV caches expire on their own schedule + // (provider/cache-ttl.ts); compressing after that expiry makes the next + // turn a cheaper write and later reads compound on the shrunk context. This + // fires on any live re-entry once `now - lastCacheWrite >= ttl`, including + // over-threshold sessions whose threshold path has disarmed (`pending` is + // false via growth hysteresis — the threshold path owns only armed + // over-threshold; the fold is still window- and cap-bounded). Guards, in + // order: threshold arming defers (pending), the fresh-tail floor (turns at + // or under it are all kept, so a fold would shrink nothing), the + // consecutive-compact cap (existing death-spiral bound, shared with the + // threshold path), providers with no TTL (undefined/empty model, local + // inference), no observed cache write yet, the TTL window itself, and one + // fire per window (a fresh fold rewrites the prefix; the summary call + // bypasses this governor so lastCacheWriteAt cannot observe it — + // lastCompactAt covers that). Never fires with a tool batch outstanding: + // the stall ping that triggers this can arrive mid-work. + function isTtlRecompressDue(nowMs: number): boolean { + if (pending) return false; + if (turnCount <= MIN_TURNS_TO_COMPACT) return false; + if (atThresholdCompactCap()) return false; + if (outstandingToolCalls > 0) return false; + const ttl = cacheTtlMsFor(lastCycleSource); + if (ttl === undefined) return false; + if (lastCacheWriteAt === undefined) return false; + if (nowMs - lastCacheWriteAt < ttl) return false; + if (lastCompactAt !== undefined && nowMs - lastCompactAt < ttl) + return false; + return true; + } + + function inboundText(event: ReactorInboundEvent): string { + if (event.type !== "message.received") return ""; + return typeof event.message.content === "string" + ? event.message.content + : ""; + } + + // Idle empty compact needs a meter-only re-entry; a raced operator message + // needs a follow-up infer. Shared by the threshold idle path and TTL fold. + function issueIdleFold( + content: string, + capabilities: ReactorCapabilities, + reason: string, + ): ReactorAction[] { + if (content.length > 0) postCompactInfer = true; + else postCompactMeter = true; + issueThresholdCompact(); + return [ + capabilities.compact(COMPACTOR_NAME, reason), + ...continuationActions(capabilities), + ]; + } + function interceptIdleContinuation( event: ReactorInboundEvent, capabilities: ReactorCapabilities, ): ReactorAction[] | null { - if (!idlePending || event.type !== "message.received") return null; - if (atThresholdCompactCap()) { + if (event.type !== "message.received") return null; + if (idlePending) { + if (atThresholdCompactCap()) { + idlePending = false; + return null; + } idlePending = false; - return null; - } - idlePending = false; - pending = false; - const content = - typeof event.message.content === "string" ? event.message.content : ""; - // The reactor delivers no event after compact, so always request a - // continuation to re-enter decide against the shrunk turns: - // - raced operator content → re-infer to answer it - // - empty synthetic continuation → meter-only sync (no infer) - if (content.length > 0) { - postCompactInfer = true; - } else { - postCompactMeter = true; + pending = false; + // The reactor delivers no event after compact, so always request a + // continuation to re-enter decide against the shrunk turns: + // - raced operator content → re-infer to answer it + // - empty synthetic continuation → meter-only sync (no infer) + return issueIdleFold( + inboundText(event), + capabilities, + "context-threshold", + ); } - issueThresholdCompact(); - return [ - capabilities.compact(COMPACTOR_NAME, "context-threshold"), - ...continuationActions(capabilities), - ]; + // Unarmed idle re-entry past the provider TTL: same fold, same + // keep-recent tail, same cap — but a "cache-ttl-recompress" reason so the + // fold is attributable. No arming: every live re-entry re-checks the + // window, so a sub-agent stall ping or operator message is the trigger. + if (!isTtlRecompressDue(now())) return null; + return issueIdleFold( + inboundText(event), + capabilities, + "cache-ttl-recompress", + ); } // A context-overflow inference error would otherwise terminate the loop @@ -316,9 +411,7 @@ export function createCompactionGovernor( event: ReactorInboundEvent, ): "infer" | "meter" | null { if (event.type !== "message.received") return null; - const content = - typeof event.message.content === "string" ? event.message.content : ""; - if (content.length > 0) return null; + if (inboundText(event).length > 0) return null; if (postCompactInfer) { postCompactInfer = false; return "infer"; @@ -336,6 +429,10 @@ export function createCompactionGovernor( function notePostCompact(turns: readonly ConversationTurn[]): void { syncFromTurns(turns); usingEstimate = true; + // A fold rewrites the turns: results already applied vanish from the + // live set, and post-compact stall pings (empty continuations) carry no + // tool traffic. Reset so a stale count cannot pin the TTL window shut. + outstandingToolCalls = 0; } // True while the governor expects the host to answer a continuation emit. diff --git a/src/provider/cache-ttl.test.ts b/src/provider/cache-ttl.test.ts new file mode 100644 index 000000000..0bf58fbf4 --- /dev/null +++ b/src/provider/cache-ttl.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test"; +import { cacheTtlMsFor } from "./cache-ttl.js"; + +const MINUTE_MS = 60_000; + +describe("cacheTtlMsFor", () => { + test("maps providers to their documented cache TTL windows", () => { + expect(cacheTtlMsFor("anthropic/claude-opus-4-6")).toBe(5 * MINUTE_MS); + expect(cacheTtlMsFor("openai-responses/gpt-5.6")).toBe(10 * MINUTE_MS); + expect(cacheTtlMsFor("codex-responses/gpt-5.6")).toBe(10 * MINUTE_MS); + expect(cacheTtlMsFor("openai-compatible/custom")).toBe(10 * MINUTE_MS); + expect(cacheTtlMsFor("xai/thegreataxios")).toBe(10 * MINUTE_MS); + expect(cacheTtlMsFor("gemini/gemini-3-pro")).toBe(15 * MINUTE_MS); + expect(cacheTtlMsFor("deepseek/deepseek-chat")).toBe(60 * MINUTE_MS); + }); + + test("covers Anthropic-protocol adapters with the 5-minute window", () => { + expect(cacheTtlMsFor("zen-messages/claude-opus-4-6")).toBe(5 * MINUTE_MS); + expect(cacheTtlMsFor("opencode-go-messages/claude-opus-4-6")).toBe( + 5 * MINUTE_MS, + ); + }); + + test("disables idle recompress for local inference and missing model ids", () => { + expect(cacheTtlMsFor("ollama/llama3.1")).toBeUndefined(); + expect(cacheTtlMsFor(undefined)).toBeUndefined(); + expect(cacheTtlMsFor("")).toBeUndefined(); + }); + + test("keys ollama off production LastCycleSource, not a slash-form model", () => { + expect( + cacheTtlMsFor({ + sourceId: "ollama/default", + provider: "openai-compatible", + model: "llama3", + }), + ).toBeUndefined(); + }); + + test("maps bare LastCycleSource ids through provider and family", () => { + expect( + cacheTtlMsFor({ + provider: "anthropic", + model: "claude-opus-4-6", + }), + ).toBe(5 * MINUTE_MS); + expect( + cacheTtlMsFor({ + provider: "codex-responses", + model: "gpt-5.6-luna", + }), + ).toBe(10 * MINUTE_MS); + }); + + test("falls back to model family for unrecognized provider prefixes", () => { + expect(cacheTtlMsFor("proxy-acme/grok-4")).toBe(10 * MINUTE_MS); + expect(cacheTtlMsFor("proxy-acme/gemini-3-pro")).toBe(15 * MINUTE_MS); + expect(cacheTtlMsFor("proxy-acme/ollama-qwen")).toBeUndefined(); + // An exact provider-segment match wins over the model family: an + // openai-compatible account fronting Claude keeps the generic window. + expect(cacheTtlMsFor("openai-compatible/claude-opus-4-6")).toBe( + 10 * MINUTE_MS, + ); + }); + + test("assumes OpenAI-style economics for unrecognized providers", () => { + expect(cacheTtlMsFor("bifrost/some-model")).toBe(10 * MINUTE_MS); + expect(cacheTtlMsFor("totally-new-provider/model-x")).toBe(10 * MINUTE_MS); + // A truly unknown id (no provider segment, no family substring) still + // gets the 10-minute default — not the local-inference disable. + expect(cacheTtlMsFor("unknown-id")).toBe(10 * MINUTE_MS); + }); +}); diff --git a/src/provider/cache-ttl.ts b/src/provider/cache-ttl.ts new file mode 100644 index 000000000..a3803b52f --- /dev/null +++ b/src/provider/cache-ttl.ts @@ -0,0 +1,136 @@ +// Provider prompt-cache TTLs for idle recompression (CL-8745). +// +// The fold is a re-compress, not a cache play: provider KV caches expire on +// their own schedule, and idle compression fires *after* that expiry so the +// next turn is a cheaper write (a smaller prefix to re-process) and later +// reads compound on the shrunk context. The interval therefore follows +// provider cache economics, not a single global N minutes. +// +// Mapping — idle recompress is allowed once `now - lastCacheWrite >= ttl`: +// - Anthropic (`anthropic`, plus `zen-messages` / `opencode-go-messages`, +// which speak the Anthropic messages protocol): default 5-minute ephemeral +// cache TTL. Hour-long breakpoints exist but we never set them. 5 min. +// - OpenAI family (`openai-responses`, `codex-responses`, +// `openai-compatible`): in-memory prefixes are typically evicted after +// 5-10 minutes idle (retained up to an hour off-peak); GPT-5.6+ families +// default to a 30-minute minimum TTL instead. 10 min splits the range. +// - xAI (`xai`, `grok-*`): per-server cache, no published TTL; assumed +// OpenAI-style in-memory economics. 10 min. +// - Gemini (`gemini`, `gemini-*`): implicit caching on 2.5+ with no published +// eviction window; explicit caches default to 1 hour. Conservative. 15 min. +// - DeepSeek (`deepseek`): on-disk context cache cleared only after +// hours-to-days idle, so an early recompress would fold a still-warm cache. +// 60 min. +// - ollama: local inference has no remote cache to expire, so an idle +// recompress would burn local compute for no cache benefit. Disabled. +// Production LastCycleSource is `{ sourceId, provider, model }` with a +// bare model; Ollama is `buildOpenAISource` (`provider: openai-compatible`, +// `sourceId` like `ollama/default`, model `llama3` / `qwen3`). Identity is +// `sourceId` via `isOllamaProviderId`, not a slash-form string the harness +// never stamps. +// - Unknown/custom providers (bifrost proxy, `openai-compatible` fronting an +// unlisted upstream, unrecognized strings): assumed OpenAI-style in-memory +// economics. 10 min. +// +// Deliberate non-goal: the summary call itself carries no `cache_control` — +// prompt-caching the fold is not attempted. + +import { isOllamaProviderId } from "./ollama.js"; + +const MINUTE_MS = 60_000; + +const PROVIDER_TTLS_MS: Record = { + anthropic: 5 * MINUTE_MS, + "zen-messages": 5 * MINUTE_MS, + "opencode-go-messages": 5 * MINUTE_MS, + "openai-responses": 10 * MINUTE_MS, + "codex-responses": 10 * MINUTE_MS, + "openai-compatible": 10 * MINUTE_MS, + gemini: 15 * MINUTE_MS, + deepseek: 60 * MINUTE_MS, + xai: 10 * MINUTE_MS, + // No remote cache to expire; idle recompress would only burn local compute. + ollama: undefined, +}; + +const DEFAULT_TTL_MS = 10 * MINUTE_MS; + +// Model-family fallback for unrecognized provider prefixes (custom proxies, +// account names). An exact provider-segment match always wins — e.g. an +// `openai-compatible` account fronting Claude keeps the generic 10-minute +// window rather than Claude's 5. Checked in order; first substring wins. +const FAMILY_TTLS_MS: readonly (readonly [string, number | undefined])[] = [ + ["claude", 5 * MINUTE_MS], + ["anthropic", 5 * MINUTE_MS], + ["gpt", 10 * MINUTE_MS], + ["codex", 10 * MINUTE_MS], + ["openai", 10 * MINUTE_MS], + ["grok", 10 * MINUTE_MS], + ["xai", 10 * MINUTE_MS], + ["gemini", 15 * MINUTE_MS], + ["deepseek", 60 * MINUTE_MS], + ["ollama", undefined], +]; + +type CacheTtlIdentity = { + sourceId?: string; + provider?: string; + model?: string; +}; + +// Canonical provider segment of a `provider:model` string: the account or +// adapter name before the first "/" (custom names like `xai/thegreataxios` +// carry the provider there), else the head before ":". Mirrors the +// segmentation in provider/context-window.ts without its window-table +// fallback semantics. +function canonicalSegment(model: string): string { + const lower = model.toLowerCase(); + const slash = lower.indexOf("/"); + const head = slash >= 0 ? lower.slice(0, slash) : lower; + const colon = head.indexOf(":"); + return colon >= 0 ? head.slice(0, colon) : head; +} + +function ttlForModelString(model: string | undefined): number | undefined { + if (model === undefined) return undefined; + const segment = canonicalSegment(model); + if (segment.length === 0) return undefined; + if (Object.hasOwn(PROVIDER_TTLS_MS, segment)) + return PROVIDER_TTLS_MS[segment]; + const lower = model.toLowerCase(); + for (const [family, ttl] of FAMILY_TTLS_MS) { + if (lower.includes(family)) return ttl; + } + return DEFAULT_TTL_MS; +} + +/** + * Milliseconds of provider-cache idle after which a recompress is allowed, + * or `undefined` when the identity is missing/empty or the provider has no + * remote cache (local inference). Never throws; unknown strings get the + * default 10-minute window. + * + * Accepts a slash-form `provider/model` string or a LastCycleSource-shaped + * identity. Production events stamp a bare `model`; Ollama is recognized + * from `sourceId` (`isOllamaProviderId`), not from a combined string the + * harness never stamps. + */ +export function cacheTtlMsFor( + identity: string | CacheTtlIdentity | undefined, +): number | undefined { + if (identity === undefined) return undefined; + if (typeof identity === "string") return ttlForModelString(identity); + + if (identity.sourceId !== undefined && isOllamaProviderId(identity.sourceId)) + return undefined; + if (identity.provider !== undefined && isOllamaProviderId(identity.provider)) + return undefined; + + if (identity.provider !== undefined) { + const provider = identity.provider.toLowerCase(); + if (Object.hasOwn(PROVIDER_TTLS_MS, provider)) + return PROVIDER_TTLS_MS[provider]; + } + + return ttlForModelString(identity.model); +}