diff --git a/src/provider/anthropic-cache-breakpoint.test.ts b/src/provider/anthropic-cache-breakpoint.test.ts index c498d2e14..616a9681b 100644 --- a/src/provider/anthropic-cache-breakpoint.test.ts +++ b/src/provider/anthropic-cache-breakpoint.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import type { AdapterRegistry } from "@intx/inference"; +import type { AdapterRegistry, ProviderAdapter } from "@intx/inference"; import { createBuiltinRegistry } from "@intx/inference/providers"; import type { ConversationTurn, @@ -25,6 +25,41 @@ const inner: AdapterRegistry = { const adapters = withAnthropicCacheBreakpoint(inner); +// Builtin Anthropic still hoists `role: "system"` into `body.system`. Newer +// models keep mid-conversation system in `messages`; this stub is that wire. +function preservingSystemAdapter(): ProviderAdapter { + return { + buildRequest(turns) { + const messages = turns.map((turn, index) => { + const content: WireBlock[] = turn.content.flatMap((block) => { + if (block.type !== "text") return []; + return [{ text: block.text }]; + }); + const isLastNonAssistant = + turn.role !== "assistant" && + turns.slice(index + 1).every((rest) => rest.role === "assistant"); + const last = content[content.length - 1]; + if (isLastNonAssistant && last !== undefined) { + last.cache_control = { type: "ephemeral" }; + } + return { role: turn.role, content }; + }); + return { + url: "https://example.test/messages", + headers: {}, + body: JSON.stringify({ messages }), + }; + }, + parseResponse: () => [], + parseJSONResponse: () => [], + }; +} + +const systemWireAdapters = withAnthropicCacheBreakpoint({ + has: (provider) => provider === "anthropic" || provider === "zen-messages", + resolve: () => preservingSystemAdapter(), +}); + function userTurn(text: string): ConversationTurn { return { role: "user", @@ -41,6 +76,14 @@ function assistantTurn(text: string): ConversationTurn { }; } +function systemTurn(text: string): ConversationTurn { + return { + role: "system", + timestamp: 0, + content: [{ type: "text", text }], + }; +} + type WireBlock = { cache_control?: unknown; text?: string; @@ -90,6 +133,56 @@ describe("anthropic cache breakpoint with ephemeral turns", () => { }); } + for (const provider of ["anthropic", "zen-messages"]) { + test(`${provider}: system-role ephemeral stays in messages and off the cached prefix`, () => { + const persisted = [userTurn("q1"), assistantTurn("a1"), userTurn("q2")]; + const inject = systemTurn("inject"); + const request = systemWireAdapters + .resolve(sourceFor(provider)) + .buildRequest([...persisted, inject], "test-model", { + ephemeralTurns: [inject], + } as InferenceOptions); + const body = wireBody(request.body); + + expect(body.messages).toHaveLength(4); + expect(body.messages[3]?.role).toBe("system"); + expect(body.messages[3]?.content[0]?.text).toBe("inject"); + expect( + body.messages[2]?.content[body.messages[2].content.length - 1] + ?.cache_control, + ).toEqual({ type: "ephemeral" }); + expect( + body.messages[3]?.content.filter( + (block) => block.cache_control !== undefined, + ), + ).toEqual([]); + }); + } + + test("builtin adapter + systemPrompt drops system-role ephemeral from messages", () => { + const persisted = [userTurn("q1"), assistantTurn("a1"), userTurn("q2")]; + const inject = systemTurn("inject"); + const request = adapters + .resolve(sourceFor("anthropic")) + .buildRequest([...persisted, inject], "test-model", { + systemPrompt: "stable prompt", + ephemeralTurns: [inject], + } as InferenceOptions); + const body = wireBody(request.body); + + expect(body.messages.map((message) => message.role)).toEqual([ + "user", + "assistant", + "user", + ]); + expect( + body.messages.flatMap((message) => + message.content.map((block) => block.text), + ), + ).not.toContain("inject"); + expect(body.system?.[0]?.text).toBe("stable prompt"); + }); + test("system and tools breakpoints stay untouched while the nudge is attached", () => { const persisted = [ { ...userTurn("sys"), role: "system" as const }, diff --git a/src/provider/anthropic-cache-breakpoint.ts b/src/provider/anthropic-cache-breakpoint.ts index 34417b84d..7f39dae2c 100644 --- a/src/provider/anthropic-cache-breakpoint.ts +++ b/src/provider/anthropic-cache-breakpoint.ts @@ -23,9 +23,10 @@ type WireMessage = { }; function ephemeralSuffixLength(options: ExtendedInferenceOptions): number { - const turns = options.ephemeralTurns; - if (turns === undefined) return 0; - return turns.filter((turn) => turn.role !== "system").length; + // Every ephemeral turn is suffix, including mid-conversation `role: "system"` + // injects. Role choice is ephemeralInjectRole; this wrapper only moves the + // cache breakpoint before that suffix. + return options.ephemeralTurns?.length ?? 0; } function isWireBlock(block: unknown): block is WireBlock { diff --git a/src/provider/ephemeral-inject-role.test.ts b/src/provider/ephemeral-inject-role.test.ts new file mode 100644 index 000000000..02d4cee3e --- /dev/null +++ b/src/provider/ephemeral-inject-role.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from "bun:test"; +import { + ephemeralInjectRole, + midConversationSystemSupported, +} from "./ephemeral-inject-role.js"; + +const SUPPORT_TABLE = [ + ["anthropic", "claude-sonnet-4-6", false], + ["zen-messages", "claude-sonnet-4-6", false], + ["anthropic", "claude-fable-5-1", true], + ["zen-messages", "claude-fable-5", true], + ["opencode-go-messages", "mythos", true], + ["anthropic", "claude-opus-4-8", true], + ["anthropic", "claude-opus-4.8", true], + ["anthropic", "claude-opus-5", true], + ["anthropic", "claude-opus-4-6", false], + ["openai", "claude-fable-5-1", false], + ["grok-responses", "grok-4", false], + ["openai", "gpt-5", false], +] as const; + +describe("midConversationSystemSupported", () => { + test.each(SUPPORT_TABLE)("%s / %s → %s", (provider, model, supported) => { + expect(midConversationSystemSupported(provider, model)).toBe(supported); + }); +}); + +describe("ephemeralInjectRole", () => { + test.each(SUPPORT_TABLE.map(([provider, model]) => [provider, model]))( + "%s / %s → user until the adapter preserves mid-conversation system", + (provider, model) => { + expect(ephemeralInjectRole(provider, model)).toBe("user"); + }, + ); +}); diff --git a/src/provider/ephemeral-inject-role.ts b/src/provider/ephemeral-inject-role.ts new file mode 100644 index 000000000..90155f3d4 --- /dev/null +++ b/src/provider/ephemeral-inject-role.ts @@ -0,0 +1,55 @@ +// Mailbox is for work; ephemeralTurns are for inject. +// +// Adapter hoist blocks `role: "system"` on the wire today: vendor +// `buildRequest` lifts every system turn into `body.system`, then drops +// those turns when `options.systemPrompt` is set. Until the adapter keeps +// mid-conversation system in `messages` under a system-prompt override, +// inject as `"user"`. +// +// Intended support (models that *could* keep mid-conversation system once +// the adapter preserves it): anthropic / zen-messages / opencode-go-messages +// with model-id substrings fable, mythos, opus-4.8, opus-4-8, opus-5. + +import { + OPENCODE_GO_MESSAGES_PROVIDER, + ZEN_MESSAGES_PROVIDER, +} from "./anthropic-session-adapter.js"; + +const ANTHROPIC_PROTOCOL_PROVIDERS: ReadonlySet = new Set([ + "anthropic", + ZEN_MESSAGES_PROVIDER, + OPENCODE_GO_MESSAGES_PROVIDER, +]); + +const MID_CONVERSATION_SYSTEM_MODEL_MARKERS = [ + "fable", + "mythos", + "opus-4.8", + "opus-4-8", + "opus-5", +] as const; + +export function midConversationSystemSupported( + provider: string, + model: string, +): boolean { + if (!ANTHROPIC_PROTOCOL_PROVIDERS.has(provider)) return false; + const id = model.toLowerCase(); + return MID_CONVERSATION_SYSTEM_MODEL_MARKERS.some((marker) => + id.includes(marker), + ); +} + +function adapterPreservesMidSystem(): boolean { + return false; +} + +export function ephemeralInjectRole( + provider: string, + model: string, +): "system" | "user" { + return midConversationSystemSupported(provider, model) && + adapterPreservesMidSystem() + ? "system" + : "user"; +}