diff --git a/.github/workflows/llm-price-drift.yml b/.github/workflows/llm-price-drift.yml new file mode 100644 index 00000000000..c3a3f50db7d --- /dev/null +++ b/.github/workflows/llm-price-drift.yml @@ -0,0 +1,27 @@ +name: LLM price drift + +# Checks the hard-coded per-token prices in core/llm/utils/calculateRequestCost.ts +# against the official Anthropic and OpenAI pricing pages, so the cost shown in +# the console and CLI does not silently go stale. +on: + pull_request: + branches: [main] + paths: + - "core/llm/utils/calculateRequestCost.ts" + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: Mattbusel/llm-prices@v1 + with: + paths: core/llm/utils/calculateRequestCost.ts + providers: anthropic,openai + fail-on: drift diff --git a/core/llm/utils/calculateRequestCost.ts b/core/llm/utils/calculateRequestCost.ts index 794524d448d..8d945337f62 100644 --- a/core/llm/utils/calculateRequestCost.ts +++ b/core/llm/utils/calculateRequestCost.ts @@ -41,7 +41,44 @@ function calculateAnthropicCost( cacheRead: 0.5, }, - // Claude Opus 4 (legacy) + // Claude Opus 4.1 and Opus 4 (dated ids and the -0 alias; a bare + // "claude-opus-4" prefix would also catch the cheaper Opus 4.5+ ids) + "claude-opus-4-1": { + input: 15, + output: 75, + cacheWrite: 18.75, + cacheRead: 1.5, + }, + "claude-opus-4-0": { + input: 15, + output: 75, + cacheWrite: 18.75, + cacheRead: 1.5, + }, + "claude-opus-4-2025": { + input: 15, + output: 75, + cacheWrite: 18.75, + cacheRead: 1.5, + }, + + // Claude Sonnet 4 and Sonnet 4.5 (same price as Sonnet 4.6) + "claude-sonnet-4": { + input: 3, + output: 15, + cacheWrite: 3.75, + cacheRead: 0.3, + }, + + // Claude Haiku 4.5 + "claude-haiku-4-5": { + input: 1, + output: 5, + cacheWrite: 1.25, + cacheRead: 0.1, + }, + + // Claude 3 Opus (legacy) "claude-3-opus": { input: 15, output: 75, @@ -49,7 +86,15 @@ function calculateAnthropicCost( cacheRead: 1.5, }, - // Claude Sonnet 4 (optimal balance) + // Claude 3.7 Sonnet + "claude-3-7-sonnet": { + input: 3, + output: 15, + cacheWrite: 3.75, + cacheRead: 0.3, + }, + + // Claude 3.5 Sonnet "claude-3-5-sonnet": { input: 3, output: 15, @@ -157,17 +202,31 @@ function calculateOpenAICost( // Define pricing per million tokens (MTok) by model family prefix const pricing: Record = { + // GPT-4.1 models (must be listed so they don't fall back to base GPT-4) + "gpt-4.1-nano": { input: 0.1, output: 0.4 }, + "gpt-4.1-mini": { input: 0.4, output: 1.6 }, + "gpt-4.1": { input: 2, output: 8 }, + // GPT-4o models (most specific first) "gpt-4o-mini": { input: 0.15, output: 0.6 }, + "gpt-4o-2024-05-13": { input: 5, output: 15 }, "gpt-4o": { input: 2.5, output: 10 }, // GPT-4 Turbo models "gpt-4-turbo": { input: 10, output: 30 }, // GPT-3.5 Turbo models (most specific first) + "gpt-3.5-turbo-instruct": { input: 1.5, output: 2 }, + "gpt-3.5-turbo-16k": { input: 3, output: 4 }, "gpt-3.5-turbo-0125": { input: 0.5, output: 1.5 }, "gpt-3.5-turbo-1106": { input: 1, output: 2 }, - "gpt-3.5-turbo": { input: 1.5, output: 2 }, + "gpt-3.5-turbo-0613": { input: 1.5, output: 2 }, + "gpt-3.5-turbo-0301": { input: 1.5, output: 2 }, + // The bare alias points to gpt-3.5-turbo-0125 + "gpt-3.5-turbo": { input: 0.5, output: 1.5 }, + + // GPT-4 32k context + "gpt-4-32k": { input: 60, output: 120 }, // Base GPT-4 (fallback for other gpt-4 variants) "gpt-4": { input: 30, output: 60 }, diff --git a/core/llm/utils/calculateRequestCost.vitest.ts b/core/llm/utils/calculateRequestCost.vitest.ts index 32183ca1b49..f98464296bf 100644 --- a/core/llm/utils/calculateRequestCost.vitest.ts +++ b/core/llm/utils/calculateRequestCost.vitest.ts @@ -119,6 +119,57 @@ describe("calculateRequestCost", () => { description: "Claude Opus 4.5 with cache writes", }, + // Anthropic Claude 4.x models that are not Opus 4.5+ + { + provider: "anthropic", + model: "claude-sonnet-4-5-20250929", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0105, + description: "Claude Sonnet 4.5", + }, + { + provider: "anthropic", + model: "claude-sonnet-4-20250514", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0105, + description: "Claude Sonnet 4", + }, + { + provider: "anthropic", + model: "claude-haiku-4-5-20251001", + promptTokens: 1000, + completionTokens: 500, + cachedTokens: 2000, + expectedCost: 0.0037, + description: "Claude Haiku 4.5 with cache reads", + }, + { + provider: "anthropic", + model: "claude-opus-4-1-20250805", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0525, + description: "Claude Opus 4.1", + }, + { + provider: "anthropic", + model: "claude-opus-4-20250514", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0525, + description: "Claude Opus 4 (dated id)", + }, + { + provider: "anthropic", + model: "claude-opus-4-6", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0175, + description: "Claude Opus 4.6 is not priced as Opus 4", + }, + // OpenAI GPT-4 { provider: "openai", @@ -162,6 +213,72 @@ describe("calculateRequestCost", () => { expectedCost: 0.00125, description: "GPT-3.5 Turbo", }, + { + provider: "openai", + model: "gpt-3.5-turbo", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.00125, + description: "GPT-3.5 Turbo bare alias (points to 0125)", + }, + { + provider: "openai", + model: "gpt-3.5-turbo-instruct", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0025, + description: "GPT-3.5 Turbo Instruct", + }, + { + provider: "openai", + model: "gpt-3.5-turbo-16k-0613", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.005, + description: "GPT-3.5 Turbo 16k", + }, + + // OpenAI models that used to fall through to base GPT-4 / GPT-4o + { + provider: "openai", + model: "gpt-4.1", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.006, + description: "GPT-4.1 is not priced as base GPT-4", + }, + { + provider: "openai", + model: "gpt-4.1-mini-2025-04-14", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0012, + description: "GPT-4.1 mini", + }, + { + provider: "openai", + model: "gpt-4.1-nano", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0003, + description: "GPT-4.1 nano", + }, + { + provider: "openai", + model: "gpt-4-32k", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.12, + description: "GPT-4 32k", + }, + { + provider: "openai", + model: "gpt-4o-2024-05-13", + promptTokens: 1000, + completionTokens: 500, + expectedCost: 0.0125, + description: "GPT-4o original snapshot", + }, // Edge cases {