Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions evals/capability/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,14 @@ bun run eval:capability -- --case simple-health --provider xai --model grok-4.5

# Multi-model matrix (cases × variants)
bun run eval:capability -- \
--matrix "xai:grok-4.5,openai:gpt-4.1" \
--matrix "xai:grok-4.5,openai:gpt-5.6" \
--out evals/capability/results/matrix.json

# Faster live matrix (independent cells; default is serial)
bun run eval:capability -- --provider <name> --model <id> --concurrency 4

# Labeled variants
bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-4.1"
bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-5.6"

# Baseline improve/regress (keys by variantId::caseId)
bun run eval:capability -- --provider <name> --model <id> \
Expand Down
16 changes: 8 additions & 8 deletions evals/capability/lib.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -314,17 +314,17 @@ describe("parseMatrix", () => {
});

test("parses provider:model cells", () => {
const v = parseMatrix("xai:grok-4.5,openai:gpt-4.1", {});
const v = parseMatrix("xai:grok-4.5,openai:gpt-5.6", {});
expect(v).toHaveLength(2);
expect(v[0]).toEqual({
id: "xai:grok-4.5",
provider: "xai",
model: "grok-4.5",
});
expect(v[1]).toEqual({
id: "openai:gpt-4.1",
id: "openai:gpt-5.6",
provider: "openai",
model: "gpt-4.1",
model: "gpt-5.6",
});
});

Expand Down Expand Up @@ -693,7 +693,7 @@ describe("detectProviderFallback", () => {
const info = detectProviderFallback({
requestedProvider: "xai",
resolvedProvider: "openai",
resolvedModel: "gpt-4.1",
resolvedModel: "gpt-5.6",
});
expect(info?.requestedProvider).toBe("xai");
expect(info?.resolvedProvider).toBe("openai");
Expand All @@ -704,12 +704,12 @@ describe("detectProviderFallback", () => {
requestedProvider: "xai",
requestedModel: "grok-4.5",
resolvedProvider: "openai",
resolvedModel: "gpt-4.1",
resolvedModel: "gpt-5.6",
});
expect(info).not.toBeNull();
const message = formatProviderFallback(defined(info));
expect(message).toContain("xai/grok-4.5");
expect(message).toContain("openai/gpt-4.1");
expect(message).toContain("openai/gpt-5.6");
});
});

Expand Down Expand Up @@ -776,10 +776,10 @@ describe("resolveRequestedProviderModel", () => {

test("a matrix cell's own override wins over the run's resolved labels", () => {
const requested = resolveRequestedProviderModel(
{ provider: "openai", model: "gpt-4.1" },
{ provider: "openai", model: "gpt-5.6" },
{ provider: "xai", model: "grok-4.5" },
);
expect(requested).toEqual({ provider: "openai", model: "gpt-4.1" });
expect(requested).toEqual({ provider: "openai", model: "gpt-5.6" });
});

test("the '(default)' placeholder from an unresolvable ambient probe is not a real request", () => {
Expand Down
2 changes: 2 additions & 0 deletions packages/first-class-providers/src/providers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ describe("FIRST_CLASS_PROVIDERS", () => {
for (const model of api?.maxCompletionTokensModels ?? []) {
expect(api?.models).toContain(model);
}
// Keep: gpt-4.1 remains the catalog's non-reasoning OpenAI API model and
// must not join the max_completion_tokens set.
expect(api?.maxCompletionTokensModels).not.toContain("gpt-4.1");
});

Expand Down
3 changes: 3 additions & 0 deletions packages/first-class-providers/src/providers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ const OPENAI_API_MODELS = [
"gpt-6-astra",
"gpt-5.4",
"gpt-5.4-mini",
// Keep: still a first-party API catalog id; non-reasoning so it stays on
// max_tokens (not in OPENAI_API_MAX_COMPLETION_TOKENS_MODELS). Dropping it
// would change Connect/API-path model lists.
"gpt-4.1",
"o3",
"o4-mini",
Expand Down
4 changes: 2 additions & 2 deletions src/agent/prompt-sizes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export const CANONICAL_PROMPT_ENV: EnvironmentInfo = {

const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" };
// Default-family probe: anthropic/claude-sonnet-4 hits the claude row and
// openai/gpt-4.1 hits the gpt row, so an unrecognized provider is the probe
// openai/gpt-5.6 hits the gpt row, so an unrecognized provider is the probe
// that still resolves to the default family (no residual).
const MUSE_PROVIDER = {
providerName: "opencode-go",
Expand All @@ -73,7 +73,7 @@ const CLAUDE_PROVIDER = {
providerName: "anthropic",
model: "claude-sonnet-4",
};
const GPT_PROVIDER = { providerName: "openai", model: "gpt-4.1" };
const GPT_PROVIDER = { providerName: "openai", model: "gpt-5.6" };

/** Families in the size table: default (no residual), muse, grok, claude, gpt. */
export type PromptSizeFamily = "default" | "muse" | "grok" | "claude" | "gpt";
Expand Down
2 changes: 1 addition & 1 deletion src/agent/tool-schema-normalize.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ describe("normalizeToolDefinitionsForProvider", () => {
test("non-kimi providers get identity schemas (recursive present kept)", () => {
for (const ctx of [
{ providerName: "anthropic", model: "claude-sonnet-4" },
{ providerName: "openai", model: "gpt-4.1" },
{ providerName: "openai", model: "gpt-5.6" },
{ providerName: "xai/default", model: "grok-4.5" },
{ providerName: "opencode-go", model: "gpt-5.1" },
] as const) {
Expand Down
2 changes: 2 additions & 0 deletions src/config/inference-sources.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ describe("OpenAI reasoning max_completion_tokens quirk (CL-7785)", () => {
// max_tokens. Adding a preset model requires a decision here AND in the
// preset's maxCompletionTokensModels — the union below fails loudly
// otherwise instead of silently sending max_tokens.
// Keep: gpt-4.1 is the catalog's remaining non-reasoning preset. The
// union with maxCompletionTokensModels must equal OPENAI_API_MODELS.
const explicitMaxTokensModels = new Set(["gpt-4.1"]);
expect([...flagged, ...explicitMaxTokensModels].sort()).toEqual(
[...new Set(presetModels)].sort(),
Expand Down
4 changes: 2 additions & 2 deletions src/tui/model-catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ describe("buildModelCatalog", () => {
test("maps provider array to id/label picker options", () => {
const options = buildModelCatalog([
{ name: "xai", models: ["grok-4", "grok-3"], label: "xAI" },
{ name: "openai", models: ["gpt-4.1"] },
{ name: "openai", models: ["gpt-5.6"] },
]);
expect(options).toEqual([
{ id: "xai:grok-4", label: "grok-4 * [xAI]" },
{ id: "xai:grok-3", label: "grok-3 * [xAI]" },
{ id: "openai:gpt-4.1", label: "gpt-4.1 * [openai]" },
{ id: "openai:gpt-5.6", label: "gpt-5.6 * [openai]" },
]);
});

Expand Down
10 changes: 5 additions & 5 deletions tests/unit/pricing-fetcher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ test("parseModelsDevContextWindows reads limit.context per model", () => {
},
openai: {
models: {
"gpt-4.1": { id: "openai/gpt-4.1", limit: { context: 1_000_000 } },
"gpt-5.6": { id: "openai/gpt-5.6", limit: { context: 1_000_000 } },
"no-limit": { id: "openai/no-limit" },
},
},
});
expect(windows["z-ai/glm-4.6"]).toBe(64_000);
expect(windows["openai/gpt-4.1"]).toBe(1_000_000);
expect(windows["openai/gpt-5.6"]).toBe(1_000_000);
expect(windows["openai/no-limit"]).toBeUndefined();
});

Expand All @@ -53,8 +53,8 @@ test("parseModelsDevPricing supports multiple model IDs", () => {
const models = parseModelsDevPricing({
openai: {
models: {
"gpt-4.1": {
id: "openai/gpt-4.1",
"gpt-5.6": {
id: "openai/gpt-5.6",
input_cost_per_million: 2,
output_cost_per_million: 8,
cache_read_cost_per_million: 0.5,
Expand All @@ -72,7 +72,7 @@ test("parseModelsDevPricing supports multiple model IDs", () => {
},
});

expect(models["openai/gpt-4.1"]).toEqual({
expect(models["openai/gpt-5.6"]).toEqual({
inputPricePerToken: 0.000002,
outputPricePerToken: 0.000008,
cacheReadPricePerToken: 0.0000005,
Expand Down
Loading