Skip to content

Commit 36014b7

Browse files
TheGreatAxiosTheGreatAxios
committed
test(models): modernize leftover gpt-4.1 fixtures to gpt-5.6
1 parent fc5e047 commit 36014b7

9 files changed

Lines changed: 27 additions & 20 deletions

File tree

‎evals/capability/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,14 +138,14 @@ bun run eval:capability -- --case simple-health --provider xai --model grok-4.5
138138

139139
# Multi-model matrix (cases × variants)
140140
bun run eval:capability -- \
141-
--matrix "xai:grok-4.5,openai:gpt-4.1" \
141+
--matrix "xai:grok-4.5,openai:gpt-5.6" \
142142
--out evals/capability/results/matrix.json
143143

144144
# Faster live matrix (independent cells; default is serial)
145145
bun run eval:capability -- --provider <name> --model <id> --concurrency 4
146146

147147
# Labeled variants
148-
bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-4.1"
148+
bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-5.6"
149149

150150
# Baseline improve/regress (keys by variantId::caseId)
151151
bun run eval:capability -- --provider <name> --model <id> \

‎evals/capability/lib.test.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -314,17 +314,17 @@ describe("parseMatrix", () => {
314314
});
315315

316316
test("parses provider:model cells", () => {
317-
const v = parseMatrix("xai:grok-4.5,openai:gpt-4.1", {});
317+
const v = parseMatrix("xai:grok-4.5,openai:gpt-5.6", {});
318318
expect(v).toHaveLength(2);
319319
expect(v[0]).toEqual({
320320
id: "xai:grok-4.5",
321321
provider: "xai",
322322
model: "grok-4.5",
323323
});
324324
expect(v[1]).toEqual({
325-
id: "openai:gpt-4.1",
325+
id: "openai:gpt-5.6",
326326
provider: "openai",
327-
model: "gpt-4.1",
327+
model: "gpt-5.6",
328328
});
329329
});
330330

@@ -693,7 +693,7 @@ describe("detectProviderFallback", () => {
693693
const info = detectProviderFallback({
694694
requestedProvider: "xai",
695695
resolvedProvider: "openai",
696-
resolvedModel: "gpt-4.1",
696+
resolvedModel: "gpt-5.6",
697697
});
698698
expect(info?.requestedProvider).toBe("xai");
699699
expect(info?.resolvedProvider).toBe("openai");
@@ -704,12 +704,12 @@ describe("detectProviderFallback", () => {
704704
requestedProvider: "xai",
705705
requestedModel: "grok-4.5",
706706
resolvedProvider: "openai",
707-
resolvedModel: "gpt-4.1",
707+
resolvedModel: "gpt-5.6",
708708
});
709709
expect(info).not.toBeNull();
710710
const message = formatProviderFallback(defined(info));
711711
expect(message).toContain("xai/grok-4.5");
712-
expect(message).toContain("openai/gpt-4.1");
712+
expect(message).toContain("openai/gpt-5.6");
713713
});
714714
});
715715

@@ -776,10 +776,10 @@ describe("resolveRequestedProviderModel", () => {
776776

777777
test("a matrix cell's own override wins over the run's resolved labels", () => {
778778
const requested = resolveRequestedProviderModel(
779-
{ provider: "openai", model: "gpt-4.1" },
779+
{ provider: "openai", model: "gpt-5.6" },
780780
{ provider: "xai", model: "grok-4.5" },
781781
);
782-
expect(requested).toEqual({ provider: "openai", model: "gpt-4.1" });
782+
expect(requested).toEqual({ provider: "openai", model: "gpt-5.6" });
783783
});
784784

785785
test("the '(default)' placeholder from an unresolvable ambient probe is not a real request", () => {

‎packages/first-class-providers/src/providers.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ describe("FIRST_CLASS_PROVIDERS", () => {
6969
for (const model of api?.maxCompletionTokensModels ?? []) {
7070
expect(api?.models).toContain(model);
7171
}
72+
// Keep: gpt-4.1 remains the catalog's non-reasoning OpenAI API model and
73+
// must not join the max_completion_tokens set.
7274
expect(api?.maxCompletionTokensModels).not.toContain("gpt-4.1");
7375
});
7476

‎packages/first-class-providers/src/providers.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ const OPENAI_API_MODELS = [
1919
"gpt-6-astra",
2020
"gpt-5.4",
2121
"gpt-5.4-mini",
22+
// Keep: still a first-party API catalog id; non-reasoning so it stays on
23+
// max_tokens (not in OPENAI_API_MAX_COMPLETION_TOKENS_MODELS). Dropping it
24+
// would change Connect/API-path model lists.
2225
"gpt-4.1",
2326
"o3",
2427
"o4-mini",

‎src/agent/prompt-sizes.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ export const CANONICAL_PROMPT_ENV: EnvironmentInfo = {
5959

6060
const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" };
6161
// Default-family probe: anthropic/claude-sonnet-4 hits the claude row and
62-
// openai/gpt-4.1 hits the gpt row, so an unrecognized provider is the probe
62+
// openai/gpt-5.6 hits the gpt row, so an unrecognized provider is the probe
6363
// that still resolves to the default family (no residual).
6464
const MUSE_PROVIDER = {
6565
providerName: "opencode-go",
@@ -73,7 +73,7 @@ const CLAUDE_PROVIDER = {
7373
providerName: "anthropic",
7474
model: "claude-sonnet-4",
7575
};
76-
const GPT_PROVIDER = { providerName: "openai", model: "gpt-4.1" };
76+
const GPT_PROVIDER = { providerName: "openai", model: "gpt-5.6" };
7777

7878
/** Families in the size table: default (no residual), muse, grok, claude, gpt. */
7979
export type PromptSizeFamily = "default" | "muse" | "grok" | "claude" | "gpt";

‎src/agent/tool-schema-normalize.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ describe("normalizeToolDefinitionsForProvider", () => {
122122
test("non-kimi providers get identity schemas (recursive present kept)", () => {
123123
for (const ctx of [
124124
{ providerName: "anthropic", model: "claude-sonnet-4" },
125-
{ providerName: "openai", model: "gpt-4.1" },
125+
{ providerName: "openai", model: "gpt-5.6" },
126126
{ providerName: "xai/default", model: "grok-4.5" },
127127
{ providerName: "opencode-go", model: "gpt-5.1" },
128128
] as const) {

‎src/config/inference-sources.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,8 @@ describe("OpenAI reasoning max_completion_tokens quirk (CL-7785)", () => {
172172
// max_tokens. Adding a preset model requires a decision here AND in the
173173
// preset's maxCompletionTokensModels — the union below fails loudly
174174
// otherwise instead of silently sending max_tokens.
175+
// Keep: gpt-4.1 is the catalog's remaining non-reasoning preset. The
176+
// union with maxCompletionTokensModels must equal OPENAI_API_MODELS.
175177
const explicitMaxTokensModels = new Set(["gpt-4.1"]);
176178
expect([...flagged, ...explicitMaxTokensModels].sort()).toEqual(
177179
[...new Set(presetModels)].sort(),

‎src/tui/model-catalog.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ describe("buildModelCatalog", () => {
1111
test("maps provider array to id/label picker options", () => {
1212
const options = buildModelCatalog([
1313
{ name: "xai", models: ["grok-4", "grok-3"], label: "xAI" },
14-
{ name: "openai", models: ["gpt-4.1"] },
14+
{ name: "openai", models: ["gpt-5.6"] },
1515
]);
1616
expect(options).toEqual([
1717
{ id: "xai:grok-4", label: "grok-4 * [xAI]" },
1818
{ id: "xai:grok-3", label: "grok-3 * [xAI]" },
19-
{ id: "openai:gpt-4.1", label: "gpt-4.1 * [openai]" },
19+
{ id: "openai:gpt-5.6", label: "gpt-5.6 * [openai]" },
2020
]);
2121
});
2222

‎tests/unit/pricing-fetcher.test.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@ test("parseModelsDevContextWindows reads limit.context per model", () => {
2525
},
2626
openai: {
2727
models: {
28-
"gpt-4.1": { id: "openai/gpt-4.1", limit: { context: 1_000_000 } },
28+
"gpt-5.6": { id: "openai/gpt-5.6", limit: { context: 1_000_000 } },
2929
"no-limit": { id: "openai/no-limit" },
3030
},
3131
},
3232
});
3333
expect(windows["z-ai/glm-4.6"]).toBe(64_000);
34-
expect(windows["openai/gpt-4.1"]).toBe(1_000_000);
34+
expect(windows["openai/gpt-5.6"]).toBe(1_000_000);
3535
expect(windows["openai/no-limit"]).toBeUndefined();
3636
});
3737

@@ -53,8 +53,8 @@ test("parseModelsDevPricing supports multiple model IDs", () => {
5353
const models = parseModelsDevPricing({
5454
openai: {
5555
models: {
56-
"gpt-4.1": {
57-
id: "openai/gpt-4.1",
56+
"gpt-5.6": {
57+
id: "openai/gpt-5.6",
5858
input_cost_per_million: 2,
5959
output_cost_per_million: 8,
6060
cache_read_cost_per_million: 0.5,
@@ -72,7 +72,7 @@ test("parseModelsDevPricing supports multiple model IDs", () => {
7272
},
7373
});
7474

75-
expect(models["openai/gpt-4.1"]).toEqual({
75+
expect(models["openai/gpt-5.6"]).toEqual({
7676
inputPricePerToken: 0.000002,
7777
outputPricePerToken: 0.000008,
7878
cacheReadPricePerToken: 0.0000005,

0 commit comments

Comments
 (0)