Skip to content

Commit 80730a6

Browse files
TheGreatAxiosTheGreatAxios
committed
feat(provider): let custom providers set sampling and token knobs
1 parent c704dd7 commit 80730a6

16 files changed

Lines changed: 448 additions & 9 deletions

‎src/config.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1344,6 +1344,51 @@ describe("buildOpenAISource", () => {
13441344
});
13451345
});
13461346

1347+
test("overrides maxTokens and adds temperature to providerOptions", () => {
1348+
const source = buildOpenAISource({
1349+
id: "fp",
1350+
baseURL: "https://fp/v1",
1351+
apiKey: "k",
1352+
model: "m",
1353+
maxTokens: 8192,
1354+
temperature: 0.7,
1355+
});
1356+
expect(source.defaults).toEqual({
1357+
maxTokens: 8192,
1358+
providerOptions: { temperature: 0.7 },
1359+
});
1360+
});
1361+
1362+
test("adds top_p to providerOptions when topP is present", () => {
1363+
const source = buildOpenAISource({
1364+
id: "fp",
1365+
baseURL: "https://fp/v1",
1366+
apiKey: "k",
1367+
model: "m",
1368+
topP: 0.9,
1369+
});
1370+
expect(source.defaults).toEqual({
1371+
maxTokens: SOURCE_MAX_TOKENS,
1372+
providerOptions: { top_p: 0.9 },
1373+
});
1374+
});
1375+
1376+
test("coexists temperature with reasoning_effort in one providerOptions", () => {
1377+
const source = buildOpenAISource({
1378+
id: "fp",
1379+
baseURL: "https://fp/v1",
1380+
apiKey: "k",
1381+
model: "gpt-5.1",
1382+
reasoningEffort: "medium",
1383+
temperature: 1.0,
1384+
maxTokens: 4096,
1385+
});
1386+
expect(source.defaults).toEqual({
1387+
maxTokens: 4096,
1388+
providerOptions: { reasoning_effort: "medium", temperature: 1.0 },
1389+
});
1390+
});
1391+
13471392
test("projects an Ollama root URL to the OpenAI-compatible /v1 endpoint", () => {
13481393
const source = buildOpenAISource({
13491394
id: "ollama/default",

‎src/config/index.ts‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -272,12 +272,17 @@ export function buildOpenAISource(fields: {
272272
apiKey?: string;
273273
model: string;
274274
reasoningEffort?: ReasoningEffort;
275+
maxTokens?: number;
276+
temperature?: number;
277+
topP?: number;
275278
quirks?: Record<string, unknown>;
276279
}): InferenceSource {
277-
const overrides =
278-
fields.reasoningEffort !== undefined
279-
? { providerOptions: { reasoning_effort: fields.reasoningEffort } }
280-
: {};
280+
const providerOptions: Record<string, unknown> = {};
281+
if (fields.reasoningEffort !== undefined)
282+
providerOptions["reasoning_effort"] = fields.reasoningEffort;
283+
if (fields.temperature !== undefined)
284+
providerOptions["temperature"] = fields.temperature;
285+
if (fields.topP !== undefined) providerOptions["top_p"] = fields.topP;
281286
registerSourceSecret(fields.id, fields.apiKey);
282287
return {
283288
id: fields.id,
@@ -287,7 +292,10 @@ export function buildOpenAISource(fields: {
287292
: normalizeOpenAICompatibleBaseURL(fields.baseURL),
288293
credentialId: fields.id,
289294
model: fields.model,
290-
defaults: { maxTokens: SOURCE_MAX_TOKENS, ...overrides },
295+
defaults: {
296+
maxTokens: fields.maxTokens ?? SOURCE_MAX_TOKENS,
297+
...(Object.keys(providerOptions).length > 0 ? { providerOptions } : {}),
298+
},
291299
...(fields.quirks !== undefined ? { quirks: fields.quirks } : {}),
292300
};
293301
}
@@ -1405,6 +1413,11 @@ export function catalogEntryAsProviderSettings(
14051413
? { defaultModel: entry.defaultModel }
14061414
: {}),
14071415
...(entry.free !== undefined ? { free: entry.free } : {}),
1416+
...(entry.maxTokens !== undefined ? { maxTokens: entry.maxTokens } : {}),
1417+
...(entry.temperature !== undefined
1418+
? { temperature: entry.temperature }
1419+
: {}),
1420+
...(entry.topP !== undefined ? { topP: entry.topP } : {}),
14081421
...(entry.bifrostVirtualKey === true ? { bifrostVirtualKey: true } : {}),
14091422
...(entry.anthropic === true ? { anthropic: true } : {}),
14101423
...(go ? { opencodeGo: true } : {}),
@@ -1482,6 +1495,11 @@ export function buildProviderCatalog(
14821495
? { defaultModel: p.defaultModel }
14831496
: {}),
14841497
...(p.free !== undefined ? { free: p.free } : {}),
1498+
...(p.maxTokens !== undefined ? { maxTokens: p.maxTokens } : {}),
1499+
...(p.temperature !== undefined
1500+
? { temperature: p.temperature }
1501+
: {}),
1502+
...(p.topP !== undefined ? { topP: p.topP } : {}),
14851503
...(p.bifrostVirtualKey === true ? { bifrostVirtualKey: true } : {}),
14861504
...(p.anthropic === true ? { anthropic: true } : {}),
14871505
...(go ? { opencodeGo: true } : {}),

‎src/config/inference-sources.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,67 @@ describe("contextWindow / maxTokens split (CL-7784)", () => {
205205
});
206206
});
207207

208+
describe("custom provider token/sampling knobs flow into the source", () => {
209+
test("maxTokens/temperature/topP from provider settings reach defaults", () => {
210+
const entryCatalog: ProviderCatalogEntry[] = [
211+
{
212+
name: "custom",
213+
baseURL: "https://api.example.com/v1",
214+
apiKey: "sk-key",
215+
models: ["m"],
216+
},
217+
];
218+
const settings: Settings = {
219+
providers: {
220+
custom: {
221+
baseURL: "https://api.example.com/v1",
222+
apiKey: "sk-key",
223+
models: ["m"],
224+
maxTokens: 4096,
225+
temperature: 0.7,
226+
},
227+
},
228+
};
229+
const source = buildInferenceSourceForRef(
230+
{ provider: "custom", model: "m" },
231+
{ sessionId: "sess-1", catalog: entryCatalog },
232+
settings,
233+
);
234+
expect(source).not.toBeNull();
235+
expect(source?.defaults?.maxTokens).toBe(4096);
236+
expect(source?.defaults?.providerOptions).toEqual({ temperature: 0.7 });
237+
});
238+
239+
test("topP flows as top_p and overrides nothing else", () => {
240+
const entryCatalog: ProviderCatalogEntry[] = [
241+
{
242+
name: "custom",
243+
baseURL: "https://api.example.com/v1",
244+
apiKey: "sk-key",
245+
models: ["m"],
246+
topP: 0.9,
247+
},
248+
];
249+
const source = buildInferenceSourceForRef(
250+
{ provider: "custom", model: "m" },
251+
{ sessionId: "sess-1", catalog: entryCatalog },
252+
undefined,
253+
);
254+
expect(source?.defaults?.maxTokens).toBe(SOURCE_MAX_TOKENS);
255+
expect(source?.defaults?.providerOptions).toEqual({ top_p: 0.9 });
256+
});
257+
258+
test("unset knobs leave the default maxTokens and no providerOptions", () => {
259+
const source = buildInferenceSourceForRef(
260+
{ provider: "fp", model: "fp-large" },
261+
ctx(),
262+
undefined,
263+
);
264+
expect(source?.defaults?.maxTokens).toBe(SOURCE_MAX_TOKENS);
265+
expect(source?.defaults?.providerOptions).toBeUndefined();
266+
});
267+
});
268+
208269
describe("OpenAI reasoning max_completion_tokens quirk (CL-7785)", () => {
209270
const openaiApi = firstClassProviderById("openai")?.paths?.find(
210271
(p) => p.id === "api",

‎src/config/inference-sources.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ export function buildInferenceSourceForRef(
179179
}
180180

181181
const quirks = openAISourceQuirks(baseURL, ref.model);
182+
const maxTokens = entry?.maxTokens ?? providerSettings?.maxTokens;
183+
const temperature = entry?.temperature ?? providerSettings?.temperature;
184+
const topP = entry?.topP ?? providerSettings?.topP;
182185
return buildOpenAISource({
183186
id: ref.provider,
184187
baseURL,
@@ -189,6 +192,9 @@ export function buildInferenceSourceForRef(
189192
: {}),
190193
model: ref.model,
191194
...(effort !== undefined ? { reasoningEffort: effort } : {}),
195+
...(maxTokens !== undefined ? { maxTokens } : {}),
196+
...(temperature !== undefined ? { temperature } : {}),
197+
...(topP !== undefined ? { topP } : {}),
192198
...(quirks !== undefined ? { quirks } : {}),
193199
});
194200
}

‎src/config/providers.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,3 +298,31 @@ describe("buildProviderEntry protocol flag preservation", () => {
298298
expect(result.entry.opencodeGo).toBe(true);
299299
});
300300
});
301+
302+
describe("buildProviderEntry token/sampling knobs", () => {
303+
test("forwards maxTokens, temperature, and topP when present", () => {
304+
const entry = expectBuiltEntry({
305+
name: "custom",
306+
baseURL: "https://api.example.com/v1",
307+
apiKey: "sk-key",
308+
models: ["m"],
309+
maxTokens: 4096,
310+
temperature: 0.7,
311+
});
312+
expect(entry.maxTokens).toBe(4096);
313+
expect(entry.temperature).toBe(0.7);
314+
expect(entry.topP).toBeUndefined();
315+
});
316+
317+
test("omits all three when submission leaves them unset", () => {
318+
const entry = expectBuiltEntry({
319+
name: "custom",
320+
baseURL: "https://api.example.com/v1",
321+
apiKey: "sk-key",
322+
models: ["m"],
323+
});
324+
expect(entry.maxTokens).toBeUndefined();
325+
expect(entry.temperature).toBeUndefined();
326+
expect(entry.topP).toBeUndefined();
327+
});
328+
});

‎src/config/providers.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ export interface ProviderSubmission {
1717
bifrostVirtualKey?: boolean;
1818
anthropic?: boolean;
1919
opencodeGo?: boolean;
20+
maxTokens?: number;
21+
temperature?: number;
22+
topP?: number;
2023
}
2124

2225
// Precedence for the active/backup model of a provider: defaultModel wins
@@ -100,6 +103,13 @@ export function buildProviderEntry(
100103
: {}),
101104
...(anthropic ? { anthropic: true } : {}),
102105
...(opencodeGo ? { opencodeGo: true } : {}),
106+
...(submission.maxTokens !== undefined
107+
? { maxTokens: submission.maxTokens }
108+
: {}),
109+
...(submission.temperature !== undefined
110+
? { temperature: submission.temperature }
111+
: {}),
112+
...(submission.topP !== undefined ? { topP: submission.topP } : {}),
103113
};
104114
const catalog = currentCatalog
105115
.filter(

‎src/config/settings.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,15 @@ export interface ProviderSettings {
4444
// config load into contextWindowFor. OAuth-projected Codex/xAI entries drop
4545
// this field, so a hand-edited value on those providers is ignored.
4646
contextWindow?: number;
47+
// Per-call output cap for this provider, overriding the default
48+
// SOURCE_MAX_TOKENS when set.
49+
maxTokens?: number;
50+
// Sampling temperature (0..2, OpenAI-compatible). Mutually exclusive with
51+
// topP — never send both on the wire.
52+
temperature?: number;
53+
// Sampling top-p (0..1, OpenAI-compatible). Mutually exclusive with
54+
// temperature — never send both on the wire.
55+
topP?: number;
4756
// When true, this provider uses a Bifrost virtual key (sk-bf-...).
4857
// The marker causes the inference source to route through the Bifrost
4958
// adapter (which injects the x-bf-vk header) and enables model
@@ -242,6 +251,13 @@ function providerSelectionMetadata(
242251
...(provider.contextWindow !== undefined
243252
? { contextWindow: provider.contextWindow }
244253
: {}),
254+
...(provider.maxTokens !== undefined
255+
? { maxTokens: provider.maxTokens }
256+
: {}),
257+
...(provider.temperature !== undefined
258+
? { temperature: provider.temperature }
259+
: {}),
260+
...(provider.topP !== undefined ? { topP: provider.topP } : {}),
245261
...(provider.bifrostVirtualKey === true ? { bifrostVirtualKey: true } : {}),
246262
...(provider.anthropic === true ? { anthropic: true } : {}),
247263
...(provider.opencodeGo === true ? { opencodeGo: true } : {}),
@@ -616,6 +632,9 @@ const ProviderSettingsSchema = type({
616632
"keyless?": "boolean",
617633
"free?": "boolean",
618634
"contextWindow?": "number",
635+
"maxTokens?": "number",
636+
"temperature?": "number",
637+
"topP?": "number",
619638
"bifrostVirtualKey?": "boolean",
620639
"anthropic?": "boolean",
621640
"opencodeGo?": "boolean",

‎src/provider/openai-compatible-adapter.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,14 @@ describe("openai-compatible adapter providerOptions passthrough", () => {
6666
expect(body["reasoning_effort"]).toBe("high");
6767
});
6868

69+
test("merges temperature and top_p together from providerOptions", () => {
70+
const body = bodyFor({
71+
providerOptions: { temperature: 0.7, top_p: 0.9 },
72+
} as InferenceOptions);
73+
expect(body["temperature"]).toBe(0.7);
74+
expect(body["top_p"]).toBe(0.9);
75+
});
76+
6977
test("leaves the body untouched when no providerOptions are present", () => {
7078
const body = bodyFor({} as InferenceOptions);
7179
expect("reasoning_effort" in body).toBe(false);

‎src/tui/onboarding.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@ const CUSTOM_PROVIDER: ProviderFormValues = {
8585
apiKey: "test-key",
8686
model: "test-model",
8787
oauthProfile: "",
88+
contextWindow: "",
89+
maxTokens: "",
90+
temperature: "",
91+
topP: "",
8892
};
8993

9094
const ISOLATED_PROVIDER: ProviderFormValues = {
@@ -93,6 +97,10 @@ const ISOLATED_PROVIDER: ProviderFormValues = {
9397
apiKey: "isolated-key",
9498
model: "isolated-model",
9599
oauthProfile: "",
100+
contextWindow: "",
101+
maxTokens: "",
102+
temperature: "",
103+
topP: "",
96104
};
97105

98106
/** Stub the setup flow to submit one provider form, validation skipped. */
@@ -237,6 +245,10 @@ describe("runOnboarding settings source", () => {
237245
apiKey: "",
238246
model: "grok-4",
239247
oauthProfile: "work",
248+
contextWindow: "",
249+
maxTokens: "",
250+
temperature: "",
251+
topP: "",
240252
},
241253
{
242254
skipValidation: true,

0 commit comments

Comments
 (0)