Skip to content

Commit c3719f0

Browse files
TheGreatAxiosTheGreatAxios
committed
fix(inference): tolerate null role and tool_calls deltas
1 parent c1c1dba commit c3719f0

5 files changed

Lines changed: 299 additions & 8 deletions

File tree

‎src/agent/retry-policy.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ describe("createCorbitsRetryPolicy", () => {
259259
error: {
260260
category: "protocol_mismatch",
261261
message:
262-
"openai parseResponse: SSE chunk failed schema validation: choices0.delta.role must be a string (was null)",
262+
"openai parseResponse: SSE chunk failed schema validation: choices[0].delta.role must be a string or null (was a number)",
263263
},
264264
});
265265
expect(decision).toEqual({ kind: "abort" });

‎src/provider/chat-completions-null-delta.test.ts‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
// Regression coverage for the locally patched OpenAI delta schema (see
2+
// vendor/intx-inference/PATCHES.md#providers-ts-openai-null-delta-tolerance).
3+
// vLLM serializers behind OpenAI-compatible relays emit `"role": null` on
4+
// streamed deltas and `"tool_calls": null` on non-tool chunks; the patch
5+
// tolerates them (null ≡ absent) while wrong types stay rejected.
6+
import { describe, expect, test } from "bun:test";
7+
import { ProtocolMismatchError } from "@intx/inference";
8+
import { createOpenAIAdapter } from "@intx/inference/providers";
9+
import type { LastCycleSource } from "@intx/types/runtime";
10+
import { createOpenAICompatibleAdapter } from "./openai-compatible-adapter.js";
11+
import { defined } from "../../testkit/defined.js";
12+
13+
const source: LastCycleSource = {
14+
sourceId: "null-delta-test",
15+
provider: "openai-compatible",
16+
model: "null-delta-fixture",
17+
};
18+
19+
const builtin = createOpenAIAdapter(source);
20+
21+
describe("chat completions null delta tolerance", () => {
22+
test("(a) null role is equivalent to an absent role", () => {
23+
const withNull = builtin.parseResponse(
24+
JSON.stringify({
25+
choices: [{ index: 0, delta: { role: null, content: "hi" } }],
26+
}),
27+
);
28+
const absent = builtin.parseResponse(
29+
JSON.stringify({ choices: [{ index: 0, delta: { content: "hi" } }] }),
30+
);
31+
expect(withNull).toEqual(absent);
32+
const textDelta = withNull.find(
33+
(event) => event.type === "inference.text.delta",
34+
);
35+
expect(
36+
(textDelta as unknown as { data: Record<string, unknown> } | undefined)
37+
?.data,
38+
).toEqual({
39+
token: "hi",
40+
partial: { text: "" },
41+
index: 0,
42+
});
43+
});
44+
45+
test("(b) tool_calls null and empty array are equivalent to absent", () => {
46+
const frames = [{ tool_calls: null }, { tool_calls: [] }, {}] as const;
47+
const shapes = frames.map((delta) =>
48+
builtin.parseResponse(JSON.stringify({ choices: [{ index: 0, delta }] })),
49+
);
50+
expect(shapes[0]).toEqual(shapes[2]);
51+
expect(shapes[1]).toEqual(shapes[2]);
52+
expect(shapes[2]).toEqual([]);
53+
});
54+
55+
test("(c) wrong-type role still throws ProtocolMismatchError", () => {
56+
expect(() =>
57+
builtin.parseResponse(
58+
JSON.stringify({ choices: [{ index: 0, delta: { role: 42 } }] }),
59+
),
60+
).toThrow(ProtocolMismatchError);
61+
});
62+
63+
test("(c) wrong-type tool_calls still throws ProtocolMismatchError", () => {
64+
for (const tool_calls of [42, "nope"]) {
65+
expect(() =>
66+
builtin.parseResponse(
67+
JSON.stringify({ choices: [{ index: 0, delta: { tool_calls } }] }),
68+
),
69+
).toThrow(ProtocolMismatchError);
70+
}
71+
});
72+
73+
test("(d) incident-shape first chunk with null role and tool_calls parses", () => {
74+
const events = builtin
75+
.parseResponse(
76+
JSON.stringify({
77+
choices: [
78+
{
79+
index: 0,
80+
delta: { role: null, tool_calls: null, content: "hello" },
81+
},
82+
],
83+
}),
84+
)
85+
.concat(
86+
builtin.parseResponse(
87+
JSON.stringify({
88+
choices: [
89+
{
90+
index: 0,
91+
delta: { role: null, tool_calls: null, content: " world" },
92+
},
93+
],
94+
}),
95+
),
96+
);
97+
const textDeltas = events.filter(
98+
(event) => event.type === "inference.text.delta",
99+
);
100+
expect(
101+
textDeltas.map((event) => {
102+
const { token, index, partial } = event.data as unknown as {
103+
token: string;
104+
index: number;
105+
partial: { text: string };
106+
};
107+
return { token, index, partial };
108+
}),
109+
).toEqual([
110+
{ token: "hello", partial: { text: "" }, index: 0 },
111+
{ token: " world", partial: { text: "" }, index: 0 },
112+
]);
113+
});
114+
115+
test("(e1) non-streaming null role is equivalent to absent role", () => {
116+
const wireUsage = {
117+
prompt_tokens: 10,
118+
completion_tokens: 5,
119+
};
120+
const withNull = defined(builtin.parseJSONResponse)(
121+
JSON.stringify({
122+
object: "chat.completion",
123+
choices: [{ message: { role: null, content: "done" } }],
124+
usage: wireUsage,
125+
}),
126+
);
127+
const absent = defined(builtin.parseJSONResponse)(
128+
JSON.stringify({
129+
object: "chat.completion",
130+
choices: [{ message: { content: "done" } }],
131+
usage: wireUsage,
132+
}),
133+
);
134+
expect(withNull).toEqual(absent);
135+
expect(
136+
withNull.some((event) => event.type === "inference.text.delta"),
137+
).toBe(true);
138+
expect(withNull.some((event) => event.type === "inference.usage")).toBe(
139+
true,
140+
);
141+
});
142+
143+
test("(e2) non-streaming null tool_calls is equivalent to absent tool_calls", () => {
144+
const wireUsage = {
145+
prompt_tokens: 10,
146+
completion_tokens: 5,
147+
};
148+
const withNull = defined(builtin.parseJSONResponse)(
149+
JSON.stringify({
150+
object: "chat.completion",
151+
choices: [
152+
{ message: { role: null, tool_calls: null, content: "done" } },
153+
],
154+
usage: wireUsage,
155+
}),
156+
);
157+
const absent = defined(builtin.parseJSONResponse)(
158+
JSON.stringify({
159+
object: "chat.completion",
160+
choices: [{ message: { content: "done" } }],
161+
usage: wireUsage,
162+
}),
163+
);
164+
expect(withNull).toEqual(absent);
165+
});
166+
167+
test("(e3) non-streaming wrong-type role still throws ProtocolMismatchError", () => {
168+
expect(() =>
169+
defined(builtin.parseJSONResponse)(
170+
JSON.stringify({
171+
object: "chat.completion",
172+
choices: [{ message: { role: 42, content: "done" } }],
173+
usage: { prompt_tokens: 10, completion_tokens: 5 },
174+
}),
175+
),
176+
).toThrow(ProtocolMismatchError);
177+
});
178+
});
179+
180+
describe("openai-compatible adapter null delta wiring parity", () => {
181+
const wrapperSource = {
182+
id: "test",
183+
provider: "openai-compatible",
184+
baseURL: "https://example.test",
185+
apiKey: "sk-test",
186+
model: "gpt-5.1",
187+
} as unknown as Parameters<typeof createOpenAICompatibleAdapter>[0];
188+
189+
test("(a) null role is equivalent to an absent role", () => {
190+
const adapter = createOpenAICompatibleAdapter(wrapperSource);
191+
expect(
192+
adapter.parseResponse(
193+
JSON.stringify({
194+
choices: [{ index: 0, delta: { role: null, content: "hi" } }],
195+
}),
196+
),
197+
).toEqual(
198+
adapter.parseResponse(
199+
JSON.stringify({ choices: [{ index: 0, delta: { content: "hi" } }] }),
200+
),
201+
);
202+
});
203+
204+
test("(b) tool_calls null is equivalent to absent", () => {
205+
const adapter = createOpenAICompatibleAdapter(wrapperSource);
206+
expect(
207+
adapter.parseResponse(
208+
JSON.stringify({
209+
choices: [{ index: 0, delta: { tool_calls: null } }],
210+
}),
211+
),
212+
).toEqual(
213+
adapter.parseResponse(
214+
JSON.stringify({ choices: [{ index: 0, delta: {} }] }),
215+
),
216+
);
217+
});
218+
219+
test("(c) wrong-type role still throws ProtocolMismatchError", () => {
220+
const adapter = createOpenAICompatibleAdapter(wrapperSource);
221+
expect(() =>
222+
adapter.parseResponse(
223+
JSON.stringify({ choices: [{ index: 0, delta: { role: 42 } }] }),
224+
),
225+
).toThrow(ProtocolMismatchError);
226+
});
227+
});

‎src/provider/openai-compatible-adapter.test.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,12 +109,41 @@ describe("openai-compatible adapter SSE parse count", () => {
109109
});
110110

111111
describe("openai-compatible adapter null delta fields", () => {
112-
test.each(["role", "tool_calls"])("rejects null %s", (field) => {
112+
test.each(["role", "tool_calls"])(
113+
"treats null %s the same as absent",
114+
(field) => {
115+
const adapter = createOpenAICompatibleAdapter(source);
116+
expect(
117+
adapter.parseResponse(
118+
JSON.stringify({
119+
choices: [{ index: 0, delta: { content: "hello", [field]: null } }],
120+
}),
121+
),
122+
).toEqual(
123+
adapter.parseResponse(
124+
JSON.stringify({
125+
choices: [{ index: 0, delta: { content: "hello" } }],
126+
}),
127+
),
128+
);
129+
},
130+
);
131+
132+
test("still rejects wrong-type role", () => {
133+
const adapter = createOpenAICompatibleAdapter(source);
134+
expect(() =>
135+
adapter.parseResponse(
136+
JSON.stringify({ choices: [{ index: 0, delta: { role: 42 } }] }),
137+
),
138+
).toThrow(ProtocolMismatchError);
139+
});
140+
141+
test("still rejects wrong-type tool_calls", () => {
113142
const adapter = createOpenAICompatibleAdapter(source);
114143
expect(() =>
115144
adapter.parseResponse(
116145
JSON.stringify({
117-
choices: [{ index: 0, delta: { content: "hello", [field]: null } }],
146+
choices: [{ index: 0, delta: { tool_calls: "nope" } }],
118147
}),
119148
),
120149
).toThrow(ProtocolMismatchError);

‎vendor/intx-inference/PATCHES.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -780,3 +780,29 @@ retry entirely into `harness.ts`'s `runInference` wrapper, which now emits
780780
`inference.retry` itself before the commitment boundary — see
781781
harness-ts-commitment-boundary-streaming; a reactor-level emission would
782782
double the event).
783+
784+
## providers-ts-openai-null-delta-tolerance
785+
786+
**Promotion candidate.** `providers/openai.ts` tolerates explicit `null` for
787+
`role` and `tool_calls` on Chat Completions streamed deltas and the
788+
non-streaming message (CL-3010: vLLM serializers behind OpenAI-compatible
789+
relays — NEAR AI, Qwen3.8-Flash-Next on vast.ai 2xH200 — emit `"role": null`
790+
on every delta and `"tool_calls": null` on non-tool chunks; the strict schema
791+
rejected each frame, so `protocol_mismatch` aborted the turn). `null` now
792+
means absent, mirroring how sibling(`content`/`refusal`/`reasoning*`) and
793+
nested tool-call (`id`/`function.name`/`function.arguments`) fields already
794+
treat it: `role` widens to `string | null`, `tool_calls` widens to
795+
array-or-null with the streaming parser coalescing null → undefined before
796+
iteration. Wrong-type values stay rejected (`role: 42` still fails schema
797+
validation → `ProtocolMismatchError`, keeping the `protocol_mismatch` abort
798+
path and its rework in `src/agent/retry-policy.test.ts` intact). No consumer
799+
reads `delta.role`; `tool_calls: []` on every chunk (vLLM) was already
800+
tolerated and keeps its semantics.
801+
802+
**Re-carry:** spans `OpenAIToolCallDelta`/`OpenAIChunkDelta`, the streaming
803+
`tool_calls` null-coalesce in `parseResponse`, and the mirrored
804+
`NonStreamingMessage` widening; regression coverage lives in
805+
`src/provider/chat-completions-null-delta.test.ts` (Corbits-side — vendored
806+
OpenAI adapter has no co-located test file at this pin and vendor tests are
807+
outside the repo gate). Drop when upstream opens `delta.role` /
808+
`delta.tool_calls` (and the message schema) to null.

‎vendor/intx-inference/src/providers/openai.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -507,6 +507,12 @@ const EMPTY_PARTIAL: PartialMessage = { text: "" };
507507
// drop the whole chunk silently — taking the argument fragments with
508508
// it. Accept `string | null` here and treat null the same as the field
509509
// being absent at the consumer site.
510+
// Locally patched — see vendor/intx-inference/PATCHES.md#providers-ts-openai-null-delta-tolerance.
511+
// vLLM serializers behind OpenAI-compatible relays emit `role: null` on
512+
// streamed deltas and `tool_calls: null` on non-tool chunks. The strict
513+
// schema rejected each frame, so `protocol_mismatch` aborted the turn.
514+
// `null` means absent here, mirroring the sibling and nested-tool-call
515+
// fields below; wrong-type values stay rejected.
510516
const OpenAIToolCallDelta = type({
511517
"index?": "number",
512518
"id?": "string | null",
@@ -517,7 +523,7 @@ const OpenAIToolCallDelta = type({
517523
});
518524

519525
const OpenAIChunkDelta = type({
520-
"role?": "string",
526+
"role?": "string | null",
521527
"content?": "string | null",
522528
"reasoning_content?": "string | null",
523529
"reasoning?": "string | null",
@@ -527,7 +533,7 @@ const OpenAIChunkDelta = type({
527533
// OpenAI-compatible relays strip it before forwarding; the parser
528534
// emits refusal events only when the field is present.
529535
"refusal?": "string | null",
530-
"tool_calls?": OpenAIToolCallDelta.array(),
536+
"tool_calls?": OpenAIToolCallDelta.array().or("null"),
531537
});
532538

533539
const PromptTokensDetails = type({ "cached_tokens?": "number" }).or("null");
@@ -758,7 +764,10 @@ function parseResponse(
758764
});
759765
}
760766

761-
const { tool_calls: toolCallDeltas } = delta;
767+
// Normalize null → undefined (same idiom as below for function fields):
768+
// the schema tolerates `tool_calls: null`, but a bare widened schema
769+
// would hand the null to `for...of` and crash with a TypeError.
770+
const toolCallDeltas = delta.tool_calls ?? undefined;
762771

763772
if (toolCallDeltas !== undefined) {
764773
for (const tcDelta of toolCallDeltas) {
@@ -881,12 +890,12 @@ const NonStreamingToolCall = type({
881890
});
882891

883892
const NonStreamingMessage = type({
884-
"role?": "string",
893+
"role?": "string | null",
885894
"content?": "string | null",
886895
"reasoning_content?": "string | null",
887896
"reasoning?": "string | null",
888897
"refusal?": "string | null",
889-
"tool_calls?": NonStreamingToolCall.array(),
898+
"tool_calls?": NonStreamingToolCall.array().or("null"),
890899
});
891900

892901
const NonStreamingCompletion = type({

0 commit comments

Comments
 (0)