From d327fc6d3f83305f8f69dbcbfc19e8135479edd3 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 3 Nov 2026 20:27:59 -0800 Subject: [PATCH 1/4] fix: send text document attachments as input text --- README.md | 2 +- src/protocol/body.ts | 31 ++++++++++++- src/responses.test.ts | 101 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 18f31fa..f96c543 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # @corbits/openai-responses -An `@intx/inference` provider adapter for the OpenAI Responses API (`/v1/responses`): SSE and JSON responses, text, tool calls, image and PDF input, and reasoning replay. An inference provider for Corbits and Interchange agents that also works in any host that runs `@intx/inference`. +An `@intx/inference` provider adapter for the OpenAI Responses API (`/v1/responses`): SSE and JSON responses, text, tool calls, image, PDF and text document input, and reasoning replay. An inference provider for Corbits and Interchange agents that also works in any host that runs `@intx/inference`. ## Why @corbits/openai-responses? diff --git a/src/protocol/body.ts b/src/protocol/body.ts index 5228c79..033546b 100644 --- a/src/protocol/body.ts +++ b/src/protocol/body.ts @@ -54,17 +54,38 @@ function toolResultText( return parts.join(""); } +// Interchange's attachment allowlist counts these as documents; input_file +// rejects them, so their decoded text is sent inline instead. +const TEXT_DOCUMENT_MIME_TYPES: ReadonlySet = new Set([ + "text/plain", + "text/csv", + "text/markdown", + "application/json", +]); + // Grounded on the OpenAI Responses API document input shape: a base64 // document synthesizes a deterministic filename from its mimeType, since // MediaSource carries no filename field of its own. Only PDF is supported -// today, matching the Chat Completions adapter's own document support. +// as a file, matching the Chat Completions adapter's own document support. function filenameForDocumentMime(mimeType: string): string { if (mimeType === "application/pdf") return "document.pdf"; throw new Error( - `@corbits/openai-responses: document input currently supports application/pdf only; received mimeType: ${mimeType}`, + `@corbits/openai-responses: document input supports application/pdf and ${[...TEXT_DOCUMENT_MIME_TYPES].join(", ")} only; received mimeType: ${mimeType}`, ); } +function decodeTextDocument(data: string, mimeType: string): string { + try { + const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0)); + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (cause) { + throw new Error( + `@corbits/openai-responses: ${mimeType} document is not valid base64-encoded UTF-8 text`, + { cause }, + ); + } +} + // Converts one content block to zero or one Responses content part. Blocks // handled as whole `ResponsesInputItem`s elsewhere (tool_call, tool_result, // signed thinking) are not passed here. @@ -97,6 +118,12 @@ function toResponsesContentPart( case "document": { const source = block.source; if (source.kind === "base64") { + if (TEXT_DOCUMENT_MIME_TYPES.has(source.mimeType)) { + return { + type: textKind, + text: decodeTextDocument(source.data, source.mimeType), + }; + } return { type: "input_file", filename: filenameForDocumentMime(source.mimeType), diff --git a/src/responses.test.ts b/src/responses.test.ts index 52f90ca..3a93cfa 100644 --- a/src/responses.test.ts +++ b/src/responses.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { ProtocolMismatchError } from "@intx/inference"; import type { + ContentBlock, ConversationTurn, InferenceEvent, LastCycleSource, @@ -273,6 +274,106 @@ describe("Responses request builder — quirk matrix", () => { }); }); +describe("Responses request builder — media input", () => { + const adapter = createOpenAIResponsesAdapter(source, {}); + const contentOf = (block: ContentBlock): unknown => + inputItemsOf( + adapter.buildRequest( + [{ role: "user", timestamp: 0, content: [block] }], + "model", + {}, + ), + )[0]?.["content"]; + + test("a base64 PDF goes to input_file with file_data", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "application/pdf", data: "JVBERi0=" }, + }; + expect(contentOf(block)).toEqual([ + { + type: "input_file", + filename: "document.pdf", + file_data: "data:application/pdf;base64,JVBERi0=", + }, + ]); + }); + + test.each([ + { mimeType: "text/plain", text: "héllo" }, + { mimeType: "text/csv", text: "a,b\n1,2" }, + { mimeType: "text/markdown", text: "# Plan" }, + { mimeType: "application/json", text: '{"a":1}' }, + ])("a base64 $mimeType document goes to input_text", ({ mimeType, text }) => { + const data = Buffer.from(text, "utf-8").toString("base64"); + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType, data }, + }; + expect(contentOf(block)).toEqual([{ type: "input_text", text }]); + }); + + test("a text document that is not valid UTF-8 throws instead of sending garbage", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "text/plain", data: "/w==" }, + }; + expect(() => contentOf(block)).toThrow( + "text/plain document is not valid base64-encoded UTF-8 text", + ); + }); + + test("a base64 document of another binary type still throws", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "application/zip", data: "UEs=" }, + }; + expect(() => contentOf(block)).toThrow( + "received mimeType: application/zip", + ); + }); + + test.each<{ name: string; block: ContentBlock; expected: unknown }>([ + { + name: "base64", + block: { + type: "image", + source: { kind: "base64", mimeType: "image/png", data: "iVBORw==" }, + }, + expected: { + type: "input_image", + image_url: "data:image/png;base64,iVBORw==", + }, + }, + { + name: "url", + block: { + type: "image", + source: { + kind: "url", + mimeType: "image/png", + url: "https://example.com/a.png", + }, + }, + expected: { type: "input_image", image_url: "https://example.com/a.png" }, + }, + { + name: "file-reference", + block: { + type: "image", + source: { + kind: "file-reference", + mimeType: "image/png", + reference: "file-abc", + }, + }, + expected: { type: "input_image", file_id: "file-abc" }, + }, + ])("a $name image goes to input_image", ({ block, expected }) => { + expect(contentOf(block)).toEqual([expected]); + }); +}); + describe("Responses parser — event schema validation", () => { test("a function_call output_item.added missing id throws rather than silently dropping the start", () => { const adapter = createOpenAIResponsesAdapter(source, {}); From 78f720656ce315c4c6ba74f145cf7a1bfda774ce Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 3 Nov 2026 20:30:46 -0800 Subject: [PATCH 2/4] fix: classify text documents with interchange's text-like mime helper --- src/protocol/body.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/protocol/body.ts b/src/protocol/body.ts index 033546b..2b064ea 100644 --- a/src/protocol/body.ts +++ b/src/protocol/body.ts @@ -4,6 +4,8 @@ import { type BuiltRequest, type ToolNameLimit, } from "@intx/inference"; +import { isTextLikeMimeType } from "@intx/types/attachments"; +import { base64Decode } from "@intx/types/base64"; import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ContentBlock, @@ -54,15 +56,6 @@ function toolResultText( return parts.join(""); } -// Interchange's attachment allowlist counts these as documents; input_file -// rejects them, so their decoded text is sent inline instead. -const TEXT_DOCUMENT_MIME_TYPES: ReadonlySet = new Set([ - "text/plain", - "text/csv", - "text/markdown", - "application/json", -]); - // Grounded on the OpenAI Responses API document input shape: a base64 // document synthesizes a deterministic filename from its mimeType, since // MediaSource carries no filename field of its own. Only PDF is supported @@ -70,14 +63,13 @@ const TEXT_DOCUMENT_MIME_TYPES: ReadonlySet = new Set([ function filenameForDocumentMime(mimeType: string): string { if (mimeType === "application/pdf") return "document.pdf"; throw new Error( - `@corbits/openai-responses: document input supports application/pdf and ${[...TEXT_DOCUMENT_MIME_TYPES].join(", ")} only; received mimeType: ${mimeType}`, + `@corbits/openai-responses: document input supports application/pdf and text types only; received mimeType: ${mimeType}`, ); } function decodeTextDocument(data: string, mimeType: string): string { try { - const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0)); - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + return new TextDecoder("utf-8", { fatal: true }).decode(base64Decode(data)); } catch (cause) { throw new Error( `@corbits/openai-responses: ${mimeType} document is not valid base64-encoded UTF-8 text`, @@ -118,7 +110,8 @@ function toResponsesContentPart( case "document": { const source = block.source; if (source.kind === "base64") { - if (TEXT_DOCUMENT_MIME_TYPES.has(source.mimeType)) { + // input_file rejects text types, so their decoded text goes inline. + if (isTextLikeMimeType(source.mimeType)) { return { type: textKind, text: decodeTextDocument(source.data, source.mimeType), From ed4f5d13d94304cbcdab954bef0812605c188305 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 3 Nov 2026 20:30:46 -0800 Subject: [PATCH 3/4] test: cover mime parameters and flat shape for text documents --- src/responses.test.ts | 88 +++++++++++-------------------------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/src/responses.test.ts b/src/responses.test.ts index 3a93cfa..e394ca8 100644 --- a/src/responses.test.ts +++ b/src/responses.test.ts @@ -274,43 +274,41 @@ describe("Responses request builder — quirk matrix", () => { }); }); -describe("Responses request builder — media input", () => { - const adapter = createOpenAIResponsesAdapter(source, {}); - const contentOf = (block: ContentBlock): unknown => +describe("Responses request builder — document input", () => { + const contentOf = ( + block: ContentBlock, + quirks: Record = {}, + ): unknown => inputItemsOf( - adapter.buildRequest( + createOpenAIResponsesAdapter(source, quirks).buildRequest( [{ role: "user", timestamp: 0, content: [block] }], "model", {}, ), )[0]?.["content"]; - - test("a base64 PDF goes to input_file with file_data", () => { - const block: ContentBlock = { - type: "document", - source: { kind: "base64", mimeType: "application/pdf", data: "JVBERi0=" }, - }; - expect(contentOf(block)).toEqual([ - { - type: "input_file", - filename: "document.pdf", - file_data: "data:application/pdf;base64,JVBERi0=", - }, - ]); + const textDocument = (mimeType: string, text: string): ContentBlock => ({ + type: "document", + source: { + kind: "base64", + mimeType, + data: Buffer.from(text, "utf-8").toString("base64"), + }, }); test.each([ { mimeType: "text/plain", text: "héllo" }, - { mimeType: "text/csv", text: "a,b\n1,2" }, - { mimeType: "text/markdown", text: "# Plan" }, + { mimeType: "text/csv; charset=utf-8", text: "a,b\n1,2" }, { mimeType: "application/json", text: '{"a":1}' }, ])("a base64 $mimeType document goes to input_text", ({ mimeType, text }) => { - const data = Buffer.from(text, "utf-8").toString("base64"); - const block: ContentBlock = { - type: "document", - source: { kind: "base64", mimeType, data }, - }; - expect(contentOf(block)).toEqual([{ type: "input_text", text }]); + expect(contentOf(textDocument(mimeType, text))).toEqual([ + { type: "input_text", text }, + ]); + }); + + test("flat contentShape keeps a text document as string content", () => { + expect( + contentOf(textDocument("text/plain", "héllo"), { contentShape: "flat" }), + ).toBe("héllo"); }); test("a text document that is not valid UTF-8 throws instead of sending garbage", () => { @@ -332,46 +330,6 @@ describe("Responses request builder — media input", () => { "received mimeType: application/zip", ); }); - - test.each<{ name: string; block: ContentBlock; expected: unknown }>([ - { - name: "base64", - block: { - type: "image", - source: { kind: "base64", mimeType: "image/png", data: "iVBORw==" }, - }, - expected: { - type: "input_image", - image_url: "data:image/png;base64,iVBORw==", - }, - }, - { - name: "url", - block: { - type: "image", - source: { - kind: "url", - mimeType: "image/png", - url: "https://example.com/a.png", - }, - }, - expected: { type: "input_image", image_url: "https://example.com/a.png" }, - }, - { - name: "file-reference", - block: { - type: "image", - source: { - kind: "file-reference", - mimeType: "image/png", - reference: "file-abc", - }, - }, - expected: { type: "input_image", file_id: "file-abc" }, - }, - ])("a $name image goes to input_image", ({ block, expected }) => { - expect(contentOf(block)).toEqual([expected]); - }); }); describe("Responses parser — event schema validation", () => { From f71e109f09294fa6b3ca555d55d4a6a23e94bf4e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 3 Nov 2026 20:31:35 -0800 Subject: [PATCH 4/4] test: restore pdf and image input encoding tests --- src/responses.test.ts | 56 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/responses.test.ts b/src/responses.test.ts index e394ca8..8720118 100644 --- a/src/responses.test.ts +++ b/src/responses.test.ts @@ -274,7 +274,7 @@ describe("Responses request builder — quirk matrix", () => { }); }); -describe("Responses request builder — document input", () => { +describe("Responses request builder — media input", () => { const contentOf = ( block: ContentBlock, quirks: Record = {}, @@ -330,6 +330,60 @@ describe("Responses request builder — document input", () => { "received mimeType: application/zip", ); }); + + test("a base64 PDF goes to input_file with file_data", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "application/pdf", data: "JVBERi0=" }, + }; + expect(contentOf(block)).toEqual([ + { + type: "input_file", + filename: "document.pdf", + file_data: "data:application/pdf;base64,JVBERi0=", + }, + ]); + }); + + test.each<{ name: string; block: ContentBlock; expected: unknown }>([ + { + name: "base64", + block: { + type: "image", + source: { kind: "base64", mimeType: "image/png", data: "iVBORw==" }, + }, + expected: { + type: "input_image", + image_url: "data:image/png;base64,iVBORw==", + }, + }, + { + name: "url", + block: { + type: "image", + source: { + kind: "url", + mimeType: "image/png", + url: "https://example.com/a.png", + }, + }, + expected: { type: "input_image", image_url: "https://example.com/a.png" }, + }, + { + name: "file-reference", + block: { + type: "image", + source: { + kind: "file-reference", + mimeType: "image/png", + reference: "file-abc", + }, + }, + expected: { type: "input_image", file_id: "file-abc" }, + }, + ])("a $name image goes to input_image", ({ block, expected }) => { + expect(contentOf(block)).toEqual([expected]); + }); }); describe("Responses parser — event schema validation", () => {