diff --git a/README.md b/README.md index 18f31fa..f96c543 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # @corbits/openai-responses -An `@intx/inference` provider adapter for the OpenAI Responses API (`/v1/responses`): SSE and JSON responses, text, tool calls, image and PDF input, and reasoning replay. An inference provider for Corbits and Interchange agents that also works in any host that runs `@intx/inference`. +An `@intx/inference` provider adapter for the OpenAI Responses API (`/v1/responses`): SSE and JSON responses, text, tool calls, image, PDF and text document input, and reasoning replay. An inference provider for Corbits and Interchange agents that also works in any host that runs `@intx/inference`. ## Why @corbits/openai-responses? diff --git a/src/protocol/body.ts b/src/protocol/body.ts index 5228c79..2b064ea 100644 --- a/src/protocol/body.ts +++ b/src/protocol/body.ts @@ -4,6 +4,8 @@ import { type BuiltRequest, type ToolNameLimit, } from "@intx/inference"; +import { isTextLikeMimeType } from "@intx/types/attachments"; +import { base64Decode } from "@intx/types/base64"; import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ContentBlock, @@ -57,14 +59,25 @@ function toolResultText( // Grounded on the OpenAI Responses API document input shape: a base64 // document synthesizes a deterministic filename from its mimeType, since // MediaSource carries no filename field of its own. Only PDF is supported -// today, matching the Chat Completions adapter's own document support. +// as a file, matching the Chat Completions adapter's own document support. function filenameForDocumentMime(mimeType: string): string { if (mimeType === "application/pdf") return "document.pdf"; throw new Error( - `@corbits/openai-responses: document input currently supports application/pdf only; received mimeType: ${mimeType}`, + `@corbits/openai-responses: document input supports application/pdf and text types only; received mimeType: ${mimeType}`, ); } +function decodeTextDocument(data: string, mimeType: string): string { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(base64Decode(data)); + } catch (cause) { + throw new Error( + `@corbits/openai-responses: ${mimeType} document is not valid base64-encoded UTF-8 text`, + { cause }, + ); + } +} + // Converts one content block to zero or one Responses content part. Blocks // handled as whole `ResponsesInputItem`s elsewhere (tool_call, tool_result, // signed thinking) are not passed here. @@ -97,6 +110,13 @@ function toResponsesContentPart( case "document": { const source = block.source; if (source.kind === "base64") { + // input_file rejects text types, so their decoded text goes inline. + if (isTextLikeMimeType(source.mimeType)) { + return { + type: textKind, + text: decodeTextDocument(source.data, source.mimeType), + }; + } return { type: "input_file", filename: filenameForDocumentMime(source.mimeType), diff --git a/src/responses.test.ts b/src/responses.test.ts index 52f90ca..8720118 100644 --- a/src/responses.test.ts +++ b/src/responses.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { ProtocolMismatchError } from "@intx/inference"; import type { + ContentBlock, ConversationTurn, InferenceEvent, LastCycleSource, @@ -273,6 +274,118 @@ describe("Responses request builder — quirk matrix", () => { }); }); +describe("Responses request builder — media input", () => { + const contentOf = ( + block: ContentBlock, + quirks: Record = {}, + ): unknown => + inputItemsOf( + createOpenAIResponsesAdapter(source, quirks).buildRequest( + [{ role: "user", timestamp: 0, content: [block] }], + "model", + {}, + ), + )[0]?.["content"]; + const textDocument = (mimeType: string, text: string): ContentBlock => ({ + type: "document", + source: { + kind: "base64", + mimeType, + data: Buffer.from(text, "utf-8").toString("base64"), + }, + }); + + test.each([ + { mimeType: "text/plain", text: "héllo" }, + { mimeType: "text/csv; charset=utf-8", text: "a,b\n1,2" }, + { mimeType: "application/json", text: '{"a":1}' }, + ])("a base64 $mimeType document goes to input_text", ({ mimeType, text }) => { + expect(contentOf(textDocument(mimeType, text))).toEqual([ + { type: "input_text", text }, + ]); + }); + + test("flat contentShape keeps a text document as string content", () => { + expect( + contentOf(textDocument("text/plain", "héllo"), { contentShape: "flat" }), + ).toBe("héllo"); + }); + + test("a text document that is not valid UTF-8 throws instead of sending garbage", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "text/plain", data: "/w==" }, + }; + expect(() => contentOf(block)).toThrow( + "text/plain document is not valid base64-encoded UTF-8 text", + ); + }); + + test("a base64 document of another binary type still throws", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "application/zip", data: "UEs=" }, + }; + expect(() => contentOf(block)).toThrow( + "received mimeType: application/zip", + ); + }); + + test("a base64 PDF goes to input_file with file_data", () => { + const block: ContentBlock = { + type: "document", + source: { kind: "base64", mimeType: "application/pdf", data: "JVBERi0=" }, + }; + expect(contentOf(block)).toEqual([ + { + type: "input_file", + filename: "document.pdf", + file_data: "data:application/pdf;base64,JVBERi0=", + }, + ]); + }); + + test.each<{ name: string; block: ContentBlock; expected: unknown }>([ + { + name: "base64", + block: { + type: "image", + source: { kind: "base64", mimeType: "image/png", data: "iVBORw==" }, + }, + expected: { + type: "input_image", + image_url: "data:image/png;base64,iVBORw==", + }, + }, + { + name: "url", + block: { + type: "image", + source: { + kind: "url", + mimeType: "image/png", + url: "https://example.com/a.png", + }, + }, + expected: { type: "input_image", image_url: "https://example.com/a.png" }, + }, + { + name: "file-reference", + block: { + type: "image", + source: { + kind: "file-reference", + mimeType: "image/png", + reference: "file-abc", + }, + }, + expected: { type: "input_image", file_id: "file-abc" }, + }, + ])("a $name image goes to input_image", ({ block, expected }) => { + expect(contentOf(block)).toEqual([expected]); + }); +}); + describe("Responses parser — event schema validation", () => { test("a function_call output_item.added missing id throws rather than silently dropping the start", () => { const adapter = createOpenAIResponsesAdapter(source, {});