From e60d3b245ba23a8ed872eda312457e1eb9bd36bf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 13:48:46 +0000 Subject: [PATCH] feat(ai-sdk): serialize tool outputs like OpenAPI responses and send files to the model A default `toModelOutput` serializes outputs like OpenAPI responses, so types such as `Date`, `BigInt`, and `Set` reach the model as JSON, and sends `Blob`/`File` outputs, nested or not, as file content named by their path, such as `report/q3.pdf`. The new `serializer` option shares custom types with an OpenAPI handler, and `toModelOutput` in tool options or `aiSdkTool` meta still takes over. The TODO about a JSON Schema `converter` option is removed: tools keep leaving JSON Schema conversion to the AI SDK. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BsvL9tattvbASxVptTw9kS --- apps/content/docs/integrations/ai-sdk.mdx | 4 + packages/ai-sdk/package.json | 1 + packages/ai-sdk/src/tool.test.ts | 105 ++++++++++++++++++---- packages/ai-sdk/src/tool.ts | 54 +++++++++-- packages/ai-sdk/tsconfig.json | 1 + pnpm-lock.yaml | 3 + 6 files changed, 146 insertions(+), 22 deletions(-) diff --git a/apps/content/docs/integrations/ai-sdk.mdx b/apps/content/docs/integrations/ai-sdk.mdx index a4c61ea07e..c0c8c55a99 100644 --- a/apps/content/docs/integrations/ai-sdk.mdx +++ b/apps/content/docs/integrations/ai-sdk.mdx @@ -206,3 +206,7 @@ const deployProcedure = os const deployTool = createTool(deployProcedure) ``` + +## Tool Outputs + +The model receives outputs serialized like [OpenAPI responses](/docs/openapi/serializer), and each `Blob` or `File`, nested or not, as file content named by its path, such as `report/q3.pdf`. Pass your OpenAPI handler's serializer as the factory's `serializer` option to share [custom types](/docs/openapi/serializer#custom-serializers), or set `toModelOutput` to take over. diff --git a/packages/ai-sdk/package.json b/packages/ai-sdk/package.json index 43b0977a36..eade8ddf2a 100644 --- a/packages/ai-sdk/package.json +++ b/packages/ai-sdk/package.json @@ -53,6 +53,7 @@ "@orpc/client": "workspace:*", "@orpc/contract": "workspace:*", "@orpc/json-schema": "workspace:*", + "@orpc/openapi": "workspace:*", "@orpc/server": "workspace:*", "@orpc/shared": "workspace:*" }, diff --git a/packages/ai-sdk/src/tool.test.ts b/packages/ai-sdk/src/tool.test.ts index b0ffcb5e50..4cccafc77a 100644 --- a/packages/ai-sdk/src/tool.test.ts +++ b/packages/ai-sdk/src/tool.test.ts @@ -1,4 +1,5 @@ import { asyncIteratorObject, oc, type } from '@orpc/contract' +import { OpenAPIJsonSerializer } from '@orpc/openapi' import { os } from '@orpc/server' import { generateText } from 'ai' import { MockLanguageModelV4 } from 'ai/test' @@ -6,6 +7,20 @@ import z from 'zod' import { createToolFactory, implementToolFactory } from './tool' import { aiSdkTool } from './tool-meta' +function toolCallModel(toolName: string, args: unknown) { + return new MockLanguageModelV4({ + doGenerate: async () => ({ + content: [{ type: 'tool-call', toolCallId: 'call-1', toolName, input: JSON.stringify(args) }], + finishReason: { unified: 'tool-calls', raw: undefined }, + usage: { + inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 20, text: 20, reasoning: undefined }, + }, + warnings: [], + }), + }) +} + describe('implementToolFactory', () => { const inputSchema = z.object({ name: z.string().describe('Name of the person'), @@ -260,22 +275,7 @@ describe('implementToolFactory', () => { }) const result = await generateText({ - model: new MockLanguageModelV4({ - doGenerate: async () => ({ - content: [{ - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'greet', - input: JSON.stringify({ name: 'Alice' }), - }], - finishReason: { unified: 'tool-calls', raw: undefined }, - usage: { - inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined }, - outputTokens: { total: 20, text: 20, reasoning: undefined }, - }, - warnings: [], - }), - }), + model: toolCallModel('greet', { name: 'Alice' }), tools: { greet }, prompt: 'Greet Alice', }) @@ -299,6 +299,79 @@ describe('implementToolFactory', () => { ).toBe(yieldSchema) }) }) + + describe('model output', () => { + const toModelOutput = (tool: any, output: unknown) => tool.toModelOutput({ toolCallId: 'call-1', input: {}, output }) + const pdfBase64 = btoa('pdf') + + it('serializes outputs like OpenAPI responses', async () => { + const tool = implementToolFactory()(oc) + + await expect(toModelOutput(tool, { date: new Date('2024-01-01T00:00:00.000Z'), big: 1n, set: new Set([1, 2]) })).resolves.toEqual({ + type: 'json', + value: { date: '2024-01-01T00:00:00.000Z', big: '1', set: [1, 2] }, + }) + await expect(toModelOutput(tool, 'hello')).resolves.toEqual({ type: 'text', value: 'hello' }) + await expect(toModelOutput(tool, undefined)).resolves.toEqual({ type: 'json', value: null }) + }) + + it('sends files as file content, nested ones named by their path', async () => { + const tool = implementToolFactory()(oc) + const pdf = new File(['pdf'], 'q3.pdf', { type: 'application/pdf' }) + + await expect(toModelOutput(tool, pdf)).resolves.toEqual({ + type: 'content', + value: [{ type: 'file', mediaType: 'application/pdf', data: { type: 'data', data: pdfBase64 }, filename: 'q3.pdf' }], + }) + + await expect(toModelOutput(tool, new Blob(['bytes']))).resolves.toEqual({ + type: 'content', + value: [{ type: 'file', mediaType: 'application/octet-stream', data: { type: 'data', data: btoa('bytes') } }], + }) + + await expect(toModelOutput(tool, { title: 'Q3', files: [pdf] })).resolves.toEqual({ + type: 'content', + value: [ + { type: 'text', text: '{"title":"Q3","files":[{}]}' }, + { type: 'file', mediaType: 'application/pdf', data: { type: 'data', data: pdfBase64 }, filename: 'files/0/q3.pdf' }, + ], + }) + }) + + it('uses the serializer option', async () => { + class Money { + constructor(readonly cents: number) {} + } + + const serializer = new OpenAPIJsonSerializer({ + handlers: { money: { condition: data => data instanceof Money, serialize: (data: Money) => `$${data.cents / 100}`, isTerminal: true } }, + }) + + await expect(toModelOutput(implementToolFactory({ serializer })(oc), { price: new Money(150) })).resolves.toEqual({ + type: 'json', + value: { price: '$1.5' }, + }) + }) + + it('can be overridden by tool options or meta', () => { + const custom = () => ({ type: 'text' as const, value: 'custom' }) + + expect(implementToolFactory()(oc, { toModelOutput: custom }).toModelOutput).toBe(custom) + expect(implementToolFactory()(oc.meta(aiSdkTool({ toModelOutput: custom }))).toModelOutput).toBe(custom) + }) + + it('sends files to the model through the AI SDK', async () => { + const report = implementToolFactory()(oc, { + execute: async () => new File(['pdf'], 'q3.pdf', { type: 'application/pdf' }), + }) + + const result = await generateText({ model: toolCallModel('report', {}), tools: { report }, prompt: 'Report' }) + + expect(result.response.messages.at(-1)?.content).toEqual([expect.objectContaining({ + output: { type: 'content', value: [{ type: 'file', mediaType: 'application/pdf', data: { type: 'data', data: pdfBase64 }, filename: 'q3.pdf' }] }, + })]) + }) + }) }) describe('createToolFactory', () => { diff --git a/packages/ai-sdk/src/tool.ts b/packages/ai-sdk/src/tool.ts index 6fdee969fb..c8e3820b46 100644 --- a/packages/ai-sdk/src/tool.ts +++ b/packages/ai-sdk/src/tool.ts @@ -2,13 +2,14 @@ import type { ClientOptions } from '@orpc/client' import type { AnySchema, InferSchemaInput, InferSchemaOutput, ProcedureContract, Schema } from '@orpc/contract' import type { JsonSchema } from '@orpc/json-schema' import type { Context, ProcedureClientOptions } from '@orpc/server' -import type { MaybeOptionalOptions } from '@orpc/shared' -import type { FlexibleSchema, Tool } from 'ai' +import type { MaybeOptionalOptions, Public } from '@orpc/shared' +import type { FlexibleSchema, JSONValue, Tool, ToolResultPart } from 'ai' import type { FunctionTool } from './tool-meta' import { getAsyncIteratorObjectSchemasDetails } from '@orpc/contract' import { combineJsonSchemasWithComposition } from '@orpc/json-schema' +import { OpenAPIJsonSerializer } from '@orpc/openapi' import { call, Procedure } from '@orpc/server' -import { isAsyncGeneratorFunction, mergeTwoLevels, ORPC_NAME, resolveMaybeOptionalOptions, toArray } from '@orpc/shared' +import { encodeBase64, isAsyncGeneratorFunction, loadBytes, mergeTwoLevels, ORPC_NAME, resolveMaybeOptionalOptions, stringifyJSON, toArray } from '@orpc/shared' import { tool } from 'ai' import { getAiSdkToolMeta } from './tool-meta' @@ -94,6 +95,37 @@ function combineOutputSchemas(outputSchemas: AnySchema[]): FlexibleSchema | unde return combineSchemas([...(yieldSchemas ?? outputSchemas)].reverse(), false) } +/** + * Serializes an output like an OpenAPI response body. JSON shows a `Blob` as an empty object, + * so blobs, nested or not, are sent as file parts named by their path, such as `report/q3.pdf`. + */ +async function toModelOutput(serializer: Public, output: unknown): Promise { + const { json, maps, blobs } = serializer.serializeJson(output) + + if (typeof json === 'string') { + return { type: 'text', value: json } + } + + if (!blobs?.length) { + return { type: 'json', value: (json ?? null) as JSONValue } + } + + const files = await Promise.all(blobs.map(async (blob, i) => { + const path = maps[i]! + const names = blob instanceof File && blob.name !== '' ? [...path, blob.name] : path + + return { + type: 'file' as const, + mediaType: blob.type || 'application/octet-stream', + // base64 is encoded once, providers would otherwise re-encode the bytes on every step + data: { type: 'data' as const, data: encodeBase64(await loadBytes(blob)) }, + ...(names.length === 0 ? {} : { filename: names.join('/') }), + } + })) + + return { type: 'content', value: json instanceof Blob ? files : [{ type: 'text', text: stringifyJSON(json)! }, ...files] } +} + /** * Infers the value an AI SDK tool outputs for a given oRPC output type. * @@ -108,8 +140,14 @@ export type ToolOutput = T extends AsyncIteratorObject | undefined } export interface ToolImplementer { @@ -162,7 +200,10 @@ export interface ToolImplementer { * * @see {@link https://orpc.dev/docs/integrations/ai-sdk#tool-implementer | AI SDK Integration - Tool Implementer} */ -export function implementToolFactory(_options: ImplementToolFactoryOptions = {}): ToolImplementer { +export function implementToolFactory(options: ImplementToolFactoryOptions = {}): ToolImplementer { + const serializer = options.serializer ?? new OpenAPIJsonSerializer() + const defaultToModelOutput: NonNullable = ({ output }) => toModelOutput(serializer, output) + const factory: ToolImplementer = (contract, ...rest) => { const toolOptions = resolveMaybeOptionalOptions(rest) @@ -174,6 +215,7 @@ export function implementToolFactory(_options: ImplementToolFactoryOptions = {}) return tool({ ...defaults, ...toolOptions, + toModelOutput: toolOptions.toModelOutput ?? defaults?.toModelOutput ?? defaultToModelOutput, inputSchema: combineSchemas(inputSchemas, true) ?? ANY_SCHEMA, outputSchema: combineOutputSchemas(outputSchemas), }) diff --git a/packages/ai-sdk/tsconfig.json b/packages/ai-sdk/tsconfig.json index e8c164f3c2..7a3f25403a 100644 --- a/packages/ai-sdk/tsconfig.json +++ b/packages/ai-sdk/tsconfig.json @@ -4,6 +4,7 @@ { "path": "../client" }, { "path": "../contract" }, { "path": "../json-schema" }, + { "path": "../openapi" }, { "path": "../server" }, { "path": "../shared" } ], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eabc0dc28e..f145a39415 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -451,6 +451,9 @@ importers: '@orpc/json-schema': specifier: workspace:* version: link:../json-schema + '@orpc/openapi': + specifier: workspace:* + version: link:../openapi '@orpc/server': specifier: workspace:* version: link:../server