feat(ai-sdk): serialize tool outputs like OpenAPI responses and send files to the model

apps/content/docs/integrations/ai-sdk.mdx

@@ -206,3 +206,7 @@ const deployProcedure = os
 
 const deployTool = createTool(deployProcedure)
 ```
+
+## Tool Outputs
+
+The model receives outputs serialized like [OpenAPI responses](/docs/openapi/serializer), and each `Blob` or `File`, nested or not, as file content named by its path, such as `report/q3.pdf`. Pass your OpenAPI handler's serializer as the factory's `serializer` option to share [custom types](/docs/openapi/serializer#custom-serializers), or set `toModelOutput` to take over.

packages/ai-sdk/package.json

@@ -53,6 +53,7 @@
     "@orpc/client": "workspace:*",
     "@orpc/contract": "workspace:*",
     "@orpc/json-schema": "workspace:*",
+    "@orpc/openapi": "workspace:*",
     "@orpc/server": "workspace:*",
     "@orpc/shared": "workspace:*"
   },

packages/ai-sdk/src/tool.test.ts

@@ -1,11 +1,26 @@
 import { asyncIteratorObject, oc, type } from '@orpc/contract'
+import { OpenAPIJsonSerializer } from '@orpc/openapi'
 import { os } from '@orpc/server'
 import { generateText } from 'ai'
 import { MockLanguageModelV4 } from 'ai/test'
 import z from 'zod'
 import { createToolFactory, implementToolFactory } from './tool'
 import { aiSdkTool } from './tool-meta'
 
+function toolCallModel(toolName: string, args: unknown) {
+  return new MockLanguageModelV4({
+    doGenerate: async () => ({
+      content: [{ type: 'tool-call', toolCallId: 'call-1', toolName, input: JSON.stringify(args) }],
+      finishReason: { unified: 'tool-calls', raw: undefined },
+      usage: {
+        inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined },
+        outputTokens: { total: 20, text: 20, reasoning: undefined },
+      },
+      warnings: [],
+    }),
+  })
+}
+
 describe('implementToolFactory', () => {
   const inputSchema = z.object({
     name: z.string().describe('Name of the person'),
@@ -260,22 +275,7 @@ describe('implementToolFactory', () => {
     })
 
     const result = await generateText({
-      model: new MockLanguageModelV4({
-        doGenerate: async () => ({
-          content: [{
-            type: 'tool-call',
-            toolCallId: 'call-1',
-            toolName: 'greet',
-            input: JSON.stringify({ name: 'Alice' }),
-          }],
-          finishReason: { unified: 'tool-calls', raw: undefined },
-          usage: {
-            inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined },
-            outputTokens: { total: 20, text: 20, reasoning: undefined },
-          },
-          warnings: [],
-        }),
-      }),
+      model: toolCallModel('greet', { name: 'Alice' }),
       tools: { greet },
       prompt: 'Greet Alice',
     })
@@ -299,6 +299,79 @@ describe('implementToolFactory', () => {
       ).toBe(yieldSchema)
     })
   })
+
+  describe('model output', () => {
+    const toModelOutput = (tool: any, output: unknown) => tool.toModelOutput({ toolCallId: 'call-1', input: {}, output })
+    const pdfBase64 = btoa('pdf')
+
+    it('serializes outputs like OpenAPI responses', async () => {
+      const tool = implementToolFactory()(oc)
+
+      await expect(toModelOutput(tool, { date: new Date('2024-01-01T00:00:00.000Z'), big: 1n, set: new Set([1, 2]) })).resolves.toEqual({
+        type: 'json',
+        value: { date: '2024-01-01T00:00:00.000Z', big: '1', set: [1, 2] },
+      })
+      await expect(toModelOutput(tool, 'hello')).resolves.toEqual({ type: 'text', value: 'hello' })
+      await expect(toModelOutput(tool, undefined)).resolves.toEqual({ type: 'json', value: null })
+    })
+
+    it('sends files as file content, nested ones named by their path', async () => {
+      const tool = implementToolFactory()(oc)
+      const pdf = new File(['pdf'], 'q3.pdf', { type: 'application/pdf' })
+
+      await expect(toModelOutput(tool, pdf)).resolves.toEqual({
+        type: 'content',
+        value: [{ type: 'file', mediaType: 'application/pdf', data: { type: 'data', data: pdfBase64 }, filename: 'q3.pdf' }],
+      })
+
+      await expect(toModelOutput(tool, new Blob(['bytes']))).resolves.toEqual({
+        type: 'content',
+        value: [{ type: 'file', mediaType: 'application/octet-stream', data: { type: 'data', data: btoa('bytes') } }],
+      })
+
+      await expect(toModelOutput(tool, { title: 'Q3', files: [pdf] })).resolves.toEqual({
+        type: 'content',
+        value: [
+          { type: 'text', text: '{"title":"Q3","files":[{}]}' },
+          { type: 'file', mediaType: 'application/pdf', data: { type: 'data', data: pdfBase64 }, filename: 'files/0/q3.pdf' },
+        ],
+      })
+    })
+
+    it('uses the serializer option', async () => {
+      class Money {
+        constructor(readonly cents: number) {}
+      }
+
+      const serializer = new OpenAPIJsonSerializer({
+        handlers: { money: { condition: data => data instanceof Money, serialize: (data: Money) => `$${data.cents / 100}`, isTerminal: true } },
+      })
+
+      await expect(toModelOutput(implementToolFactory({ serializer })(oc), { price: new Money(150) })).resolves.toEqual({
+        type: 'json',
+        value: { price: '$1.5' },
+      })
+    })
+
+    it('can be overridden by tool options or meta', () => {
+      const custom = () => ({ type: 'text' as const, value: 'custom' })
+
+      expect(implementToolFactory()(oc, { toModelOutput: custom }).toModelOutput).toBe(custom)
+      expect(implementToolFactory()(oc.meta(aiSdkTool({ toModelOutput: custom }))).toModelOutput).toBe(custom)
+    })
+
+    it('sends files to the model through the AI SDK', async () => {
+      const report = implementToolFactory()(oc, {
+        execute: async () => new File(['pdf'], 'q3.pdf', { type: 'application/pdf' }),
+      })
+
+      const result = await generateText({ model: toolCallModel('report', {}), tools: { report }, prompt: 'Report' })
+
+      expect(result.response.messages.at(-1)?.content).toEqual([expect.objectContaining({
+        output: { type: 'content', value: [{ type: 'file', mediaType: 'application/pdf', data: { type: 'data', data: pdfBase64 }, filename: 'q3.pdf' }] },
+      })])
+    })
+  })
 })
 
 describe('createToolFactory', () => {

packages/ai-sdk/src/tool.ts

@@ -2,13 +2,14 @@ import type { ClientOptions } from '@orpc/client'
 import type { AnySchema, InferSchemaInput, InferSchemaOutput, ProcedureContract, Schema } from '@orpc/contract'
 import type { JsonSchema } from '@orpc/json-schema'
 import type { Context, ProcedureClientOptions } from '@orpc/server'
-import type { MaybeOptionalOptions } from '@orpc/shared'
-import type { FlexibleSchema, Tool } from 'ai'
+import type { MaybeOptionalOptions, Public } from '@orpc/shared'
+import type { FlexibleSchema, JSONValue, Tool, ToolResultPart } from 'ai'
 import type { FunctionTool } from './tool-meta'
 import { getAsyncIteratorObjectSchemasDetails } from '@orpc/contract'
 import { combineJsonSchemasWithComposition } from '@orpc/json-schema'
+import { OpenAPIJsonSerializer } from '@orpc/openapi'
 import { call, Procedure } from '@orpc/server'
-import { isAsyncGeneratorFunction, mergeTwoLevels, ORPC_NAME, resolveMaybeOptionalOptions, toArray } from '@orpc/shared'
+import { encodeBase64, isAsyncGeneratorFunction, loadBytes, mergeTwoLevels, ORPC_NAME, resolveMaybeOptionalOptions, stringifyJSON, toArray } from '@orpc/shared'
 import { tool } from 'ai'
 import { getAiSdkToolMeta } from './tool-meta'
 
@@ -94,6 +95,37 @@ function combineOutputSchemas(outputSchemas: AnySchema[]): FlexibleSchema | unde
   return combineSchemas([...(yieldSchemas ?? outputSchemas)].reverse(), false)
 }
 
+/**
+ * Serializes an output like an OpenAPI response body. JSON shows a `Blob` as an empty object,
+ * so blobs, nested or not, are sent as file parts named by their path, such as `report/q3.pdf`.
+ */
+async function toModelOutput(serializer: Public<OpenAPIJsonSerializer>, output: unknown): Promise<ToolResultPart['output']> {
+  const { json, maps, blobs } = serializer.serializeJson(output)
+
+  if (typeof json === 'string') {
+    return { type: 'text', value: json }
+  }
+
+  if (!blobs?.length) {
+    return { type: 'json', value: (json ?? null) as JSONValue }
+  }
+
+  const files = await Promise.all(blobs.map(async (blob, i) => {
+    const path = maps[i]!
+    const names = blob instanceof File && blob.name !== '' ? [...path, blob.name] : path
+
+    return {
+      type: 'file' as const,
+      mediaType: blob.type || 'application/octet-stream',
+      // base64 is encoded once, providers would otherwise re-encode the bytes on every step
+      data: { type: 'data' as const, data: encodeBase64(await loadBytes(blob)) },
+      ...(names.length === 0 ? {} : { filename: names.join('/') }),
+    }
+  }))
+
+  return { type: 'content', value: json instanceof Blob ? files : [{ type: 'text', text: stringifyJSON(json)! }, ...files] }
+}
+
 /**
  * Infers the value an AI SDK tool outputs for a given oRPC output type.
  *
@@ -108,8 +140,14 @@ export type ToolOutput<T> = T extends AsyncIteratorObject<infer TYield, any, any
   : T
 
 export interface ImplementToolFactoryOptions {
-  // TODO: add a `converter` option after JSON Schema converters support synchronous conversion,
-  // because the AI SDK reads JSON Schemas synchronously
+  /**
+   * Serializes outputs for the default `toModelOutput`, converting types such as `Date`, `BigInt`, and `Set`
+   * the same way OpenAPI responses do.
+   *
+   * @default new OpenAPIJsonSerializer()
+   * @see {@link https://orpc.dev/docs/integrations/ai-sdk#tool-outputs | AI SDK Integration - Tool Outputs}
+   */
+  serializer?: Public<OpenAPIJsonSerializer> | undefined
 }
 
 export interface ToolImplementer {
@@ -162,7 +200,10 @@ export interface ToolImplementer {
  *
  * @see {@link https://orpc.dev/docs/integrations/ai-sdk#tool-implementer | AI SDK Integration - Tool Implementer}
  */
-export function implementToolFactory(_options: ImplementToolFactoryOptions = {}): ToolImplementer {
+export function implementToolFactory(options: ImplementToolFactoryOptions = {}): ToolImplementer {
+  const serializer = options.serializer ?? new OpenAPIJsonSerializer()
+  const defaultToModelOutput: NonNullable<FunctionTool['toModelOutput']> = ({ output }) => toModelOutput(serializer, output)
+
   const factory: ToolImplementer = (contract, ...rest) => {
     const toolOptions = resolveMaybeOptionalOptions(rest)
 
@@ -174,6 +215,7 @@ export function implementToolFactory(_options: ImplementToolFactoryOptions = {})
     return tool({
       ...defaults,
       ...toolOptions,
+      toModelOutput: toolOptions.toModelOutput ?? defaults?.toModelOutput ?? defaultToModelOutput,
       inputSchema: combineSchemas(inputSchemas, true) ?? ANY_SCHEMA,
       outputSchema: combineOutputSchemas(outputSchemas),
     })

packages/ai-sdk/tsconfig.json

@@ -4,6 +4,7 @@
     { "path": "../client" },
     { "path": "../contract" },
     { "path": "../json-schema" },
+    { "path": "../openapi" },
     { "path": "../server" },
     { "path": "../shared" }
   ],