From 2c18452e5f7b016d4fc17e033af6a45edeff8d16 Mon Sep 17 00:00:00 2001 From: dev-seth-vapi Date: Tue, 15 Sep 2026 15:03:42 -0700 Subject: [PATCH 1/2] docs: add GPT-Live architecture and implementation guides --- fern/docs.yml | 21 ++++ fern/gpt-live/configuration.mdx | 145 ++++++++++++++++++++++++++ fern/gpt-live/conversation-design.mdx | 75 +++++++++++++ fern/gpt-live/overview.mdx | 85 +++++++++++++++ fern/gpt-live/prompts.mdx | 98 +++++++++++++++++ fern/gpt-live/quickstart.mdx | 121 +++++++++++++++++++++ fern/gpt-live/testing.mdx | 112 ++++++++++++++++++++ fern/gpt-live/tools.mdx | 125 ++++++++++++++++++++++ fern/gpt-live/voices.mdx | 83 +++++++++++++++ fern/providers/gpt-live.mdx | 28 +++++ 10 files changed, 893 insertions(+) create mode 100644 fern/gpt-live/configuration.mdx create mode 100644 fern/gpt-live/conversation-design.mdx create mode 100644 fern/gpt-live/overview.mdx create mode 100644 fern/gpt-live/prompts.mdx create mode 100644 fern/gpt-live/quickstart.mdx create mode 100644 fern/gpt-live/testing.mdx create mode 100644 fern/gpt-live/tools.mdx create mode 100644 fern/gpt-live/voices.mdx create mode 100644 fern/providers/gpt-live.mdx diff --git a/fern/docs.yml b/fern/docs.yml index d4ce25581..900952cd6 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -123,6 +123,25 @@ navigation: icon: fa-light fa-terminal path: cli/overview.mdx + - section: GPT-Live + contents: + - page: Architecture + path: gpt-live/overview.mdx + - page: Quickstart + path: gpt-live/quickstart.mdx + - page: Conversation design + path: gpt-live/conversation-design.mdx + - page: Speaker and reasoner + path: gpt-live/prompts.mdx + - page: Tools and business logic + path: gpt-live/tools.mdx + - page: Voices and languages + path: gpt-live/voices.mdx + - page: Configuration and compatibility + path: gpt-live/configuration.mdx + - page: Migration and testing + path: gpt-live/testing.mdx + - section: Assistants collapsed: open-by-default contents: @@ -733,6 +752,8 @@ navigation: collapsed: true icon: fa-light fa-link contents: + - page: GPT-Live + path: providers/gpt-live.mdx - section: Transcribers (Speech-to-text) path: providers/transcriber/overview.mdx icon: fa-light fa-microphone diff --git a/fern/gpt-live/configuration.mdx b/fern/gpt-live/configuration.mdx new file mode 100644 index 000000000..fcb43f032 --- /dev/null +++ b/fern/gpt-live/configuration.mdx @@ -0,0 +1,145 @@ +--- +title: GPT-Live configuration and compatibility reference +subtitle: Look up supported fields, transports, and feature restrictions +description: Look up GPT-Live model settings, defaults, voice configuration, call transports, tools, greetings, recording behavior, and unsupported Vapi features. +slug: gpt-live/configuration +--- + +Configure GPT-Live through a Vapi assistant with `model.provider: "openai"`, `model.model: "gpt-live-1"`, and an OpenAI voice. The tables below describe this integration's supported behavior. + +## Model and voice fields + +These fields belong to the assistant configuration. See [Create Assistant](/api-reference/assistants/create) for the complete generated schema. + +| Field | Values or behavior | +| --- | --- | +| `model.provider` | `openai` | +| `model.model` | `gpt-live-1` | +| `model.speaker.instructions` | Speaker prompt; see [instruction precedence](/gpt-live/prompts#understand-instruction-precedence). | +| `model.speaker.personalityPacks` | Array of `eager-listener`, `idle-hummer`, `bouncy`, and `unhurried`. | +| `model.reasoner.provider` | `openai`; defaults to OpenAI when omitted. | +| `model.reasoner.model` | `gpt-5.6-sol`, `gpt-5.6-terra`, or `gpt-5.6-luna`; defaults to `gpt-5.6-terra`. | +| `model.reasoner.reasoningEffort` | `none`, `low`, `medium`, `high`, `xhigh`, or `max`; defaults to `low`. | +| `model.reasoner.instructions` | Custom reasoner prompt; omission uses Vapi's default. | +| `model.tools` | Inline supported tool definitions. | +| `model.toolIds` | IDs of saved tools to resolve for the assistant. | +| `voice.provider` | `openai` | +| `voice.voiceId` | One of the [22 supported voices](/gpt-live/voices). | + +A custom prompt replaces the corresponding fallback prompt. An explicit empty string is preserved. See [Configure the speaker and reasoner](/gpt-live/prompts) before overriding defaults. + +## Greetings and duration + +Use a text greeting if the assistant should begin the conversation. + +| Field or feature | Behavior | +| --- | --- | +| `firstMessageMode: "assistant-speaks-first"` | Speaks the configured text greeting. | +| `firstMessageMode: "assistant-waits-for-user"` | Waits for the caller. | +| `firstMessage` | Text greeting; without usable text, the assistant waits. | +| `maxDurationSeconds` | Maximum call duration in seconds. | +| Audio URL greeting | Unsupported; not played as a greeting. | +| Generated-message first-message mode | Unsupported; the assistant waits for the caller. | + +Generated speech may vary from the supplied greeting. A text greeting is not a guarantee of exact prerecorded playback. + +## Call transports + +Use the [Create Call endpoint](/api-reference/calls/create) or the dashboard's web-call interface. + +| Transport | Support | Requirements | +| --- | --- | --- | +| WebRTC web call | Supported | Dashboard **Talk** or Vapi web-call interface. | +| `vapi.websocket` | Supported with restrictions | Raw, mono PCM16 audio at 24 kHz. | +| `vapi.sip` | Supported | Vapi SIP transport configuration. | +| Native Twilio | Supported | A configured Twilio phone number and call transport. | +| Native Telnyx or Vonage | Unsupported | Choose a supported transport. | + +### WebSocket configuration + +Send this body to `POST https://api.vapi.ai/call` with your Vapi private API key: + +```json +{ + "assistantId": "YOUR_ASSISTANT_ID", + "transport": { + "provider": "vapi.websocket", + "audioFormat": { + "format": "pcm_s16le", + "container": "raw", + "sampleRate": 24000 + } + } +} +``` + +Connect to `transport.websocketCallUrl` in the response. Send and receive binary audio frames containing signed 16-bit little-endian samples at 24 kHz. Convert your microphone audio to that format before sending it. + +The default WebSocket sample rate is not sufficient for GPT-Live; set `24000` explicitly. This audio requirement applies to the Vapi WebSocket transport, not to browser-negotiated WebRTC audio or carrier audio. + +Close the WebSocket to end the call, or have the assistant invoke `endCall`. Incoming JSON call-control commands are not supported by this GPT-Live connection. See [WebSocket transport](/calls/websocket-transport) for the general transport, applying the restrictions above. + +## Tool and transfer compatibility + +The supported tool types are `function`, `apiRequest`, `endCall`, `dtmf`, and `transferCall`. All other tool types are rejected. + +| Feature | Support or restriction | +| --- | --- | +| Function tools | Synchronous and asynchronous functions. | +| API request tools | Configured HTTP requests. | +| Outgoing DTMF | Supported on SIP using RTP DTMF. Use digits as a string, such as `"0011*#"`; native Twilio, web calls, and raw WebSocket calls do not support this tool. | +| SIP INFO DTMF | Unsupported; use RTP DTMF on SIP. | +| Incoming keypad collection | Rejected when `keypadInputPlan.enabled` is true. | +| Blind transfer | Supported on native Twilio and Vapi SIP to configured phone or SIP destinations. | +| Warm transfer or generated transfer summary | Rejected. | +| Transfer fallback plans or SIP `bye` verb | Rejected. | +| SIP transfer with extension dialing | Unsupported; use a direct destination. | +| Squads and assistant handoffs | Unsupported. | + +A configured transfer tool does not make every transport transfer-capable. Web calls and raw WebSocket calls do not gain phone transfer support by adding the tool. Carrier acceptance of a transfer does not confirm that the destination answered. + +## Settings that do not apply + +GPT-Live handles speech directly. Classic transcriber selection, text-to-speech controls, and start/stop-speaking plans do not configure this audio conversation. + +General model fields such as `temperature` and `maxTokens` do not tune this integration. Use the speaker prompt for conversational behavior and `model.reasoner.reasoningEffort` for reasoning effort. + +## Unsupported features and artifact behavior + +Check these restrictions before moving an existing assistant: + +| Feature | Behavior | +| --- | --- | +| Model fallbacks | Rejected. GPT-Live also cannot be a fallback model. | +| Voice fallbacks | Rejected. | +| Non-OpenAI or custom voice IDs | Rejected unless included in the supported Vapi voice catalog. | +| Live message injection and call-control commands | Unsupported for this integration. | +| Listen or monitor sockets | Unsupported. | +| Reconnecting to an existing GPT-Live session | Unsupported. Start a new call and reconcile pending work in your service. | +| Recording disabled in the assistant | No recording is created. | +| Recording-consent plan configured | GPT-Live recording is disabled because this integration does not collect that consent. | + +Call artifacts and analysis are separate from your service's authoritative operation records. A transcript cannot by itself establish that a booking succeeded. See [Migrate and test GPT-Live](/gpt-live/testing). + +## Post-call analysis and observability + +Completed GPT-Live calls use Vapi's shared post-call processing. Configure analysis and observability on the assistant and inspect the resulting call data after processing finishes. + +| Capability | Configuration or result | Requirement | +| --- | --- | --- | +| Structured outputs | Attach IDs through `artifactPlan.structuredOutputIds`; read `call.artifact.structuredOutputs`. | Create the output definitions and retain the conversation data needed for extraction. | +| Legacy call analysis | Configure `analysisPlan`. | Existing configurations remain supported; prefer structured outputs for new extraction and evaluation setups. | +| Scorecards | Grade structured output values; read `call.artifact.scorecards`. | Configure a scorecard and its required structured outputs. | +| Boards | Query call data and plot outcomes. | Choose metrics available on your calls; classic pipeline timing fields do not describe GPT-Live's architecture. | +| Monitoring | Configure monitors, thresholds, and notifiers. | Retained call data is required; monitoring is unavailable with Zero Data Retention. | +| End-of-call reports | Receive configured `end-of-call-report` server messages. | Configure the assistant's Server URL and requested server messages. | + +See [Call analysis](/assistants/call-analysis), [Scorecards](/observability/scorecard-quickstart), and [Monitoring](/observability/monitoring-quickstart) for setup. These are post-call capabilities; they do not imply access to live listen sockets or message injection. + +An extracted statement such as “booking confirmed” describes the conversation. Verify the booking in your service when measuring actual task success. Analysis results do not replace the authoritative operation record. + +## Credentials and cost + +If you bring your own OpenAI key, it needs access to GPT-Live and the selected reasoner. Organization access to the Vapi feature is a separate prerequisite. + +Review Vapi call costs for your configuration. OpenAI's [model pricing](https://developers.openai.com/api/docs/models/gpt-live-1) describes direct provider charges; it is not a quote for your complete Vapi call. Reasoning, tools, and telephony can contribute additional cost. diff --git a/fern/gpt-live/conversation-design.mdx b/fern/gpt-live/conversation-design.mdx new file mode 100644 index 000000000..e1e0a3968 --- /dev/null +++ b/fern/gpt-live/conversation-design.mdx @@ -0,0 +1,75 @@ +--- +title: Design GPT-Live conversations +subtitle: Handle corrections, overlapping speech, and work in progress +description: Design GPT-Live conversations around clarification, interruptions, pending tools, corrected requests, confirmed outcomes, and recoverable business actions. +slug: gpt-live/conversation-design +--- + +Design the conversation around the caller's current intent and the status of the task. The speaker can keep talking while work proceeds, so an acknowledgment must stay distinct from a confirmed result. + +## Track conversation and action separately + +Three events can happen at different times: the caller finishes speaking, the tool returns, and the caller hears the result. Treat each as evidence of its own outcome. + +| What happens | What it establishes | What it does not establish | +| --- | --- | --- | +| Caller interrupts | New speech needs attention | A pending operation was cancelled | +| Assistant says “I'll check” | It acknowledged the request | A lookup started or succeeded | +| Tool reports available slots | The lookup returned availability | A slot is reserved | +| Booking service confirms an operation | The booking succeeded | The caller heard the confirmation | +| Connection closes | The conversation ended | External work was rolled back | + +## Handle a correction during a lookup + +Consider this exchange: + +> **Caller:** What do you have on Thursday? +> **Assistant:** I'll check. +> **Caller:** Actually, Friday afternoon. + +The corrected task is Friday afternoon. The speaker should acknowledge that change without reading out Thursday's results as the answer. + +Your service should associate requests with the current task. For example, keep a task revision with the requested date and time zone. Before using a returned result or executing a booking, verify that it still matches the current request. This is an application design pattern, not a Vapi configuration field. + +If the Thursday lookup cannot be cancelled, let it finish and mark its result as outdated. If an action already changed a record, report that state and use your service's change or cancellation procedure. + +## Ask for the missing decision + +Ask one short question that makes the next action possible. An appointment request may need a date, time zone, appointment type, or confirmation of a specific slot. + +Prefer “Which time zone should I use?” to asking the caller to repeat the entire request. When a correction invalidates a previous confirmation, obtain a new confirmation for the revised action. + +Short acknowledgments can keep a conversation understandable. Avoid instructions that require a filler phrase before every tool call; evaluate whether the assistant's timing is useful in real audio. + +## Explain work in progress accurately + +Use language that matches what the service knows: + +| Task status | Example spoken response | +| --- | --- | +| Lookup pending | “I'm checking Friday afternoon.” | +| Slots returned | “There are openings at two and four. Which works for you?” | +| Booking pending | “I'm submitting the two o'clock appointment.” | +| Booking confirmed | “Your two o'clock appointment is booked.” | +| Outcome unknown | “I haven't confirmed whether that booking went through yet.” | +| Cancellation confirmed | “The appointment has been cancelled.” | + +The examples illustrate meaning, not guaranteed verbatim speech. Choose one useful update when work is slow instead of repeatedly announcing the same pending state. + +## Treat cancellation as an operation + +“Stop talking,” “stop checking,” and “cancel my appointment” have different effects. Ask a brief clarification when the intended action is unclear. + +Put cancellation and duplicate prevention in the service that owns the action. A timeout or lost response can leave an operation's outcome unknown. Reconcile that outcome before retrying a write. Use an operation identifier so a retry cannot silently create a second booking. + +## Choose where exact wording matters + +GPT-Live generates speech and can paraphrase supplied text. Test language, pronunciation, and required phrases with audio; a matching text prompt does not establish verbatim delivery. + +If your application requires an exact prerecorded opening or approval before any audio plays, check the [compatibility reference](/gpt-live/configuration) before choosing this integration. Do not assume that withholding a reasoner result pauses the speaker. + +## Evaluate the whole exchange + +Test corrections during speech, during a lookup, and immediately before a write. For each test, inspect the audio, tool results, and final record in your service. + +OpenAI's [migration guide](https://developers.openai.com/api/docs/guides/live-migration) discusses the separation between interruption and backend cancellation. The [Vapi testing guide](/gpt-live/testing) turns these cases into a repeatable comparison. diff --git a/fern/gpt-live/overview.mdx b/fern/gpt-live/overview.mdx new file mode 100644 index 000000000..4e57fdd70 --- /dev/null +++ b/fern/gpt-live/overview.mdx @@ -0,0 +1,85 @@ +--- +title: GPT-Live architecture +subtitle: Understand how conversation, reasoning, and tools work together +description: Understand GPT-Live in Vapi, including full-duplex audio, speaker and reasoner responsibilities, delegation, tool execution, and conversation state. +slug: gpt-live/overview +--- + +GPT-Live is a voice model that can listen and speak at the same time. Vapi connects it to a separate reasoning model and your tools so an assistant can hold a conversation while task work continues. + + +GPT-Live access must be enabled for your organization. Contact Vapi if the model is unavailable. This section documents Vapi's integration; OpenAI's direct Live API exposes a different configuration and event interface. + + +## How the models work together + +The **speaker** handles the spoken conversation. The **reasoner** handles delegated requests, applies your task instructions, and selects tools. **Delegation** is the handoff from the speaker to the reasoner when a request needs backend work. + +```mermaid +flowchart LR + C[Caller] <-->|Audio| V[Vapi call transport] + V <-->|Audio| S[GPT-Live speaker] + S -->|Delegates work| R[Reasoner] + R -->|Selects tools| T[Vapi tool execution] + T <-->|Requests and results| B[Your services] + T -->|Tool results| R + R -->|Result for the conversation| S +``` + +The speaker can continue listening and speaking while the reasoner works. A completed tool request, a spoken answer, and an action recorded in your service are separate outcomes. + +| Component | Responsibility | Configuration | +| --- | --- | --- | +| Speaker | Conversation style, language, clarification, and delegation decisions | `model.speaker` | +| Reasoner | Task procedures, reasoning, tool selection, and interpreting results | `model.reasoner` | +| Vapi | Call transport, model connections, tools, post-call analysis, and observability | Assistant, call, and monitoring configuration | +| Your service | Authoritative records, permissions, input validation, and action status | Your tool implementation | + +For example, the caller asks for Friday availability. The speaker delegates a lookup. The reasoner calls your availability tool, then returns the confirmed slots for the speaker to explain. No appointment exists until a booking operation succeeds. + +## What full duplex changes + +In a speech-to-text, language-model, and text-to-speech pipeline, separate components transcribe input, compose a response, and synthesize speech. GPT-Live handles the audio conversation directly. You do not choose a separate transcriber or text-to-speech provider for this integration. + +Continuous conversation changes how you organize the application: + +- Keep short conversational guidance in the speaker prompt. +- Put detailed procedures and tool-use rules in the reasoner prompt. +- Keep durable task state and authorization in your service. +- Evaluate what the caller hears alongside what the tools actually do. + +A caller may correct a date while a lookup is running. Interrupting speech does not undo a booking or cancel an external request. The application needs a policy for those actions. See [Design GPT-Live conversations](/gpt-live/conversation-design). + +## Choose an architecture for your requirements + +GPT-Live is useful when overlapping speech, conversational timing, and continued dialogue during backend work matter. Evaluate it with recordings and tasks representative of your callers. + +Check the [compatibility reference](/gpt-live/configuration) before migrating. Existing transcriber controls, voice fallbacks, and some call features do not carry over. A workflow that depends on those features needs a supported alternative before migration. + +A natural acknowledgment is not a measure of task completion. Compare the time to a useful answer and the correctness of the final action, rather than measuring only the first sound. + +## What Vapi provides around the conversation + +Vapi manages the call lifecycle and provides tools to review outcomes after the conversation. Configure the features your application needs; selecting GPT-Live does not automatically configure analysis or quality alerts. + +| Capability | What you use it for | +| --- | --- | +| [Call artifacts](/assistants/retrieve-call-artifacts) | Review transcripts, available recordings, and tool activity. | +| [Post-call analysis](/assistants/call-analysis) | Summarize a conversation and evaluate its outcome. | +| [Structured outputs](/assistants/structured-outputs) | Extract typed results such as the requested date, disposition, or whether a booking was confirmed. | +| [Scorecards](/observability/scorecard-quickstart) | Grade completed calls against criteria based on structured outputs. | +| [Boards](/observability/boards-quickstart) | Track call outcomes and trends across assistants. | +| [Monitoring](/observability/monitoring-quickstart) | Evaluate retained call data against configured thresholds and send alerts. | +| [Simulations](/observability/simulations-overview) | Define repeatable conversation scenarios; see the current GPT-Live limitation in the [testing guide](/gpt-live/testing#use-simulations-and-evals-appropriately). | +| [Evals](/observability/evals-quickstart) | Test supported model decisions; complement them with audio tests of the complete GPT-Live assistant. | + +Monitoring here means quality evaluation and alerting from call data. It is separate from listen or monitor sockets, which this integration does not support. See the [compatibility reference](/gpt-live/configuration#post-call-analysis-and-observability) for configuration and data requirements. + +## Start building + +- [Create a GPT-Live assistant](/gpt-live/quickstart) and make a web call. +- [Configure the speaker and reasoner](/gpt-live/prompts) for your task. +- [Connect tools](/gpt-live/tools) to your services. +- [Migrate and evaluate an assistant](/gpt-live/testing) against an existing baseline. + +For the underlying model architecture, see OpenAI's [GPT-Live overview](https://developers.openai.com/api/docs/guides/live). diff --git a/fern/gpt-live/prompts.mdx b/fern/gpt-live/prompts.mdx new file mode 100644 index 000000000..78a8c8136 --- /dev/null +++ b/fern/gpt-live/prompts.mdx @@ -0,0 +1,98 @@ +--- +title: Configure the GPT-Live speaker and reasoner +subtitle: Give each model clear instructions for its part of the task +description: Configure GPT-Live speaker and reasoner prompts in Vapi, define delegation conditions, select reasoning effort, and apply optional personality packs. +slug: gpt-live/prompts +--- + +Set conversation behavior in `model.speaker.instructions`. Set task procedures and tool-use rules in `model.reasoner.instructions`. Keeping these responsibilities clear makes it easier to diagnose a missed delegation or incorrect action. + +## Assign instructions to the right model + +| Instruction | Place it in | +| --- | --- | +| Role, tone, response length, and spoken language | Speaker | +| When to ask a clarification | Speaker | +| Which requests need backend work | Speaker | +| Required inputs and business procedures | Reasoner | +| Tool selection and handling failed results | Reasoner | +| Authorization, validation, and duplicate prevention | Your service, with corresponding reasoner guidance | + +The reasoner receives conversation context for delegated work. It does not replace validation in your tool implementation. + +## Write the speaker prompt + +Describe only capabilities your configured tools provide. For an assistant with availability and booking tools, use a short prompt like this: + +```text title="Speaker instructions" +You help callers schedule appointments. Speak clearly and use short replies. +Ask one brief question when the request is unclear. + +The backend can check availability and book a confirmed appointment. +Delegate requests for availability, booking, changes to a pending request, +and requests to end the call. +Handle greetings and requests to repeat a confirmed result directly. + +When the caller corrects a date or time, acknowledge the correction and +use the revised request. Do not announce a booking before the backend +confirms it. Explain uncertainty when the outcome is unknown. +``` + +Add booking to this prompt only after you implement the booking tool. The [quickstart](/gpt-live/quickstart) deliberately starts without it. + +OpenAI's [GPT-Live prompting guide](https://developers.openai.com/api/docs/guides/live-prompting) recommends concise instructions with concrete delegation conditions. Apply those principles through Vapi's speaker field; direct OpenAI session fields are not Vapi assistant fields. + +## Write the reasoner prompt + +Name the required inputs, tool boundaries, and what counts as success. Adapt this example to your tool names: + +```text title="Reasoner instructions" +Use the current conversation to resolve delegated appointment requests. +Check the requested date and time zone before looking up availability. +Use lookupAvailability for available slots. Availability is not a booking. + +Use the booking tool only after the caller confirms a specific slot. +If the caller changes the date or time, obtain confirmation again. +Follow the service's validation and operation-status results. + +Return a short factual result for the speaker. Distinguish pending, +confirmed, failed, and unknown outcomes. Do not invent a successful action. +When the caller asks to finish the conversation, call endCall. +``` + +Your service must still reject unauthorized or invalid requests. A prompt is guidance to the model, not an enforcement boundary. + +## Understand instruction precedence + +Explicit `model.speaker.instructions` takes precedence over legacy `model.systemPrompt` and system messages in `model.messages`. If you omit the speaker field, Vapi uses that legacy prompt configuration. + +Omitting `model.reasoner.instructions` uses Vapi's default reasoner instructions. A custom reasoner prompt replaces those defaults; Vapi does not append the default behavioral instructions. An explicit empty string remains empty. + +The prompts are independent. Changing the speaker prompt does not also configure the reasoner. Personality packs add guidance to the speaker instructions when selected. + +## Choose reasoning effort + +Start with the default reasoner, `gpt-5.6-terra`, at `low` effort. Increase effort when task evaluations show that additional reasoning helps. Higher effort can increase the time before a useful result reaches the caller. + +Supported model and effort values are listed in the [configuration reference](/gpt-live/configuration). General assistant model fields such as `temperature` and `maxTokens` do not configure this reasoner path. + +## Add personality packs deliberately + +Set `model.speaker.personalityPacks` to an array containing any of these IDs: `eager-listener`, `idle-hummer`, `bouncy`, or `unhurried`. + +```json +{ + "speaker": { + "instructions": "Speak clearly and ask one question at a time.", + "personalityPacks": ["unhurried"] + } +} +``` + +This is a fragment of the `model` object. Test each pack with your actual callers and tasks. Avoid piling on style instructions that conflict with one another. + +## Diagnose prompt problems + +If the assistant talks about doing work but no tool runs, inspect the speaker's delegation instructions and the reasoner's tool access. If a tool runs with wrong inputs, inspect the reasoner procedure and your service validation. + +For language-specific prompts and audio examples, see [Voices and languages](/gpt-live/voices). diff --git a/fern/gpt-live/quickstart.mdx b/fern/gpt-live/quickstart.mdx new file mode 100644 index 000000000..3af2ed247 --- /dev/null +++ b/fern/gpt-live/quickstart.mdx @@ -0,0 +1,121 @@ +--- +title: GPT-Live quickstart +subtitle: Create an assistant and test a full-duplex conversation +description: Create a GPT-Live assistant in Vapi with speaker and reasoner prompts, choose a voice, test a web call, and end the call using the endCall tool. +slug: gpt-live/quickstart +--- + +Create an assistant that can hold a conversation and end a call on request. This first assistant has no access to appointment data; the [tools guide](/gpt-live/tools) adds that capability. + +## Prerequisites + +- A Vapi organization with GPT-Live enabled. +- A microphone and a browser with microphone permission. +- A Vapi private API key for the API method. Keep it on your server or local machine. +- If you supply an OpenAI key, access to GPT-Live and the selected reasoner model on that key. + +## Create the assistant + +Choose either method. Both create a saved assistant that you can test in the dashboard. + + + + + +Open [Assistants](https://dashboard.vapi.ai/assistants), select **Create Assistant**, and choose **GPT-Live** in the assistant configuration. + + +Choose Marin as the voice. In **Speaker**, enter: + +```text +You explain appointment scheduling. Speak clearly and briefly. +Availability checks and booking are not connected yet. +Delegate requests to end the call. Do not claim to have checked or changed a booking. +``` + +In **Reasoner**, enter: + +```text +When the caller asks to finish, call endCall. +No availability or booking tools are configured. Do not invent appointment results. +``` + +Keep or add the `endCall` tool in the assistant's tools. + + +Set the first message to “Hello. I can help explain appointment scheduling. What would you like to know?” Configure the assistant to speak first. + + +Select **Publish** and complete the publication flow. Keep the assistant ID for later API requests. + + + + + + +Save this JSON as `assistant.json`: + +```json title="assistant.json" +{ + "name": "Appointment assistant", + "model": { + "provider": "openai", + "model": "gpt-live-1", + "speaker": { + "instructions": "You are an appointment assistant. Speak clearly and briefly. Ask a short question when the request is unclear. You cannot check availability or book appointments yet. Delegate requests to end the call. Do not claim to have checked or changed a booking." + }, + "reasoner": { + "model": "gpt-5.6-terra", + "reasoningEffort": "low", + "instructions": "Handle delegated requests using the available tools. When the caller asks to finish, call endCall. No availability or booking tools are configured. Do not invent appointment results." + }, + "tools": [ + { + "type": "endCall" + } + ] + }, + "voice": { + "provider": "openai", + "voiceId": "marin" + }, + "firstMessageMode": "assistant-speaks-first", + "firstMessage": "Hello. I can help explain appointment scheduling. What would you like to know?", + "maxDurationSeconds": 600 +} +``` + + +Set `VAPI_PRIVATE_API_KEY` to your Vapi private API key, then send the configuration: + +```bash +curl --fail-with-body https://api.vapi.ai/assistant \ + -H "Authorization: Bearer $VAPI_PRIVATE_API_KEY" \ + -H "Content-Type: application/json" \ + --data-binary @assistant.json +``` + +Keep the returned `id`. This request uses the [Create Assistant endpoint](/api-reference/assistants/create). + + + + + +## Test the conversation + +Open the saved assistant in the dashboard and select **Talk**. Allow microphone access when prompted. + +Try these exchanges: + +| Say | Expected outcome | +| --- | --- | +| “What can you help with?” | The assistant explains its limited appointment role. | +| “Can you book Friday?” | It explains that booking is not connected; it does not invent a reservation. | +| Interrupt a long answer with “Keep it brief.” | It adapts the conversation to the request. | +| “That's all. Please end the call.” | It delegates the request and invokes `endCall`. | + +Check both the conversation and the call record. A spoken goodbye alone does not prove that the call ended. You can also end the web call with the dashboard's call controls. + +## Continue building + +Add an [availability lookup tool](/gpt-live/tools), then update both prompts to describe the new capability. See [Configuration and compatibility](/gpt-live/configuration) for phone calls and raw WebSocket audio. diff --git a/fern/gpt-live/testing.mdx b/fern/gpt-live/testing.mdx new file mode 100644 index 000000000..f141b9533 --- /dev/null +++ b/fern/gpt-live/testing.mdx @@ -0,0 +1,112 @@ +--- +title: Migrate and test a GPT-Live assistant +subtitle: Compare task outcomes and diagnose conversation failures +description: Migrate an existing Vapi assistant to GPT-Live, test interruptions and tool outcomes, inspect call artifacts, and troubleshoot configuration and audio failures. +slug: gpt-live/testing +--- + +Validate the migrated assistant against representative calls and final service records. A fluent conversation is only one part of success; the assistant must also perform the correct actions and end the call cleanly. + +## Prepare a migration + + + +Collect representative call scenarios, expected actions, and final records. Include errors and corrections, not only successful requests. Use test data with a repeatable starting state. + + +Compare the assistant's transports, tools, greetings, and fallback plans with the [compatibility reference](/gpt-live/configuration). Resolve dependencies on unsupported features before replacing the assistant. + + +Move conversational guidance into the speaker prompt and detailed procedures into the reasoner prompt. Retain validation and authorization in your services. See [Configure the speaker and reasoner](/gpt-live/prompts). + + +Use `gpt-live-1` and a supported OpenAI voice. Connect test tools, then confirm a basic call and an end-call request before testing business actions. + + +Repeat each scenario against both assistants. Record intentional differences and regressions. Do not route production calls to the new assistant until its required behaviors pass. + + + +## Test conversation and task outcomes + +Use this matrix as a starting point for an appointment assistant: + +| Scenario | Check in the audio | Check in your service | +| --- | --- | --- | +| Ask for availability | Speaks the returned slots | Correct date and time zone queried | +| Change Thursday to Friday during lookup | Uses the corrected request | Outdated lookup cannot drive a booking | +| Interrupt an answer | Responds to the new request | No accidental duplicate action | +| Confirm one offered slot | Announces success only after confirmation | Exactly one booking for that slot | +| Tool responds slowly | Gives accurate pending information | Operation remains traceable | +| Tool fails | Explains failure without inventing a result | No successful write recorded | +| Response is lost after a write | Treats the outcome as uncertain | Status reconciled before retry | +| Ask to cancel | Distinguishes request from completion | Cancellation status verified | +| Disconnect during work | New call does not assume completion | Pending work reconciled | +| Ask to end the call | Conversation ends | Call record reaches an ended state | + +Include the languages, background noise, names, and speaking styles your callers use. Voice previews demonstrate a voice; they do not replace application testing. + +## Measure useful outcomes + +Track task completion, incorrect actions, duplicate writes, and the time to a useful spoken answer. Record first-audio timing separately so a quick acknowledgment does not hide slow or unsuccessful work. + +Use the same starting state and success criteria when comparing configurations. Repeat trials because generated conversations vary. Increase reasoning effort only when the improvement in task outcomes justifies its effect on response time and cost. + +OpenAI's [voice evaluation guide](https://developers.openai.com/cookbook/examples/audio/voice_agent_evaluation) provides additional evaluation methods. Adapt them to the outcomes your Vapi assistant must achieve. + +## Use Simulations and Evals appropriately + +Vapi [Simulations](/observability/simulations-overview) provide scenarios, AI tester personalities, and success criteria for repeatable conversations. Use the scenario matrix above to define what a GPT-Live test should cover. + +The current voice simulation bridge uses 16 kHz audio and listen sockets. GPT-Live's raw WebSocket connection requires 24 kHz audio and does not provide those listen sockets, so the current voice simulation path is not compatible. Use calls through a supported GPT-Live transport to exercise these scenarios until a compatible simulation path is available. + +[Evals](/observability/evals-quickstart) test supported model decisions using configured inputs and expected behavior. They do not establish full-duplex timing, interruption handling, or audio quality. A passing text evaluation or chat simulation is not evidence that a GPT-Live voice call works. + +## Analyze outcomes after calls + +Configure [structured outputs](/assistants/structured-outputs) to extract consistent fields from completed calls. For the appointment example, useful fields include the requested date, whether the caller corrected it, and whether the assistant announced a confirmed booking. + +Use [scorecards](/observability/scorecard-quickstart) to grade those outputs against your criteria. Keep conversational evidence separate from service evidence: a transcript-based grade cannot establish that exactly one appointment exists in your database. + +Existing `analysisPlan` configurations also use the shared post-call analysis path. For new extraction and evaluation setups, follow the [call analysis guide's recommendation](/assistants/call-analysis) to use structured outputs. + +## Monitor production quality + +Use [Boards](/observability/boards-quickstart) to inspect outcome trends and [Monitoring](/observability/monitoring-quickstart) to alert on configured thresholds. Start with failed calls, failed task outcomes, and changes in the rate of unresolved requests. + +Choose an evaluation window and threshold that match your traffic. Review the affected calls when an alert fires, then turn confirmed failures into repeatable test scenarios. Sample unflagged calls too; a monitor only detects the conditions you configure. + +Monitoring requires retained call data and is unavailable with Zero Data Retention. It evaluates call quality and outcomes; it does not provide a live listen connection to the GPT-Live session. + +## Inspect call evidence + +Use the [Vapi call record and artifacts](/assistants/retrieve-call-artifacts) to inspect the ended reason, messages, tool activity, and available artifacts. Compare returned tool results with your service's logs and operation identifiers. + +Check recording settings before relying on audio artifacts. Recording is disabled when a recording-consent plan is configured for this integration. Preserve suitable test evidence through an approved recording setup. + +Transcripts are useful for reviewing words. Listen to audio when judging timing, interruption handling, pronunciation, or what the caller actually heard. + +## Troubleshoot common failures + +Start with the symptom, then verify its likely cause: + +| Symptom | Check | Fix | +| --- | --- | --- | +| GPT-Live is unavailable | Organization feature access | Contact Vapi to enable access. | +| Voice rejected | `voice.provider` and `voice.voiceId` | Use OpenAI and a [supported voice ID](/gpt-live/voices). | +| Assistant save rejects tools | Tool type and resolved names | Use supported types and unique names; inspect saved tools too. | +| Call fails on WebSocket startup | Audio format and sample rate | Set raw `pcm_s16le` at `24000` Hz explicitly. | +| WebSocket audio sounds distorted | Sample encoding, rate, and channels | Convert the source to mono, signed 16-bit little-endian PCM at 24 kHz. | +| Assistant never greets | First-message mode and usable text | Set `assistant-speaks-first` and a text `firstMessage`. | +| Assistant promises a lookup but no tool runs | Both prompts and attached tools | Add a concrete delegation rule and make the tool available to the reasoner. | +| Tool result never reaches the conversation | Handler response and matching ID | Return the actual `toolCallId`; complete asynchronous work through its supported contract. | +| Transfer fails | Transport and configured destination | Use a supported blind transfer; check carrier requirements and destination restrictions. | +| No recording | Recording settings and consent plan | Check the documented recording behavior before the test. | + +## Finish or roll back a test + +End active calls using the call interface or `endCall`. Disconnect test WebSocket clients and verify that the calls ended. Review any external operations still pending after disconnect. + +Keep the previous assistant available during migration. To roll back, route new calls to that assistant. Reconcile actions already submitted by the test assistant; changing call routing does not undo those actions. + +For more background on preserving application state and procedures, see OpenAI's [migration guide](https://developers.openai.com/api/docs/guides/live-migration). diff --git a/fern/gpt-live/tools.mdx b/fern/gpt-live/tools.mdx new file mode 100644 index 000000000..0ae243c31 --- /dev/null +++ b/fern/gpt-live/tools.mdx @@ -0,0 +1,125 @@ +--- +title: Connect tools to GPT-Live +subtitle: Connect an availability lookup and return confirmed results +description: Add Vapi function and API request tools to GPT-Live, configure delegation, return matching tool results, and handle pending work and restricted call tools. +slug: gpt-live/tools +--- + +Add tools to give the reasoner access to your services. This guide extends the [quickstart assistant](/gpt-live/quickstart) with an availability lookup. The lookup returns slots; it does not book an appointment. + +## Prerequisites + +- A saved GPT-Live assistant. +- A reachable HTTPS service for the lookup. +- A Vapi private API key to update the assistant. + +Your service owns appointment data and validates requests. Use fictional availability while testing. + +## Add an availability function + + + +Handle Vapi's [`tool-calls` server message](/server-url/events). Read the function name and arguments from `message.toolCallList`, validate the date and time zone, and query your availability service. + +For each call, return its matching `toolCallId`. This example response uses a placeholder ID; your handler must echo the real ID from the request: + +```json +{ + "results": [ + { + "name": "lookupAvailability", + "toolCallId": "TOOL_CALL_ID_FROM_REQUEST", + "result": "There are openings at 2 PM and 4 PM on the requested date in the requested time zone. No appointment has been booked." + } + ] +} +``` + +Return a factual failure when the lookup fails. Do not return sample slots as a successful response to a production request. + + +Add this definition to `model.tools`, keeping the quickstart's `endCall` tool. Replace the example Server URL with your handler: + +```json +{ + "type": "function", + "function": { + "name": "lookupAvailability", + "description": "Check available appointment slots. Does not reserve or book a slot.", + "parameters": { + "type": "object", + "properties": { + "date": { + "type": "string", + "description": "Requested calendar date in YYYY-MM-DD format." + }, + "timeZone": { + "type": "string", + "description": "IANA time zone, such as America/New_York." + } + }, + "required": ["date", "timeZone"] + } + }, + "server": { + "url": "https://api.example.com/vapi/tools" + } +} +``` + +You can also create a saved tool and reference its ID through `model.toolIds`. See [Create Tool](/api-reference/tools/create). + + +In the speaker prompt, explain that availability checks are now supported and delegate those requests. Keep booking explicitly unavailable. + +In the reasoner prompt, require a date and time zone, call `lookupAvailability`, and report only returned slots. Explain that availability does not reserve a slot. + + +Publish the updated assistant in the dashboard, or update it with the [Update Assistant endpoint](/api-reference/assistants/update). Preserve its other tools and model settings when preparing the update. + + +Ask for availability with an explicit date and time zone. Confirm that your service receives the request, the returned tool-call ID matches, and the spoken answer agrees with the returned slots. + +Then ask to book a slot. The assistant must explain that booking is not connected. Verify that no appointment record was created. + + + +## Choose a supported tool type + +The reasoner can use these Vapi tool types: + +| Type | Use | +| --- | --- | +| `function` | Call your Server URL with a `tool-calls` message. | +| `apiRequest` | Execute a configured HTTP request. | +| `endCall` | End the conversation on request. | +| `dtmf` | Send outgoing RTP keypad tones on SIP. | +| `transferCall` | Submit a supported blind transfer to a configured destination. | + +Give function tools unique `function.name` values and API request tools unique `name` values. Vapi validates the resolved tool set, including saved tools and tools derived from legacy assistant fields. + +Other tool types are rejected for GPT-Live. To query an external knowledge source, expose the lookup through a supported function or API request tool. See [Configuration and compatibility](/gpt-live/configuration) for transport and transfer restrictions. + +## Handle slow or asynchronous work + +A function may be synchronous or asynchronous. For asynchronous functions, follow Vapi's [function tool contract](/tools/custom-tools); accepting work is not a final result. Keep operation status in your service until completion. + +Return concise, unambiguous information: the requested operation, its status, and the verified outcome. A queued booking should return a pending status, not a booking confirmation. + +Tools may execute concurrently. Make dependent actions explicit: availability, caller confirmation, and booking have prerequisites even when unrelated reads can overlap. Your service should enforce those prerequisites. + +## Add booking safely + +Keep lookup and booking as separate operations. The booking service should validate the selected slot, caller authorization, and current confirmation before creating the record. + +Use an operation identifier to make retries safe. If a request times out, query its status before creating another booking. Report an unknown outcome until your service can establish success or failure. + +A spoken correction is new conversational information. Your backend must determine whether a pending operation can be cancelled, changed, or reconciled. See [Design GPT-Live conversations](/gpt-live/conversation-design). + +## End or transfer the call + +Include `endCall` when the assistant should hang up. Tell the speaker to delegate that request and the reasoner to use the tool. + +For `transferCall`, configure allowed destinations. Carrier acceptance does not prove the destination answered. Avoid describing a transfer as a completed conversation with the recipient until you have evidence of that outcome. + +For background on separating voice interaction from task execution, see OpenAI's [delegation guide](https://developers.openai.com/api/docs/guides/live-delegation). Use Vapi tool definitions for this integration, rather than copying OpenAI session event examples. diff --git a/fern/gpt-live/voices.mdx b/fern/gpt-live/voices.mdx new file mode 100644 index 000000000..d8584d525 --- /dev/null +++ b/fern/gpt-live/voices.mdx @@ -0,0 +1,83 @@ +--- +title: GPT-Live voices and languages +subtitle: Compare voice previews and configure the spoken language +description: Compare all 22 GPT-Live voices in Vapi, including regional tags and Portuguese previews, then configure voice IDs, language instructions, and pronunciation. +slug: gpt-live/voices +--- + +Choose a voice with `voice.provider: "openai"` and `voice.voiceId`. GPT-Live supports 22 preset voices in Vapi. Use the dashboard's voice previews or the recordings below to compare them. + +## Configure a voice + +Set this object on your assistant: + +```json +{ + "voice": { + "provider": "openai", + "voiceId": "bossa" + } +} +``` + +Publish the updated assistant and start a new call to test the change. This integration does not expose an in-call voice change. + +## Voices with language and regional tags + +The tags describe the voice's language, regional influence, presentation, and source. Regional influence does not guarantee accent fidelity. Bossa and Tempo previews are in Brazilian Portuguese; the other previews are in English. + +| Voice | API ID | Language | Regional influence | Presentation | Source | Preview | +| --- | --- | --- | --- | --- | --- | --- | +| Quartz | `quartz` | English | Australian | Feminine | Generated | [Listen to Quartz](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/quartz.mp3) | +| Ripple | `ripple` | English | Australian | Masculine | Natural | [Listen to Ripple](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/ripple.mp3) | +| Vesper | `vesper` | English | British | Masculine | Natural | [Listen to Vesper](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/vesper.mp3) | +| Willow | `willow` | English | Irish | Feminine | Natural | [Listen to Willow](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/willow.mp3) | +| Stone | `stone` | English | Irish | Masculine | Natural | [Listen to Stone](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/stone.mp3) | +| Gleam | `gleam` | English | North American | Feminine | Natural | [Listen to Gleam](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/gleam.mp3) | +| Meridian | `meridian` | English | North American | Masculine | Natural | [Listen to Meridian](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/meridian.mp3) | +| Bossa | `bossa` | Portuguese | Brazilian | Feminine | Natural | [Listen to Bossa](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/bossa.mp3) | +| Tempo | `tempo` | Portuguese | Brazilian | Masculine | Natural | [Listen to Tempo](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/tempo.mp3) | +| Beacon | `beacon` | English | Filipino | Masculine | Generated | [Listen to Beacon](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/beacon.mp3) | +| Delta | `delta` | English | Southern U.S. | Feminine | Generated | [Listen to Delta](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/delta.mp3) | +| Cinder | `cinder` | English | Southern U.S. | Masculine | Generated | [Listen to Cinder](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/cinder.mp3) | + +## Additional supported voices + +These voices remain available. Their previews use the existing recordings. + +| Voice | API ID | Preview | +| --- | --- | --- | +| Alloy | `alloy` | [Listen to Alloy](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/alloy.mp3) | +| Ash | `ash` | [Listen to Ash](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/ash.mp3) | +| Ballad | `ballad` | [Listen to Ballad](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/ballad.mp3) | +| Cedar | `cedar` | [Listen to Cedar](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/cedar.mp3) | +| Coral | `coral` | [Listen to Coral](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/coral.mp3) | +| Echo | `echo` | [Listen to Echo](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/echo.mp3) | +| Marin | `marin` | [Listen to Marin](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/marin.mp3) | +| Sage | `sage` | [Listen to Sage](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/sage.mp3) | +| Shimmer | `shimmer` | [Listen to Shimmer](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/shimmer.mp3) | +| Verse | `verse` | [Listen to Verse](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/verse.mp3) | + +## Set the spoken language + +A voice choice does not replace language instructions. Write the speaker prompt in the language you want the assistant to use, and test both its greeting and its responses. + +For a Brazilian Portuguese assistant using Bossa or Tempo, this model fragment sets a clear language policy: + +```json +{ + "speaker": { + "instructions": "Você ajuda a pessoa a agendar um atendimento. Fale em português do Brasil, com respostas curtas e claras. Faça uma pergunta por vez. Encaminhe ao backend pedidos que precisem consultar dados ou usar ferramentas. Só confirme um agendamento depois de receber a confirmação do backend." + } +} +``` + +Use the same language for `firstMessage`, such as “Olá! Como posso ajudar hoje?” Configure the reasoner's business rules and tools separately. + +## Test pronunciation and speaking style + +Include real task vocabulary in your tests: names, addresses, dates, and product terms. Add pronunciation guidance to the speaker prompt when needed, then evaluate the spoken result. + +A preview is an example recording, not a promise that every response has identical pacing or expression. Compare voices using the same prompt and representative calls. + +See OpenAI's [voice options](https://developers.openai.com/api/docs/guides/live-conversations#voice-options) for the source metadata and [language guidance](https://developers.openai.com/api/docs/guides/live-prompting#language-and-pronunciation) for prompting recommendations. Vapi's supported catalog is the list on this page; upstream custom-voice capabilities do not imply support in this integration. diff --git a/fern/providers/gpt-live.mdx b/fern/providers/gpt-live.mdx new file mode 100644 index 000000000..2a34ba9ad --- /dev/null +++ b/fern/providers/gpt-live.mdx @@ -0,0 +1,28 @@ +--- +title: GPT-Live +subtitle: Connect a full-duplex voice model to reasoning and tools in Vapi +description: Connect GPT-Live in Vapi, configure the OpenAI speaker and reasoner, and find the quickstart, voice catalog, tool guide, and compatibility reference. +slug: providers/gpt-live +--- + +## What is GPT-Live? + +GPT-Live is an OpenAI voice model that listens and speaks at the same time. In Vapi, the speaker handles conversation while a separate reasoner handles delegated tasks and tools. See [GPT-Live architecture](/gpt-live/overview) for the responsibilities of each component. + +## Connect GPT-Live + +Your organization needs GPT-Live enabled. If you bring an OpenAI key, it must have access to GPT-Live and the selected reasoning model. + +Follow the [GPT-Live quickstart](/gpt-live/quickstart) to create an assistant and make a web call. Add [tools](/gpt-live/tools) when you are ready to connect your services. + +## Settings reference + +| Setting | Configuration | +| --- | --- | +| Voice model | `model.provider: "openai"` and `model.model: "gpt-live-1"` | +| Speaker behavior | `model.speaker.instructions` | +| Reasoner behavior | `model.reasoner.instructions` | +| Reasoner selection | `model.reasoner.model` and `model.reasoner.reasoningEffort` | +| Voice | `voice.provider: "openai"` and a [supported voice ID](/gpt-live/voices) | + +Use the [configuration and compatibility reference](/gpt-live/configuration) for defaults, transports, tool restrictions, and settings that do not apply. Learn to [design conversations](/gpt-live/conversation-design) and [test a migration](/gpt-live/testing) before moving an existing assistant. From 374fd5116834093dda170479772977d36c565fa5 Mon Sep 17 00:00:00 2001 From: dev-seth-vapi Date: Tue, 15 Sep 2026 15:53:53 -0700 Subject: [PATCH 2/2] docs: refine GPT-Live guides and conversation design --- fern/docs.yml | 40 +-- fern/gpt-live/configuration.mdx | 387 ++++++++++++++++++++------ fern/gpt-live/conversation-design.mdx | 75 ----- fern/gpt-live/design.mdx | 190 +++++++++++++ fern/gpt-live/limitations.mdx | 117 ++++++++ fern/gpt-live/overview.mdx | 92 +++--- fern/gpt-live/prompts.mdx | 98 ------- fern/gpt-live/quickstart.mdx | 121 -------- fern/gpt-live/testing.mdx | 112 -------- fern/gpt-live/tools.mdx | 125 --------- fern/gpt-live/voices.mdx | 83 ------ fern/providers/gpt-live.mdx | 28 -- 12 files changed, 671 insertions(+), 797 deletions(-) delete mode 100644 fern/gpt-live/conversation-design.mdx create mode 100644 fern/gpt-live/design.mdx create mode 100644 fern/gpt-live/limitations.mdx delete mode 100644 fern/gpt-live/prompts.mdx delete mode 100644 fern/gpt-live/quickstart.mdx delete mode 100644 fern/gpt-live/testing.mdx delete mode 100644 fern/gpt-live/tools.mdx delete mode 100644 fern/gpt-live/voices.mdx delete mode 100644 fern/providers/gpt-live.mdx diff --git a/fern/docs.yml b/fern/docs.yml index 900952cd6..50265c94f 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -125,22 +125,14 @@ navigation: - section: GPT-Live contents: - - page: Architecture + - page: Overview path: gpt-live/overview.mdx - - page: Quickstart - path: gpt-live/quickstart.mdx - - page: Conversation design - path: gpt-live/conversation-design.mdx - - page: Speaker and reasoner - path: gpt-live/prompts.mdx - - page: Tools and business logic - path: gpt-live/tools.mdx - - page: Voices and languages - path: gpt-live/voices.mdx - - page: Configuration and compatibility + - page: Design your assistant + path: gpt-live/design.mdx + - page: Configuration path: gpt-live/configuration.mdx - - page: Migration and testing - path: gpt-live/testing.mdx + - page: Limitations and FAQs + path: gpt-live/limitations.mdx - section: Assistants collapsed: open-by-default @@ -752,8 +744,6 @@ navigation: collapsed: true icon: fa-light fa-link contents: - - page: GPT-Live - path: providers/gpt-live.mdx - section: Transcribers (Speech-to-text) path: providers/transcriber/overview.mdx icon: fa-light fa-microphone @@ -1082,6 +1072,24 @@ navigation: - tab: changelog redirects: + - source: /providers/gpt-live + destination: /gpt-live/overview + - source: /gpt-live/quickstart + destination: /gpt-live/configuration#create-an-assistant + - source: /gpt-live/voices + destination: /gpt-live/configuration#voices + - source: /gpt-live/prompts + destination: /gpt-live/design#speaker-and-reasoner-roles + - source: /gpt-live/conversation-design + destination: /gpt-live/design + - source: /gpt-live/operations + destination: /gpt-live/configuration#review-and-monitor-calls + - source: /gpt-live/tools + destination: /gpt-live/configuration#connect-your-tools + - source: /gpt-live/testing + destination: /gpt-live/design#testing-conversations + - source: /gpt-live/faqs + destination: /gpt-live/limitations - source: /test/test-suites destination: /test/voice-testing - source: /test/chat-testing diff --git a/fern/gpt-live/configuration.mdx b/fern/gpt-live/configuration.mdx index fcb43f032..95f11b4b2 100644 --- a/fern/gpt-live/configuration.mdx +++ b/fern/gpt-live/configuration.mdx @@ -1,32 +1,254 @@ --- -title: GPT-Live configuration and compatibility reference -subtitle: Look up supported fields, transports, and feature restrictions -description: Look up GPT-Live model settings, defaults, voice configuration, call transports, tools, greetings, recording behavior, and unsupported Vapi features. +title: Configure GPT-Live +subtitle: Create an assistant, choose its voice, and connect your tools +description: Configure a GPT-Live assistant in Vapi using the dashboard or API, with speaker and reasoner settings, voice samples, tools, call connections, and analysis. slug: gpt-live/configuration --- -Configure GPT-Live through a Vapi assistant with `model.provider: "openai"`, `model.model: "gpt-live-1"`, and an OpenAI voice. The tables below describe this integration's supported behavior. +Create an assistant that checks appointment availability and reads back the results. You'll configure the speaker and reasoner, connect a lookup tool, and make a test call. -## Model and voice fields +For help deciding what to put in each prompt, read [Design your assistant](/gpt-live/design). For feature restrictions, use [Limitations and FAQs](/gpt-live/limitations). -These fields belong to the assistant configuration. See [Create Assistant](/api-reference/assistants/create) for the complete generated schema. +**Jump to:** [Create an assistant](#create-an-assistant) · [Model settings](#model-settings) · [Personality](#personality-and-language) · [Voices](#voices) · [Tools](#connect-your-tools) · [Calls](#connect-a-call) · [Analysis](#review-and-monitor-calls) + +## Before you start + +- Enable GPT-Live for your Vapi organization. +- Have a browser and microphone ready for a test call. +- For API requests, use a Vapi private API key on your server or local machine. +- If you bring an OpenAI key, it needs access to GPT-Live and the reasoner you select. +- Prepare an HTTPS availability endpoint using the [tool response contract](#connect-your-tools). + +The example uses a placeholder endpoint. Replace it with your handler before testing a lookup. It returns availability only. Booking requires a separate tool. + +## Create an assistant + +Choose the dashboard or API. Both paths create an assistant with an availability lookup and an end-call tool. + + + + + +Open [Assistants](https://dashboard.vapi.ai/assistants), select **Create Assistant**, and choose **GPT-Live**. Choose Marin as the voice. + + +In **Speaker**, enter: + +```text title="Speaker instructions" +Help callers check appointment availability. Speak warmly and briefly. +Ask for an explicit date and time zone, then delegate the lookup. +If the caller changes the request, use the updated details. +Explain the returned slots. Booking isn't connected yet. +Delegate requests to end the call. +``` + +In **Reasoner**, enter: + +```text title="Reasoner instructions" +Use lookupAvailability after the caller provides an explicit date +and time zone. Ask for clarification if either is missing or ambiguous. +Return only slots from the current lookup. Explain failures without +inventing results. No booking tool is available. +Call endCall when the caller asks to finish. +``` + + +Create a function tool named `lookupAvailability` with the description “Check available appointment slots. Does not reserve or book a slot.” Add these required arguments: + +| Argument | Type | Description | +| --- | --- | --- | +| `date` | String | Requested calendar date in YYYY-MM-DD format | +| `timeZone` | String | IANA time zone, for example `America/New_York` | + +Set the tool's Server URL to your HTTPS handler. + +Attach the function tool and an `endCall` tool to the assistant. You can also create a saved tool through [Create Tool](/api-reference/tools/create) and select it in the dashboard. + + +Set the first message to “Hello. I can check appointment availability. What date and time zone should I check?” and configure the assistant to speak first. + +Select **Publish**, review the changes, and complete the publication flow. Keep the assistant ID for API calls. + + + + + + +Save the following as `availability-assistant.json`: + +```json title="availability-assistant.json" +{ + "name": "Appointment availability assistant", + "model": { + "provider": "openai", + "model": "gpt-live-1", + "speaker": { + "instructions": "You help callers check appointment availability. Speak briefly. Collect an explicit calendar date and time zone, then delegate availability checks to the reasoner. If the caller corrects the date, delegate the corrected request and do not present old results as current. Report only confirmed lookup results. Availability does not reserve a slot. Booking is not connected. Delegate requests to end the call." + }, + "reasoner": { + "model": "gpt-5.6-terra", + "reasoningEffort": "low", + "instructions": "Use lookupAvailability only after the caller supplies an explicit calendar date and time zone. Ask for clarification if either is missing or ambiguous. Report only slots returned for the current request. Ignore superseded lookup results. A successful lookup does not reserve or book anything. Explain lookup failures without inventing slots. No booking tool is available. Call endCall when the caller asks to finish." + }, + "tools": [ + { + "type": "function", + "function": { + "name": "lookupAvailability", + "description": "Check available appointment slots. Does not reserve or book a slot.", + "parameters": { + "type": "object", + "properties": { + "date": { + "type": "string", + "description": "Requested calendar date in YYYY-MM-DD format." + }, + "timeZone": { + "type": "string", + "description": "IANA time zone, such as America/New_York." + } + }, + "required": [ + "date", + "timeZone" + ] + } + }, + "server": { + "url": "https://api.example.com/vapi/tools" + } + }, + { + "type": "endCall" + } + ] + }, + "voice": { + "provider": "openai", + "voiceId": "marin" + }, + "firstMessageMode": "assistant-speaks-first", + "firstMessage": "Hello. I can check appointment availability. What date and time zone should I check?", + "maxDurationSeconds": 600 +} +``` + + + +Set `VAPI_PRIVATE_API_KEY` to your Vapi private API key, then create the assistant: + +```bash +curl --fail-with-body https://api.vapi.ai/assistant \ + -H "Authorization: Bearer $VAPI_PRIVATE_API_KEY" \ + -H "Content-Type: application/json" \ + --data-binary @availability-assistant.json +``` + +Keep the returned `id`. To change an existing assistant, use [Update Assistant](/api-reference/assistants/update), preserving its other tools and settings. + + + + + +### Make a test call + +Open the saved assistant in the dashboard and select **Talk**. Allow microphone access, then ask for availability on an explicit date in a specific time zone. + +Check that your handler receives the right values and the assistant reads back the returned slots. Ask to book a time. It should explain that booking isn't connected. Finish with “Please end the call” and confirm that the call ends. + +## Model settings + +These fields belong to the assistant object. See [Create Assistant](/api-reference/assistants/create) for the full schema. | Field | Values or behavior | | --- | --- | | `model.provider` | `openai` | | `model.model` | `gpt-live-1` | -| `model.speaker.instructions` | Speaker prompt; see [instruction precedence](/gpt-live/prompts#understand-instruction-precedence). | +| `model.speaker.instructions` | Speaker prompt. See [prompt behavior](/gpt-live/configuration#prompt-defaults-and-overrides). | | `model.speaker.personalityPacks` | Array of `eager-listener`, `idle-hummer`, `bouncy`, and `unhurried`. | -| `model.reasoner.provider` | `openai`; defaults to OpenAI when omitted. | -| `model.reasoner.model` | `gpt-5.6-sol`, `gpt-5.6-terra`, or `gpt-5.6-luna`; defaults to `gpt-5.6-terra`. | -| `model.reasoner.reasoningEffort` | `none`, `low`, `medium`, `high`, `xhigh`, or `max`; defaults to `low`. | -| `model.reasoner.instructions` | Custom reasoner prompt; omission uses Vapi's default. | +| `model.reasoner.provider` | `openai`. Defaults to OpenAI when omitted. | +| `model.reasoner.model` | `gpt-5.6-sol`, `gpt-5.6-terra`, or `gpt-5.6-luna`. Defaults to `gpt-5.6-terra`. | +| `model.reasoner.reasoningEffort` | `none`, `low`, `medium`, `high`, `xhigh`, or `max`. Defaults to `low`. | +| `model.reasoner.instructions` | Custom reasoner prompt. Omission uses Vapi's default. | | `model.tools` | Inline supported tool definitions. | | `model.toolIds` | IDs of saved tools to resolve for the assistant. | | `voice.provider` | `openai` | -| `voice.voiceId` | One of the [22 supported voices](/gpt-live/voices). | +| `voice.voiceId` | One of the [22 supported voices](/gpt-live/configuration#voices). | + +## Prompt defaults and overrides + +Set `model.speaker.instructions` for the spoken conversation and `model.reasoner.instructions` for task procedures. + +- Explicit speaker instructions take precedence over `model.systemPrompt` and system messages in `model.messages`. If you omit the speaker instructions, Vapi uses that legacy configuration. +- Omitting reasoner instructions uses Vapi's default reasoner prompt. Custom instructions replace that prompt entirely. +- An explicit empty string stays empty. Updating one prompt doesn't update the other. + +Personality packs append guidance to the speaker prompt. Set `model.speaker.personalityPacks` to an array of the IDs above and test one pack at a time. + +## Personality and language + +Personality packs append speaking-style guidance to your speaker prompt. Add this inside the assistant's `model` object: + +```json +{ + "speaker": { + "instructions": "Speak warmly and briefly. Slow down for appointment dates and times. Follow requests to speed up or explain more. Delegate requests that need tools.", + "personalityPacks": ["unhurried"] + } +} +``` + +Available IDs are `eager-listener`, `idle-hummer`, `bouncy`, and `unhurried`. See [Speaking style](/gpt-live/design#speaking-style) for what each pack encourages and how to combine personality with pacing. These are prompt instructions, not fixed speed controls. + +Write the speaker prompt and `firstMessage` in the language you want the assistant to use. For Bossa or Tempo, you can include: + +```text +Fale em português do Brasil, com respostas curtas e claras. +Faça uma pergunta por vez. Só confirme um agendamento +quando a ferramenta confirmar que ele foi realizado. +``` + +Keep tool and delegation instructions in the prompt too. Try names, dates, and product terms in real calls, and add pronunciation guidance where needed. + +## Voices + +Choose any of these 22 voices with `voice.provider: "openai"` and `voice.voiceId`. Save the assistant and start a new call to change voices. In-call voice changes aren't supported. -A custom prompt replaces the corresponding fallback prompt. An explicit empty string is preserved. See [Configure the speaker and reasoner](/gpt-live/prompts) before overriding defaults. +### Voices with regional tags + +Use the tags to narrow your choices, then listen to the samples. Bossa and Tempo previews are in Brazilian Portuguese. The other previews are in English. Test your preferred voice with your own prompts to hear how it handles the accent and vocabulary you need. + +| Voice | API ID | Language | Regional influence | Presentation | Source | Preview | +| --- | --- | --- | --- | --- | --- | --- | +| Quartz | `quartz` | English | Australian | Feminine | Generated | [Listen to Quartz](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/quartz.mp3) | +| Ripple | `ripple` | English | Australian | Masculine | Natural | [Listen to Ripple](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/ripple.mp3) | +| Vesper | `vesper` | English | British | Masculine | Natural | [Listen to Vesper](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/vesper.mp3) | +| Willow | `willow` | English | Irish | Feminine | Natural | [Listen to Willow](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/willow.mp3) | +| Stone | `stone` | English | Irish | Masculine | Natural | [Listen to Stone](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/stone.mp3) | +| Gleam | `gleam` | English | North American | Feminine | Natural | [Listen to Gleam](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/gleam.mp3) | +| Meridian | `meridian` | English | North American | Masculine | Natural | [Listen to Meridian](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/meridian.mp3) | +| Bossa | `bossa` | Portuguese | Brazilian | Feminine | Natural | [Listen to Bossa](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/bossa.mp3) | +| Tempo | `tempo` | Portuguese | Brazilian | Masculine | Natural | [Listen to Tempo](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/tempo.mp3) | +| Beacon | `beacon` | English | Filipino | Masculine | Generated | [Listen to Beacon](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/beacon.mp3) | +| Delta | `delta` | English | Southern U.S. | Feminine | Generated | [Listen to Delta](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/delta.mp3) | +| Cinder | `cinder` | English | Southern U.S. | Masculine | Generated | [Listen to Cinder](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/cinder.mp3) | + +### More voices + +You can also choose any of these voices. Select a preview to listen. + +| Voice | API ID | Preview | +| --- | --- | --- | +| Alloy | `alloy` | [Listen to Alloy](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/alloy.mp3) | +| Ash | `ash` | [Listen to Ash](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/ash.mp3) | +| Ballad | `ballad` | [Listen to Ballad](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/ballad.mp3) | +| Cedar | `cedar` | [Listen to Cedar](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/cedar.mp3) | +| Coral | `coral` | [Listen to Coral](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/coral.mp3) | +| Echo | `echo` | [Listen to Echo](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/echo.mp3) | +| Marin | `marin` | [Listen to Marin](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/marin.mp3) | +| Sage | `sage` | [Listen to Sage](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/sage.mp3) | +| Shimmer | `shimmer` | [Listen to Shimmer](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/shimmer.mp3) | +| Verse | `verse` | [Listen to Verse](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/verse.mp3) | ## Greetings and duration @@ -36,24 +258,64 @@ Use a text greeting if the assistant should begin the conversation. | --- | --- | | `firstMessageMode: "assistant-speaks-first"` | Speaks the configured text greeting. | | `firstMessageMode: "assistant-waits-for-user"` | Waits for the caller. | -| `firstMessage` | Text greeting; without usable text, the assistant waits. | +| `firstMessage` | Text greeting. Without usable text, the assistant waits. | | `maxDurationSeconds` | Maximum call duration in seconds. | -| Audio URL greeting | Unsupported; not played as a greeting. | -| Generated-message first-message mode | Unsupported; the assistant waits for the caller. | +| Audio URL greeting | Unsupported. Not played as a greeting. | +| Generated-message first-message mode | Unsupported. The assistant waits for the caller. | Generated speech may vary from the supplied greeting. A text greeting is not a guarantee of exact prerecorded playback. -## Call transports +## Connect your tools -Use the [Create Call endpoint](/api-reference/calls/create) or the dashboard's web-call interface. +Handle Vapi's [`tool-calls` server message](/server-url/events) at your HTTPS endpoint. Read `message.toolCallList`, validate each function's arguments, and query your availability service. -| Transport | Support | Requirements | -| --- | --- | --- | -| WebRTC web call | Supported | Dashboard **Talk** or Vapi web-call interface. | -| `vapi.websocket` | Supported with restrictions | Raw, mono PCM16 audio at 24 kHz. | -| `vapi.sip` | Supported | Vapi SIP transport configuration. | -| Native Twilio | Supported | A configured Twilio phone number and call transport. | -| Native Telnyx or Vonage | Unsupported | Choose a supported transport. | +Return a result for each request using its actual `toolCallId`: + +```json +{ + "results": [ + { + "name": "lookupAvailability", + "toolCallId": "TOOL_CALL_ID_FROM_REQUEST", + "result": "There are openings at 2 PM and 4 PM on the requested date in the requested time zone. No appointment has been booked." + } + ] +} +``` + +Replace the sample result with your service's data. If the lookup fails, return the failure so the assistant can explain it. See [Server authentication](/server-url/server-authentication) to protect the endpoint. + +Supported tool types are `function`, `apiRequest`, `endCall`, `dtmf`, and `transferCall`. Give tools unique resolved function names. Save reusable tools and attach them through `model.toolIds`, or include definitions in `model.tools`. + +Use function or API request tools to connect external data sources. For phone actions, check [transfer and keypad support](/gpt-live/limitations#transfers-and-keypad-input). + +### Slow and asynchronous requests + +A synchronous function waits for your handler's result. Set `async: true` on a function tool to let the conversation continue while that request completes. Return the final result through the original request, using the matching `toolCallId`. + +If your handler returns “queued” before the work finishes, add a status lookup tool. Tell the reasoner when to check it. A later callback doesn't automatically update the conversation. + +Tools can run concurrently. Keep dependent actions in order: check availability, confirm the caller's choice, then book. Your service should validate those requirements too. + +## Connect a call + +Start with dashboard **Talk**, then connect your intended call channel. GPT-Live supports browser WebRTC, native Twilio, Vapi SIP, and raw WebSocket audio. See [connection limits](/gpt-live/limitations#calls-and-voice) before changing an existing setup. + +### Phone calls + +For inbound Twilio calls, [import a Twilio phone number](/phone-numbers/import-twilio) and assign your assistant to it. For SIP, follow the [SIP guide](/advanced/sip). + +For an outbound Twilio test, send this body to [Create Call](/api-reference/calls/create): + +```json +{ + "assistantId": "YOUR_ASSISTANT_ID", + "phoneNumberId": "YOUR_TWILIO_PHONE_NUMBER_ID", + "customer": {"number": "+14155550100"} +} +``` + +Replace the customer phone number with one you control. `phoneNumberId` is the Vapi ID of the imported Twilio phone number. ### WebSocket configuration @@ -75,71 +337,36 @@ Send this body to `POST https://api.vapi.ai/call` with your Vapi private API key Connect to `transport.websocketCallUrl` in the response. Send and receive binary audio frames containing signed 16-bit little-endian samples at 24 kHz. Convert your microphone audio to that format before sending it. -The default WebSocket sample rate is not sufficient for GPT-Live; set `24000` explicitly. This audio requirement applies to the Vapi WebSocket transport, not to browser-negotiated WebRTC audio or carrier audio. +Set `24000` explicitly. The default WebSocket rate is too low for GPT-Live. Browser WebRTC and phone connections negotiate their own audio formats. Close the WebSocket to end the call, or have the assistant invoke `endCall`. Incoming JSON call-control commands are not supported by this GPT-Live connection. See [WebSocket transport](/calls/websocket-transport) for the general transport, applying the restrictions above. -## Tool and transfer compatibility - -The supported tool types are `function`, `apiRequest`, `endCall`, `dtmf`, and `transferCall`. All other tool types are rejected. +## Review and monitor calls -| Feature | Support or restriction | -| --- | --- | -| Function tools | Synchronous and asynchronous functions. | -| API request tools | Configured HTTP requests. | -| Outgoing DTMF | Supported on SIP using RTP DTMF. Use digits as a string, such as `"0011*#"`; native Twilio, web calls, and raw WebSocket calls do not support this tool. | -| SIP INFO DTMF | Unsupported; use RTP DTMF on SIP. | -| Incoming keypad collection | Rejected when `keypadInputPlan.enabled` is true. | -| Blind transfer | Supported on native Twilio and Vapi SIP to configured phone or SIP destinations. | -| Warm transfer or generated transfer summary | Rejected. | -| Transfer fallback plans or SIP `bye` verb | Rejected. | -| SIP transfer with extension dialing | Unsupported; use a direct destination. | -| Squads and assistant handoffs | Unsupported. | - -A configured transfer tool does not make every transport transfer-capable. Web calls and raw WebSocket calls do not gain phone transfer support by adding the tool. Carrier acceptance of a transfer does not confirm that the destination answered. - -## Settings that do not apply - -GPT-Live handles speech directly. Classic transcriber selection, text-to-speech controls, and start/stop-speaking plans do not configure this audio conversation. - -General model fields such as `temperature` and `maxTokens` do not tune this integration. Use the speaker prompt for conversational behavior and `model.reasoner.reasoningEffort` for reasoning effort. - -## Unsupported features and artifact behavior +Completed GPT-Live calls use Vapi's shared post-call processing. Configure the features you need, then inspect the results after a test call finishes. -Check these restrictions before moving an existing assistant: - -| Feature | Behavior | +| Feature | Setup and results | | --- | --- | -| Model fallbacks | Rejected. GPT-Live also cannot be a fallback model. | -| Voice fallbacks | Rejected. | -| Non-OpenAI or custom voice IDs | Rejected unless included in the supported Vapi voice catalog. | -| Live message injection and call-control commands | Unsupported for this integration. | -| Listen or monitor sockets | Unsupported. | -| Reconnecting to an existing GPT-Live session | Unsupported. Start a new call and reconcile pending work in your service. | -| Recording disabled in the assistant | No recording is created. | -| Recording-consent plan configured | GPT-Live recording is disabled because this integration does not collect that consent. | - -Call artifacts and analysis are separate from your service's authoritative operation records. A transcript cannot by itself establish that a booking succeeded. See [Migrate and test GPT-Live](/gpt-live/testing). +| [Structured outputs](/assistants/structured-outputs-quickstart) | Create output definitions, attach their IDs through `artifactPlan.structuredOutputIds`, and read `call.artifact.structuredOutputs`. Keep existing IDs and artifact settings when updating. | +| [Call analysis](/assistants/call-analysis) | Existing `analysisPlan` configurations work. Prefer structured outputs for new setups. | +| [Scorecards](/observability/scorecard-quickstart) | Grade boolean or numeric outputs and attach the scorecard to your assistant. Read `call.artifact.scorecards`. | +| [Boards](/observability/boards-quickstart) | Track available call metrics and outcome trends for your assistant. Classic transcriber and text-to-speech timing fields don't describe GPT-Live. | +| [Monitoring](/observability/monitoring-quickstart) | Configure supported conditions, thresholds, evaluation windows, and notification destinations. Requires retained data. Unavailable with Zero Data Retention. | +| [End-of-call reports](/server-url/events) | Configure a Server URL and the `end-of-call-report` server message. [Authenticate your endpoint](/server-url/server-authentication) and handle repeated delivery by call ID. | -## Post-call analysis and observability +Keep transcript collection enabled for transcript-based checks. Audio-based checks require a recording. See [recording and data restrictions](/gpt-live/limitations#recordings-and-post-call-data) before relying on those artifacts. -Completed GPT-Live calls use Vapi's shared post-call processing. Configure analysis and observability on the assistant and inspect the resulting call data after processing finishes. +Retrieve a completed call with your private API key: -| Capability | Configuration or result | Requirement | -| --- | --- | --- | -| Structured outputs | Attach IDs through `artifactPlan.structuredOutputIds`; read `call.artifact.structuredOutputs`. | Create the output definitions and retain the conversation data needed for extraction. | -| Legacy call analysis | Configure `analysisPlan`. | Existing configurations remain supported; prefer structured outputs for new extraction and evaluation setups. | -| Scorecards | Grade structured output values; read `call.artifact.scorecards`. | Configure a scorecard and its required structured outputs. | -| Boards | Query call data and plot outcomes. | Choose metrics available on your calls; classic pipeline timing fields do not describe GPT-Live's architecture. | -| Monitoring | Configure monitors, thresholds, and notifiers. | Retained call data is required; monitoring is unavailable with Zero Data Retention. | -| End-of-call reports | Receive configured `end-of-call-report` server messages. | Configure the assistant's Server URL and requested server messages. | - -See [Call analysis](/assistants/call-analysis), [Scorecards](/observability/scorecard-quickstart), and [Monitoring](/observability/monitoring-quickstart) for setup. These are post-call capabilities; they do not imply access to live listen sockets or message injection. +```bash +curl --fail-with-body "https://api.vapi.ai/call/$CALL_ID" \ + -H "Authorization: Bearer $VAPI_PRIVATE_API_KEY" +``` -An extracted statement such as “booking confirmed” describes the conversation. Verify the booking in your service when measuring actual task success. Analysis results do not replace the authoritative operation record. +Review the transcript and recording alongside tool results. A missing analysis result isn't a pass. Check attached output IDs, processing, and available artifacts. Creating a scorecard doesn't automatically configure a monitor. -## Credentials and cost +When testing a migration, keep the previous assistant available so you can route new calls back. End test calls through the call interface, `endCall`, or by closing your WebSocket. Check any external work still running. Ending the call doesn't undo it. -If you bring your own OpenAI key, it needs access to GPT-Live and the selected reasoner. Organization access to the Vapi feature is a separate prerequisite. +## Cost -Review Vapi call costs for your configuration. OpenAI's [model pricing](https://developers.openai.com/api/docs/models/gpt-live-1) describes direct provider charges; it is not a quote for your complete Vapi call. Reasoning, tools, and telephony can contribute additional cost. +Review Vapi call costs for your configuration. Reasoning, tools, and telephony can contribute to the total. OpenAI's [model pricing](https://developers.openai.com/api/docs/models/gpt-live-1) describes provider charges rather than the complete cost of a Vapi call. diff --git a/fern/gpt-live/conversation-design.mdx b/fern/gpt-live/conversation-design.mdx deleted file mode 100644 index e1e0a3968..000000000 --- a/fern/gpt-live/conversation-design.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: Design GPT-Live conversations -subtitle: Handle corrections, overlapping speech, and work in progress -description: Design GPT-Live conversations around clarification, interruptions, pending tools, corrected requests, confirmed outcomes, and recoverable business actions. -slug: gpt-live/conversation-design ---- - -Design the conversation around the caller's current intent and the status of the task. The speaker can keep talking while work proceeds, so an acknowledgment must stay distinct from a confirmed result. - -## Track conversation and action separately - -Three events can happen at different times: the caller finishes speaking, the tool returns, and the caller hears the result. Treat each as evidence of its own outcome. - -| What happens | What it establishes | What it does not establish | -| --- | --- | --- | -| Caller interrupts | New speech needs attention | A pending operation was cancelled | -| Assistant says “I'll check” | It acknowledged the request | A lookup started or succeeded | -| Tool reports available slots | The lookup returned availability | A slot is reserved | -| Booking service confirms an operation | The booking succeeded | The caller heard the confirmation | -| Connection closes | The conversation ended | External work was rolled back | - -## Handle a correction during a lookup - -Consider this exchange: - -> **Caller:** What do you have on Thursday? -> **Assistant:** I'll check. -> **Caller:** Actually, Friday afternoon. - -The corrected task is Friday afternoon. The speaker should acknowledge that change without reading out Thursday's results as the answer. - -Your service should associate requests with the current task. For example, keep a task revision with the requested date and time zone. Before using a returned result or executing a booking, verify that it still matches the current request. This is an application design pattern, not a Vapi configuration field. - -If the Thursday lookup cannot be cancelled, let it finish and mark its result as outdated. If an action already changed a record, report that state and use your service's change or cancellation procedure. - -## Ask for the missing decision - -Ask one short question that makes the next action possible. An appointment request may need a date, time zone, appointment type, or confirmation of a specific slot. - -Prefer “Which time zone should I use?” to asking the caller to repeat the entire request. When a correction invalidates a previous confirmation, obtain a new confirmation for the revised action. - -Short acknowledgments can keep a conversation understandable. Avoid instructions that require a filler phrase before every tool call; evaluate whether the assistant's timing is useful in real audio. - -## Explain work in progress accurately - -Use language that matches what the service knows: - -| Task status | Example spoken response | -| --- | --- | -| Lookup pending | “I'm checking Friday afternoon.” | -| Slots returned | “There are openings at two and four. Which works for you?” | -| Booking pending | “I'm submitting the two o'clock appointment.” | -| Booking confirmed | “Your two o'clock appointment is booked.” | -| Outcome unknown | “I haven't confirmed whether that booking went through yet.” | -| Cancellation confirmed | “The appointment has been cancelled.” | - -The examples illustrate meaning, not guaranteed verbatim speech. Choose one useful update when work is slow instead of repeatedly announcing the same pending state. - -## Treat cancellation as an operation - -“Stop talking,” “stop checking,” and “cancel my appointment” have different effects. Ask a brief clarification when the intended action is unclear. - -Put cancellation and duplicate prevention in the service that owns the action. A timeout or lost response can leave an operation's outcome unknown. Reconcile that outcome before retrying a write. Use an operation identifier so a retry cannot silently create a second booking. - -## Choose where exact wording matters - -GPT-Live generates speech and can paraphrase supplied text. Test language, pronunciation, and required phrases with audio; a matching text prompt does not establish verbatim delivery. - -If your application requires an exact prerecorded opening or approval before any audio plays, check the [compatibility reference](/gpt-live/configuration) before choosing this integration. Do not assume that withholding a reasoner result pauses the speaker. - -## Evaluate the whole exchange - -Test corrections during speech, during a lookup, and immediately before a write. For each test, inspect the audio, tool results, and final record in your service. - -OpenAI's [migration guide](https://developers.openai.com/api/docs/guides/live-migration) discusses the separation between interruption and backend cancellation. The [Vapi testing guide](/gpt-live/testing) turns these cases into a repeatable comparison. diff --git a/fern/gpt-live/design.mdx b/fern/gpt-live/design.mdx new file mode 100644 index 000000000..52779ce8f --- /dev/null +++ b/fern/gpt-live/design.mdx @@ -0,0 +1,190 @@ +--- +title: Design your GPT-Live assistant +subtitle: Plan the conversation and choose how your assistant speaks +description: Design GPT-Live conversations with speaker and reasoner roles, outcome-based delegation, useful overlap during tools, flexible pacing, and personality packs. +slug: gpt-live/design +--- + +With GPT-Live, the conversation can continue while work happens in the background. A caller can add a preference during a lookup, ask a question halfway through scheduling, or ask the assistant to slow down. + +The design task is to make those moments feel connected. Decide what the assistant is trying to accomplish, how it should guide the caller, and which decisions need help from your services. + +This guide uses scheduling as an example. The same approach applies to support, intake, and other conversations that combine questions with actions. + +**Jump to:** [Architecture](#speaker-and-reasoner-roles) · [Squads](#start-with-one-assistant) · [Conversation flow](#questions-and-changes-of-topic) · [Voice design](#speaking-style) · [Testing](#testing-conversations) + +## Speaker and reasoner roles + +The **speaker** carries the spoken interaction: listening, responding, asking questions, and deciding when to delegate. The **reasoner** takes a delegated task, follows its instructions, and calls tools. A tool connects the assistant to information or actions in your service. + +Both models belong to the same assistant. The caller stays in one conversation while the reasoner works behind it. + +| Give the speaker | Give the reasoner | +| --- | --- | +| The assistant's role and the caller's goal | The procedure for completing each task | +| Tone, language, and pacing | Required information and confirmation rules | +| When to clarify or delegate | Which tools to use and in what order | +| How to explain results to the caller | How to interpret tool results and failures | + +For example, the speaker needs to know that it can help find an appointment. The reasoner needs to know which service returns availability, what information that service requires, and when booking is allowed. + +This separation lets you keep the speaker's instructions focused. Put detailed business procedures in the reasoner prompt. Your services remain responsible for enforcing permissions and validating actions. + +### When to delegate + +Give the speaker a clear reason to ask for help: “Delegate requests to find an appointment that fits the caller's preferences.” Give the reasoner the steps needed to produce that result. + +| Layer | Example instruction | +| --- | --- | +| Speaker | “Help the caller find a suitable appointment. Delegate availability checks and booking requests.” | +| Reasoner | “Confirm the date and time zone. Look up available slots. Book only after the caller confirms a specific slot.” | +| Service | Validate the requested slot and create the booking only when the request meets your requirements. | + +Keep the conditions for delegation concrete. A greeting or a request to repeat an already confirmed time may stay with the speaker. A fresh lookup, changed booking request, or end-call request needs the reasoner and its tools. + +The reasoner's result gives the speaker information for its next response. The speaker can explain that information in the context of what the caller just said. See [Configure GPT-Live](/gpt-live/configuration#create-an-assistant) for an assistant with both prompts and a lookup tool. + +## Start with one assistant + +Early testing suggests GPT-Live handles many of the needs that previously led builders to squads out of the box. The speaker maintains the conversation while the reasoner handles detailed instructions and task sequencing. You may be able to cover the full use case with one assistant. + +Suppose your scheduling squad has separate assistants for intake, availability, and booking. Start by carrying those responsibilities into one GPT-Live assistant: + +```mermaid +flowchart TD + A[One scheduling conversation] + A --> B[Speaker guides the caller] + B --> C[Reasoner manages the task] + C --> D[Collect required details] + D --> E[Look up availability] + E --> F[Book after confirmation] +``` + +This diagram shows responsibilities, not a fixed script for every call. The caller may supply several details at once or ask a question between stages. + +Preserve the outcomes and business rules you need. Test whether the new architecture still needs the divisions you created for the old one. Pay attention to missed steps and recovery from errors before consolidating a production flow. + +Existing squad configurations and assistant handoffs are [currently unsupported](/gpt-live/limitations#can-i-use-an-existing-squad). The early finding is that some use cases need fewer assistants, not that every squad can be converted automatically. + +## Questions and changes of topic + +Give the conversation a destination and a few meaningful stages: understand the request, find suitable options, confirm the choice, and explain the result. The caller doesn't need to answer questions in the order you wrote them. + +Consider this opening: + +> **Caller:** I need something Friday afternoon. I've been there before. + +The caller has already supplied a day, a preference, and a useful detail. Design the assistant to acknowledge those details and ask for the next missing piece, such as the exact date or time zone. Repeating questions the caller has answered makes even responsive speech feel mechanical. + +A detour can help the caller make the next decision: + +> **Caller:** Before we pick a time, how long does the appointment take? +> +> **Assistant:** Let me check that for you. Then we can find a time that works. + +If the duration is already available in trusted context, the speaker can explain it. If it needs a fresh lookup, delegate that question. Then return to scheduling with the caller's earlier preferences intact. + +A useful speaker instruction is: + +```text title="Keep the conversation connected" +Use details the caller has already provided. Ask one question for the +next missing piece. If the caller asks a related question, address it +and then return to the unfinished task. Delegate questions that need +new information from our services. +``` + +## Talking while tools run + +A lookup doesn't always need to bring the conversation to a stop. While it runs, the speaker can collect information that doesn't depend on the result. + +For example, if your service can retrieve a whole day's availability, the speaker can ask about morning or afternoon while that request is running. If the lookup requires that preference as an input, ask first. + +```mermaid +sequenceDiagram + participant C as Caller + participant S as Speaker + participant R as Reasoner and tools + C->>S: What is available on Friday? + S->>R: Look up Friday availability + S->>C: Morning or afternoon? + C->>S: Afternoon, ideally after two + R-->>S: Available times + S->>C: Explain matching options +``` + +This illustrates a conversation to design and test, not a guarantee of exact event timing. Your prompts and tool requirements determine which work can overlap. + +Keep updates proportionate. One short “I'm checking” can be enough. A forced acknowledgment before every tool call, repeated status updates, or unrelated small talk can make the wait feel longer. + +### Confirming results + +The assistant can acknowledge the request before the work finishes. Its confirmation should follow the result: + +| Moment | What the caller can hear | +| --- | --- | +| The request is understood | “I'll check Friday afternoon.” | +| The lookup finishes | “There are openings at two and four.” | +| A booking tool confirms success | “Your two o'clock appointment is booked.” | + +The [example assistant](/gpt-live/configuration#create-an-assistant) only looks up availability. Add a booking tool before teaching it to offer the final step. + +### When callers change the request + +If the caller says “Actually, Thursday” during the lookup, the speaker should acknowledge the new request. Your application should match each result to the request it answers so Friday's result doesn't drive Thursday's booking. + +Interrupting speech doesn't cancel an action already sent to a service. Keep changes and cancellations explicit in your application. The [limitations FAQ](/gpt-live/limitations#does-an-interruption-cancel-a-tool-call) explains this boundary. + +## Speaking style + +Choose both a voice and a way of speaking. The voice establishes a starting sound. Speaker instructions shape pacing, warmth, emphasis, and how much the assistant says at once. + +Write a short voice brief alongside the task brief. Describe behavior someone could hear and evaluate: + +```text title="Example voice brief" +Sound warm and conversational. Keep routine answers brief. +Move briskly through familiar information, but slow down when reading +appointment dates and times. Give one instruction at a time when +explaining a process. Follow requests to speed up, slow down, or +explain more. Leave room for the caller to respond. +``` + +These are prompted behaviors, not precise speed or pause controls. Test the spoken result. [Programmatic speech control](/gpt-live/limitations#can-i-inject-commentary-or-control-speech-during-a-call) is currently unsupported in Vapi's GPT-Live integration. + +### Pacing + +A consistent personality doesn't require a constant pace. The same assistant can move quickly through familiar information and become more measured when a caller needs help. + +| Moment | Delivery to try | +| --- | --- | +| Caller knows what they want | Short questions and concise answers | +| Caller is choosing between options | A few options at a time, with room to respond | +| Assistant reads a date or unfamiliar name | Slower delivery and clear articulation | +| Caller asks “Can you walk me through it?” | One step at a time | +| Caller says “Just the short version” | A brief summary with an offer to explain more | + +Treat those requests as part of the conversation design. Test “slow down,” “a little faster,” and “say that another way” alongside your business scenarios. + +### Personality packs + +Vapi's personality packs add speaking-style guidance to your speaker prompt: + +| Pack | What its guidance encourages | +| --- | --- | +| **Active listening** (`eager-listener`) | Brief acknowledgments at natural openings, with room for the caller to continue | +| **Humming** (`idle-hummer`) | Soft humming or wordless sounds during quiet moments while waiting for tools | +| **Bouncy** (`bouncy`) | More expressive pitch, emphasis, and rhythm, adapting to the caller's mood | +| **Unhurried** (`unhurried`) | A slower pace, clear articulation, and space between ideas | + +Choose a pack because it suits the experience. Humming might fit an informal concierge and feel out of place during a serious support issue. Expressiveness and speed are separate choices: an animated assistant can still speak slowly and clearly. + +Start with one pack, listen, and adjust the speaker prompt. Packs add guidance. They don't guarantee identical delivery on every call. See [personality configuration](/gpt-live/configuration#personality-and-language) and [voice previews](/gpt-live/configuration#voices). + +## Testing conversations + +Review a short set of conversations before expanding the prompt. Include a direct request, a detour, information supplied out of order, a slow lookup, and a request to change speaking style. + +Ask whether the assistant used what the caller had already said, delegated at the right moment, and returned naturally to unfinished work. Listen for whether its pace helps the caller make progress. + +Keep a few representative calls as a baseline. Repeat them when either prompt changes. Use [post-call analysis](/gpt-live/configuration#review-and-monitor-calls) to track outcomes, while reviewing audio for timing and delivery. Check [testing limitations](/gpt-live/limitations#can-i-run-simulations-and-evals) before choosing an automated test method. + +For additional model-level examples, see OpenAI's [GPT-Live prompting guide](https://developers.openai.com/api/docs/guides/live-prompting). Use Vapi's fields when applying that guidance to your assistant. diff --git a/fern/gpt-live/limitations.mdx b/fern/gpt-live/limitations.mdx new file mode 100644 index 000000000..7a20c68e0 --- /dev/null +++ b/fern/gpt-live/limitations.mdx @@ -0,0 +1,117 @@ +--- +title: GPT-Live limitations and FAQs +subtitle: Check compatibility before you build or migrate +description: Check GPT-Live support for squads, call control, commentary injection, transfers, simulations, voices, fallbacks, recordings, and post-call data in Vapi. +slug: gpt-live/limitations +--- + +GPT-Live has a different set of supported features from Vapi's other voice architectures. Use this page to check the requirements of your application before migrating. + +These limits describe **Vapi's GPT-Live integration**. A capability in OpenAI's direct API isn't necessarily exposed through Vapi. + +**Jump to:** [Calls and voices](#calls-and-voice) · [Transfers](#transfers-and-keypad-input) · [Squads](#can-i-use-an-existing-squad) · [Live control](#can-i-inject-commentary-or-control-speech-during-a-call) · [Testing](#can-i-run-simulations-and-evals) · [Recordings](#recordings-and-post-call-data) + +## Calls and voice + +| Feature | Support | +| --- | --- | +| Browser WebRTC, native Twilio, and Vapi SIP | Supported | +| Raw WebSocket | Supported with mono, signed 16-bit little-endian PCM at 24 kHz. Set the rate explicitly | +| Native Telnyx or Vonage | Unsupported | +| 22 preset OpenAI voices | Supported. See [voice previews](/gpt-live/configuration#voices) | +| Custom or cloned voices, or other voice providers | Unsupported | +| Changing voice during a call | Unsupported. Save the assistant and start a new call | +| Model and voice fallbacks | Unsupported. GPT-Live also cannot be a fallback model | +| Reconnecting to an existing GPT-Live session | Unsupported. Start a new call and check the status of pending work | + +See [Connect a call](/gpt-live/configuration#connect-a-call) for working connection examples. + +## Transfers and keypad input + +Function tools, API request tools, and `endCall` are supported. The other supported tool types, `transferCall` and `dtmf`, depend on the call connection: + +| Feature | Support | +| --- | --- | +| Blind transfer | Supported on native Twilio and Vapi SIP to configured phone or SIP destinations | +| Transfer on browser or raw WebSocket calls | Unsupported | +| Warm transfer or generated transfer summary | Unsupported | +| Transfer fallback plans or SIP `bye` verb | Unsupported | +| SIP transfer with extension dialing | Unsupported. Use a direct destination | +| Outgoing DTMF | Supported on SIP using RTP DTMF. Pass digits as a string, for example `"0011*#"` | +| Outgoing DTMF on Twilio, browser, or raw WebSocket calls | Unsupported | +| SIP INFO DTMF | Unsupported | +| Incoming keypad collection | Unsupported. Leave `keypadInputPlan.enabled` disabled | + +Other tool types are unsupported. Expose external lookups through a function or API request tool. Carrier acceptance of a transfer doesn't establish that the destination answered. + +## Can I use an existing squad? + +Squads and assistant handoffs are currently unsupported. GPT-Live's speaker and reasoner work within one assistant. Delegation doesn't switch the caller to another assistant. + +Early testing suggests some use cases no longer need the same divisions between assistants. See [Start with one assistant](/gpt-live/design#start-with-one-assistant) for how to rethink the architecture while keeping your business requirements. + +## Can I inject commentary or control speech during a call? + +Customer-initiated commentary injection, live message injection, and live call-control commands are currently unsupported. Incoming text commands on the raw WebSocket connection are ignored. + +Vapi uses OpenAI's commentary mechanism internally to pass reasoner results to the speaker. That doesn't expose `session.commentary.append` as a customer API. Use the supported [tool response flow](/gpt-live/configuration#connect-your-tools) to return results for requested work. + +A caller can ask the assistant to slow down or explain differently, and you can prompt it to respond to those requests. That conversational behavior is separate from an application sending a control command. + +## Can I guarantee exact speech or a fixed pause? + +GPT-Live generates speech and may paraphrase supplied text. Speaker instructions and personality packs guide delivery. They don't provide exact playback, a fixed speaking rate, or a programmatic pause while work completes. + +Audio URL greetings and the generated-message first-message mode are unsupported. Use a text `firstMessage` with `assistant-speaks-first`, or let the assistant wait for the caller. Without usable greeting text, it waits. + +If exact prerecorded wording or strict control of every spoken step is essential, choose an architecture with those controls. See [voice design](/gpt-live/design#speaking-style) for the choices available through prompting. + +## Which existing settings no longer apply? + +Classic transcriber selection, text-to-speech controls, and start/stop-speaking plans don't configure the GPT-Live conversation. General model fields such as `temperature` and `maxTokens` don't tune this integration. + +Use the speaker prompt for conversation behavior and the reasoner settings for task reasoning. The reasoner currently supports only the OpenAI models listed in [Model settings](/gpt-live/configuration#model-settings). + +## Does an interruption cancel a tool call? + +No. Interrupting speech changes the conversation, not an action already submitted to your service. Your application needs to check whether that action is pending, completed, or cancellable. + +If the caller corrects a date during a lookup, use the result that matches the updated request. For an example, see [When callers change the request](/gpt-live/design#when-callers-change-the-request). + +## Can I run Simulations and Evals? + +Voice Simulations are currently incompatible with GPT-Live. The simulation bridge uses 16 kHz audio and listen sockets. GPT-Live's raw WebSocket connection requires 24 kHz and doesn't provide listen sockets. + +Run voice scenarios through a [supported call connection](/gpt-live/configuration#connect-a-call). [Evals](/observability/evals-quickstart) can supplement testing of supported model decisions, but they don't establish GPT-Live's audio quality, timing, or interruption behavior. + +## Recordings and post-call data + +GPT-Live supports shared post-call analysis, structured outputs, scorecards, Boards, and Monitoring. Availability depends on your configuration and retained call data: + +| Setting or feature | Behavior | +| --- | --- | +| Recording disabled in the assistant | No recording is created | +| Recording-consent plan configured | Recording is disabled because GPT-Live doesn't collect that consent | +| Transcript-based analysis | Requires the relevant conversation data | +| Audio-based analysis | Requires an available recording | +| Monitoring with Zero Data Retention | Unsupported | +| Live listen or monitor sockets | Unsupported. Post-call Monitoring doesn't provide a live audio connection | + +See [Review and monitor calls](/gpt-live/configuration#review-and-monitor-calls) for setup and result fields. + +## Troubleshooting + +Use the symptom to find the setting or handler to check: + +| Symptom | Check | Fix | +| --- | --- | --- | +| GPT-Live is unavailable | Organization feature access | Contact Vapi to enable access. | +| Voice unsupported | `voice.provider` and `voice.voiceId` | Use OpenAI and a [supported voice ID](/gpt-live/configuration#voices). | +| Tool configuration unsupported | Tool type and resolved names | Use supported types and unique names. Inspect saved tools too. | +| Call fails on WebSocket startup | Audio format and sample rate | Set raw `pcm_s16le` at `24000` Hz explicitly. | +| WebSocket audio sounds distorted | Sample encoding, rate, and channels | Convert the source to mono, signed 16-bit little-endian PCM at 24 kHz. | +| Assistant never greets | First-message mode and usable text | Set `assistant-speaks-first` and a text `firstMessage`. | +| Assistant promises a lookup but no tool runs | Both prompts and attached tools | Add a concrete delegation rule and make the tool available to the reasoner. | +| Tool result never reaches the conversation | Handler response and matching ID | Return the actual `toolCallId`. Return the final async result through the original handler request. | +| Transfer fails | Transport and configured destination | Use a supported blind transfer. Check carrier requirements and destination restrictions. | +| No recording | Recording settings and consent plan | Check the documented recording behavior before the test. | diff --git a/fern/gpt-live/overview.mdx b/fern/gpt-live/overview.mdx index 4e57fdd70..0751d1241 100644 --- a/fern/gpt-live/overview.mdx +++ b/fern/gpt-live/overview.mdx @@ -1,85 +1,59 @@ --- -title: GPT-Live architecture -subtitle: Understand how conversation, reasoning, and tools work together -description: Understand GPT-Live in Vapi, including full-duplex audio, speaker and reasoner responsibilities, delegation, tool execution, and conversation state. +title: GPT-Live +subtitle: Voice conversations with tools and reasoning built in +description: Learn what GPT-Live brings to Vapi, how its speaker and reasoner work together, and where to start with conversation design, configuration, and compatibility. slug: gpt-live/overview --- -GPT-Live is a voice model that can listen and speak at the same time. Vapi connects it to a separate reasoning model and your tools so an assistant can hold a conversation while task work continues. +GPT-Live is an OpenAI voice model that can listen and speak at the same time. A caller can clarify a request while the assistant is talking, ask a question during a lookup, or ask it to slow down. This is **full-duplex conversation**: audio can flow in both directions at once. + +Vapi connects that conversation to your tools and phone numbers, and provides the call records, analysis, and monitoring around it. -GPT-Live access must be enabled for your organization. Contact Vapi if the model is unavailable. This section documents Vapi's integration; OpenAI's direct Live API exposes a different configuration and event interface. +GPT-Live must be enabled for your Vapi organization. Contact Vapi if the model isn't available in your dashboard. -## How the models work together +## How it works -The **speaker** handles the spoken conversation. The **reasoner** handles delegated requests, applies your task instructions, and selects tools. **Delegation** is the handoff from the speaker to the reasoner when a request needs backend work. +A GPT-Live assistant uses a **speaker** for the spoken interaction and a **reasoner** for tasks that need tools or detailed procedures. **Delegation** is the speaker asking the reasoner to handle one of those tasks. ```mermaid -flowchart LR - C[Caller] <-->|Audio| V[Vapi call transport] - V <-->|Audio| S[GPT-Live speaker] - S -->|Delegates work| R[Reasoner] - R -->|Selects tools| T[Vapi tool execution] - T <-->|Requests and results| B[Your services] - T -->|Tool results| R +flowchart TD + C[Caller] <-->|Audio| S[Speaker] + S -->|Delegates a task| R[Reasoner] + R <-->|Requests and results| T[Tools and your services] R -->|Result for the conversation| S ``` -The speaker can continue listening and speaking while the reasoner works. A completed tool request, a spoken answer, and an action recorded in your service are separate outcomes. - -| Component | Responsibility | Configuration | -| --- | --- | --- | -| Speaker | Conversation style, language, clarification, and delegation decisions | `model.speaker` | -| Reasoner | Task procedures, reasoning, tool selection, and interpreting results | `model.reasoner` | -| Vapi | Call transport, model connections, tools, post-call analysis, and observability | Assistant, call, and monitoring configuration | -| Your service | Authoritative records, permissions, input validation, and action status | Your tool implementation | - -For example, the caller asks for Friday availability. The speaker delegates a lookup. The reasoner calls your availability tool, then returns the confirmed slots for the speaker to explain. No appointment exists until a booking operation succeeds. - -## What full duplex changes - -In a speech-to-text, language-model, and text-to-speech pipeline, separate components transcribe input, compose a response, and synthesize speech. GPT-Live handles the audio conversation directly. You do not choose a separate transcriber or text-to-speech provider for this integration. - -Continuous conversation changes how you organize the application: - -- Keep short conversational guidance in the speaker prompt. -- Put detailed procedures and tool-use rules in the reasoner prompt. -- Keep durable task state and authorization in your service. -- Evaluate what the caller hears alongside what the tools actually do. - -A caller may correct a date while a lookup is running. Interrupting speech does not undo a booking or cancel an external request. The application needs a policy for those actions. See [Design GPT-Live conversations](/gpt-live/conversation-design). +For a scheduling assistant, the speaker can guide the caller toward a suitable appointment. The reasoner checks availability through your service and returns the options. The speaker can keep listening while that work happens. -## Choose an architecture for your requirements +GPT-Live handles speech directly. You don't configure a separate speech-to-text transcriber or text-to-speech provider for the conversation. -GPT-Live is useful when overlapping speech, conversational timing, and continued dialogue during backend work matter. Evaluate it with recordings and tasks representative of your callers. +## What changes with GPT-Live -Check the [compatibility reference](/gpt-live/configuration) before migrating. Existing transcriber controls, voice fallbacks, and some call features do not carry over. A workflow that depends on those features needs a supported alternative before migration. +You can design the interaction around the caller's goal, with room for questions and changes along the way. Detailed procedures belong with the reasoner. The speaker's instructions focus on how to guide the conversation. -A natural acknowledgment is not a measure of task completion. Compare the time to a useful answer and the correctness of the final action, rather than measuring only the first sound. +You also have more to design than the words alone. Set a voice, give the speaker a delivery brief, and use personality packs to explore different styles. A concise assistant can slow down for a date, explain one step at a time, or give the short version when asked. -## What Vapi provides around the conversation +[Design your assistant](/gpt-live/design) explains these choices with examples, including how to rethink an existing squad. -Vapi manages the call lifecycle and provides tools to review outcomes after the conversation. Configure the features your application needs; selecting GPT-Live does not automatically configure analysis or quality alerts. +## What Vapi provides -| Capability | What you use it for | +| Capability | Where to learn more | | --- | --- | -| [Call artifacts](/assistants/retrieve-call-artifacts) | Review transcripts, available recordings, and tool activity. | -| [Post-call analysis](/assistants/call-analysis) | Summarize a conversation and evaluate its outcome. | -| [Structured outputs](/assistants/structured-outputs) | Extract typed results such as the requested date, disposition, or whether a booking was confirmed. | -| [Scorecards](/observability/scorecard-quickstart) | Grade completed calls against criteria based on structured outputs. | -| [Boards](/observability/boards-quickstart) | Track call outcomes and trends across assistants. | -| [Monitoring](/observability/monitoring-quickstart) | Evaluate retained call data against configured thresholds and send alerts. | -| [Simulations](/observability/simulations-overview) | Define repeatable conversation scenarios; see the current GPT-Live limitation in the [testing guide](/gpt-live/testing#use-simulations-and-evals-appropriately). | -| [Evals](/observability/evals-quickstart) | Test supported model decisions; complement them with audio tests of the complete GPT-Live assistant. | +| Browser, phone, and WebSocket calls | [Connect a call](/gpt-live/configuration#connect-a-call) | +| Tools that look up information or take actions | [Connect your tools](/gpt-live/configuration#connect-your-tools) | +| Speaker prompts, reasoner settings, and personality packs | [Model settings](/gpt-live/configuration#model-settings) | +| 22 preset voices, with audio previews | [Choose a voice](/gpt-live/configuration#voices) | +| Transcripts, available recordings, and post-call analysis | [Review calls](/gpt-live/configuration#review-and-monitor-calls) | +| Structured outputs, scorecards, Boards, and Monitoring | [Track quality](/gpt-live/configuration#review-and-monitor-calls) | -Monitoring here means quality evaluation and alerting from call data. It is separate from listen or monitor sockets, which this integration does not support. See the [compatibility reference](/gpt-live/configuration#post-call-analysis-and-observability) for configuration and data requirements. +GPT-Live has a different set of supported features from Vapi's other voice architectures. Check [Limitations and FAQs](/gpt-live/limitations) if your application depends on squads, live call control, simulations, or a specific transfer flow. -## Start building +## Get started -- [Create a GPT-Live assistant](/gpt-live/quickstart) and make a web call. -- [Configure the speaker and reasoner](/gpt-live/prompts) for your task. -- [Connect tools](/gpt-live/tools) to your services. -- [Migrate and evaluate an assistant](/gpt-live/testing) against an existing baseline. +- **Explore the experience:** read [Design your assistant](/gpt-live/design) for architecture, conversation flow, and voice design. +- **Build a working assistant:** follow [Configure GPT-Live](/gpt-live/configuration#create-an-assistant) in the dashboard or API. +- **Assess a migration:** review [Limitations and FAQs](/gpt-live/limitations) against your current requirements. -For the underlying model architecture, see OpenAI's [GPT-Live overview](https://developers.openai.com/api/docs/guides/live). +For the underlying model, see OpenAI's [GPT-Live documentation](https://developers.openai.com/api/docs/guides/live). This section describes Vapi's integration and its supported configuration. diff --git a/fern/gpt-live/prompts.mdx b/fern/gpt-live/prompts.mdx deleted file mode 100644 index 78a8c8136..000000000 --- a/fern/gpt-live/prompts.mdx +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: Configure the GPT-Live speaker and reasoner -subtitle: Give each model clear instructions for its part of the task -description: Configure GPT-Live speaker and reasoner prompts in Vapi, define delegation conditions, select reasoning effort, and apply optional personality packs. -slug: gpt-live/prompts ---- - -Set conversation behavior in `model.speaker.instructions`. Set task procedures and tool-use rules in `model.reasoner.instructions`. Keeping these responsibilities clear makes it easier to diagnose a missed delegation or incorrect action. - -## Assign instructions to the right model - -| Instruction | Place it in | -| --- | --- | -| Role, tone, response length, and spoken language | Speaker | -| When to ask a clarification | Speaker | -| Which requests need backend work | Speaker | -| Required inputs and business procedures | Reasoner | -| Tool selection and handling failed results | Reasoner | -| Authorization, validation, and duplicate prevention | Your service, with corresponding reasoner guidance | - -The reasoner receives conversation context for delegated work. It does not replace validation in your tool implementation. - -## Write the speaker prompt - -Describe only capabilities your configured tools provide. For an assistant with availability and booking tools, use a short prompt like this: - -```text title="Speaker instructions" -You help callers schedule appointments. Speak clearly and use short replies. -Ask one brief question when the request is unclear. - -The backend can check availability and book a confirmed appointment. -Delegate requests for availability, booking, changes to a pending request, -and requests to end the call. -Handle greetings and requests to repeat a confirmed result directly. - -When the caller corrects a date or time, acknowledge the correction and -use the revised request. Do not announce a booking before the backend -confirms it. Explain uncertainty when the outcome is unknown. -``` - -Add booking to this prompt only after you implement the booking tool. The [quickstart](/gpt-live/quickstart) deliberately starts without it. - -OpenAI's [GPT-Live prompting guide](https://developers.openai.com/api/docs/guides/live-prompting) recommends concise instructions with concrete delegation conditions. Apply those principles through Vapi's speaker field; direct OpenAI session fields are not Vapi assistant fields. - -## Write the reasoner prompt - -Name the required inputs, tool boundaries, and what counts as success. Adapt this example to your tool names: - -```text title="Reasoner instructions" -Use the current conversation to resolve delegated appointment requests. -Check the requested date and time zone before looking up availability. -Use lookupAvailability for available slots. Availability is not a booking. - -Use the booking tool only after the caller confirms a specific slot. -If the caller changes the date or time, obtain confirmation again. -Follow the service's validation and operation-status results. - -Return a short factual result for the speaker. Distinguish pending, -confirmed, failed, and unknown outcomes. Do not invent a successful action. -When the caller asks to finish the conversation, call endCall. -``` - -Your service must still reject unauthorized or invalid requests. A prompt is guidance to the model, not an enforcement boundary. - -## Understand instruction precedence - -Explicit `model.speaker.instructions` takes precedence over legacy `model.systemPrompt` and system messages in `model.messages`. If you omit the speaker field, Vapi uses that legacy prompt configuration. - -Omitting `model.reasoner.instructions` uses Vapi's default reasoner instructions. A custom reasoner prompt replaces those defaults; Vapi does not append the default behavioral instructions. An explicit empty string remains empty. - -The prompts are independent. Changing the speaker prompt does not also configure the reasoner. Personality packs add guidance to the speaker instructions when selected. - -## Choose reasoning effort - -Start with the default reasoner, `gpt-5.6-terra`, at `low` effort. Increase effort when task evaluations show that additional reasoning helps. Higher effort can increase the time before a useful result reaches the caller. - -Supported model and effort values are listed in the [configuration reference](/gpt-live/configuration). General assistant model fields such as `temperature` and `maxTokens` do not configure this reasoner path. - -## Add personality packs deliberately - -Set `model.speaker.personalityPacks` to an array containing any of these IDs: `eager-listener`, `idle-hummer`, `bouncy`, or `unhurried`. - -```json -{ - "speaker": { - "instructions": "Speak clearly and ask one question at a time.", - "personalityPacks": ["unhurried"] - } -} -``` - -This is a fragment of the `model` object. Test each pack with your actual callers and tasks. Avoid piling on style instructions that conflict with one another. - -## Diagnose prompt problems - -If the assistant talks about doing work but no tool runs, inspect the speaker's delegation instructions and the reasoner's tool access. If a tool runs with wrong inputs, inspect the reasoner procedure and your service validation. - -For language-specific prompts and audio examples, see [Voices and languages](/gpt-live/voices). diff --git a/fern/gpt-live/quickstart.mdx b/fern/gpt-live/quickstart.mdx deleted file mode 100644 index 3af2ed247..000000000 --- a/fern/gpt-live/quickstart.mdx +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: GPT-Live quickstart -subtitle: Create an assistant and test a full-duplex conversation -description: Create a GPT-Live assistant in Vapi with speaker and reasoner prompts, choose a voice, test a web call, and end the call using the endCall tool. -slug: gpt-live/quickstart ---- - -Create an assistant that can hold a conversation and end a call on request. This first assistant has no access to appointment data; the [tools guide](/gpt-live/tools) adds that capability. - -## Prerequisites - -- A Vapi organization with GPT-Live enabled. -- A microphone and a browser with microphone permission. -- A Vapi private API key for the API method. Keep it on your server or local machine. -- If you supply an OpenAI key, access to GPT-Live and the selected reasoner model on that key. - -## Create the assistant - -Choose either method. Both create a saved assistant that you can test in the dashboard. - - - - - -Open [Assistants](https://dashboard.vapi.ai/assistants), select **Create Assistant**, and choose **GPT-Live** in the assistant configuration. - - -Choose Marin as the voice. In **Speaker**, enter: - -```text -You explain appointment scheduling. Speak clearly and briefly. -Availability checks and booking are not connected yet. -Delegate requests to end the call. Do not claim to have checked or changed a booking. -``` - -In **Reasoner**, enter: - -```text -When the caller asks to finish, call endCall. -No availability or booking tools are configured. Do not invent appointment results. -``` - -Keep or add the `endCall` tool in the assistant's tools. - - -Set the first message to “Hello. I can help explain appointment scheduling. What would you like to know?” Configure the assistant to speak first. - - -Select **Publish** and complete the publication flow. Keep the assistant ID for later API requests. - - - - - - -Save this JSON as `assistant.json`: - -```json title="assistant.json" -{ - "name": "Appointment assistant", - "model": { - "provider": "openai", - "model": "gpt-live-1", - "speaker": { - "instructions": "You are an appointment assistant. Speak clearly and briefly. Ask a short question when the request is unclear. You cannot check availability or book appointments yet. Delegate requests to end the call. Do not claim to have checked or changed a booking." - }, - "reasoner": { - "model": "gpt-5.6-terra", - "reasoningEffort": "low", - "instructions": "Handle delegated requests using the available tools. When the caller asks to finish, call endCall. No availability or booking tools are configured. Do not invent appointment results." - }, - "tools": [ - { - "type": "endCall" - } - ] - }, - "voice": { - "provider": "openai", - "voiceId": "marin" - }, - "firstMessageMode": "assistant-speaks-first", - "firstMessage": "Hello. I can help explain appointment scheduling. What would you like to know?", - "maxDurationSeconds": 600 -} -``` - - -Set `VAPI_PRIVATE_API_KEY` to your Vapi private API key, then send the configuration: - -```bash -curl --fail-with-body https://api.vapi.ai/assistant \ - -H "Authorization: Bearer $VAPI_PRIVATE_API_KEY" \ - -H "Content-Type: application/json" \ - --data-binary @assistant.json -``` - -Keep the returned `id`. This request uses the [Create Assistant endpoint](/api-reference/assistants/create). - - - - - -## Test the conversation - -Open the saved assistant in the dashboard and select **Talk**. Allow microphone access when prompted. - -Try these exchanges: - -| Say | Expected outcome | -| --- | --- | -| “What can you help with?” | The assistant explains its limited appointment role. | -| “Can you book Friday?” | It explains that booking is not connected; it does not invent a reservation. | -| Interrupt a long answer with “Keep it brief.” | It adapts the conversation to the request. | -| “That's all. Please end the call.” | It delegates the request and invokes `endCall`. | - -Check both the conversation and the call record. A spoken goodbye alone does not prove that the call ended. You can also end the web call with the dashboard's call controls. - -## Continue building - -Add an [availability lookup tool](/gpt-live/tools), then update both prompts to describe the new capability. See [Configuration and compatibility](/gpt-live/configuration) for phone calls and raw WebSocket audio. diff --git a/fern/gpt-live/testing.mdx b/fern/gpt-live/testing.mdx deleted file mode 100644 index f141b9533..000000000 --- a/fern/gpt-live/testing.mdx +++ /dev/null @@ -1,112 +0,0 @@ ---- -title: Migrate and test a GPT-Live assistant -subtitle: Compare task outcomes and diagnose conversation failures -description: Migrate an existing Vapi assistant to GPT-Live, test interruptions and tool outcomes, inspect call artifacts, and troubleshoot configuration and audio failures. -slug: gpt-live/testing ---- - -Validate the migrated assistant against representative calls and final service records. A fluent conversation is only one part of success; the assistant must also perform the correct actions and end the call cleanly. - -## Prepare a migration - - - -Collect representative call scenarios, expected actions, and final records. Include errors and corrections, not only successful requests. Use test data with a repeatable starting state. - - -Compare the assistant's transports, tools, greetings, and fallback plans with the [compatibility reference](/gpt-live/configuration). Resolve dependencies on unsupported features before replacing the assistant. - - -Move conversational guidance into the speaker prompt and detailed procedures into the reasoner prompt. Retain validation and authorization in your services. See [Configure the speaker and reasoner](/gpt-live/prompts). - - -Use `gpt-live-1` and a supported OpenAI voice. Connect test tools, then confirm a basic call and an end-call request before testing business actions. - - -Repeat each scenario against both assistants. Record intentional differences and regressions. Do not route production calls to the new assistant until its required behaviors pass. - - - -## Test conversation and task outcomes - -Use this matrix as a starting point for an appointment assistant: - -| Scenario | Check in the audio | Check in your service | -| --- | --- | --- | -| Ask for availability | Speaks the returned slots | Correct date and time zone queried | -| Change Thursday to Friday during lookup | Uses the corrected request | Outdated lookup cannot drive a booking | -| Interrupt an answer | Responds to the new request | No accidental duplicate action | -| Confirm one offered slot | Announces success only after confirmation | Exactly one booking for that slot | -| Tool responds slowly | Gives accurate pending information | Operation remains traceable | -| Tool fails | Explains failure without inventing a result | No successful write recorded | -| Response is lost after a write | Treats the outcome as uncertain | Status reconciled before retry | -| Ask to cancel | Distinguishes request from completion | Cancellation status verified | -| Disconnect during work | New call does not assume completion | Pending work reconciled | -| Ask to end the call | Conversation ends | Call record reaches an ended state | - -Include the languages, background noise, names, and speaking styles your callers use. Voice previews demonstrate a voice; they do not replace application testing. - -## Measure useful outcomes - -Track task completion, incorrect actions, duplicate writes, and the time to a useful spoken answer. Record first-audio timing separately so a quick acknowledgment does not hide slow or unsuccessful work. - -Use the same starting state and success criteria when comparing configurations. Repeat trials because generated conversations vary. Increase reasoning effort only when the improvement in task outcomes justifies its effect on response time and cost. - -OpenAI's [voice evaluation guide](https://developers.openai.com/cookbook/examples/audio/voice_agent_evaluation) provides additional evaluation methods. Adapt them to the outcomes your Vapi assistant must achieve. - -## Use Simulations and Evals appropriately - -Vapi [Simulations](/observability/simulations-overview) provide scenarios, AI tester personalities, and success criteria for repeatable conversations. Use the scenario matrix above to define what a GPT-Live test should cover. - -The current voice simulation bridge uses 16 kHz audio and listen sockets. GPT-Live's raw WebSocket connection requires 24 kHz audio and does not provide those listen sockets, so the current voice simulation path is not compatible. Use calls through a supported GPT-Live transport to exercise these scenarios until a compatible simulation path is available. - -[Evals](/observability/evals-quickstart) test supported model decisions using configured inputs and expected behavior. They do not establish full-duplex timing, interruption handling, or audio quality. A passing text evaluation or chat simulation is not evidence that a GPT-Live voice call works. - -## Analyze outcomes after calls - -Configure [structured outputs](/assistants/structured-outputs) to extract consistent fields from completed calls. For the appointment example, useful fields include the requested date, whether the caller corrected it, and whether the assistant announced a confirmed booking. - -Use [scorecards](/observability/scorecard-quickstart) to grade those outputs against your criteria. Keep conversational evidence separate from service evidence: a transcript-based grade cannot establish that exactly one appointment exists in your database. - -Existing `analysisPlan` configurations also use the shared post-call analysis path. For new extraction and evaluation setups, follow the [call analysis guide's recommendation](/assistants/call-analysis) to use structured outputs. - -## Monitor production quality - -Use [Boards](/observability/boards-quickstart) to inspect outcome trends and [Monitoring](/observability/monitoring-quickstart) to alert on configured thresholds. Start with failed calls, failed task outcomes, and changes in the rate of unresolved requests. - -Choose an evaluation window and threshold that match your traffic. Review the affected calls when an alert fires, then turn confirmed failures into repeatable test scenarios. Sample unflagged calls too; a monitor only detects the conditions you configure. - -Monitoring requires retained call data and is unavailable with Zero Data Retention. It evaluates call quality and outcomes; it does not provide a live listen connection to the GPT-Live session. - -## Inspect call evidence - -Use the [Vapi call record and artifacts](/assistants/retrieve-call-artifacts) to inspect the ended reason, messages, tool activity, and available artifacts. Compare returned tool results with your service's logs and operation identifiers. - -Check recording settings before relying on audio artifacts. Recording is disabled when a recording-consent plan is configured for this integration. Preserve suitable test evidence through an approved recording setup. - -Transcripts are useful for reviewing words. Listen to audio when judging timing, interruption handling, pronunciation, or what the caller actually heard. - -## Troubleshoot common failures - -Start with the symptom, then verify its likely cause: - -| Symptom | Check | Fix | -| --- | --- | --- | -| GPT-Live is unavailable | Organization feature access | Contact Vapi to enable access. | -| Voice rejected | `voice.provider` and `voice.voiceId` | Use OpenAI and a [supported voice ID](/gpt-live/voices). | -| Assistant save rejects tools | Tool type and resolved names | Use supported types and unique names; inspect saved tools too. | -| Call fails on WebSocket startup | Audio format and sample rate | Set raw `pcm_s16le` at `24000` Hz explicitly. | -| WebSocket audio sounds distorted | Sample encoding, rate, and channels | Convert the source to mono, signed 16-bit little-endian PCM at 24 kHz. | -| Assistant never greets | First-message mode and usable text | Set `assistant-speaks-first` and a text `firstMessage`. | -| Assistant promises a lookup but no tool runs | Both prompts and attached tools | Add a concrete delegation rule and make the tool available to the reasoner. | -| Tool result never reaches the conversation | Handler response and matching ID | Return the actual `toolCallId`; complete asynchronous work through its supported contract. | -| Transfer fails | Transport and configured destination | Use a supported blind transfer; check carrier requirements and destination restrictions. | -| No recording | Recording settings and consent plan | Check the documented recording behavior before the test. | - -## Finish or roll back a test - -End active calls using the call interface or `endCall`. Disconnect test WebSocket clients and verify that the calls ended. Review any external operations still pending after disconnect. - -Keep the previous assistant available during migration. To roll back, route new calls to that assistant. Reconcile actions already submitted by the test assistant; changing call routing does not undo those actions. - -For more background on preserving application state and procedures, see OpenAI's [migration guide](https://developers.openai.com/api/docs/guides/live-migration). diff --git a/fern/gpt-live/tools.mdx b/fern/gpt-live/tools.mdx deleted file mode 100644 index 0ae243c31..000000000 --- a/fern/gpt-live/tools.mdx +++ /dev/null @@ -1,125 +0,0 @@ ---- -title: Connect tools to GPT-Live -subtitle: Connect an availability lookup and return confirmed results -description: Add Vapi function and API request tools to GPT-Live, configure delegation, return matching tool results, and handle pending work and restricted call tools. -slug: gpt-live/tools ---- - -Add tools to give the reasoner access to your services. This guide extends the [quickstart assistant](/gpt-live/quickstart) with an availability lookup. The lookup returns slots; it does not book an appointment. - -## Prerequisites - -- A saved GPT-Live assistant. -- A reachable HTTPS service for the lookup. -- A Vapi private API key to update the assistant. - -Your service owns appointment data and validates requests. Use fictional availability while testing. - -## Add an availability function - - - -Handle Vapi's [`tool-calls` server message](/server-url/events). Read the function name and arguments from `message.toolCallList`, validate the date and time zone, and query your availability service. - -For each call, return its matching `toolCallId`. This example response uses a placeholder ID; your handler must echo the real ID from the request: - -```json -{ - "results": [ - { - "name": "lookupAvailability", - "toolCallId": "TOOL_CALL_ID_FROM_REQUEST", - "result": "There are openings at 2 PM and 4 PM on the requested date in the requested time zone. No appointment has been booked." - } - ] -} -``` - -Return a factual failure when the lookup fails. Do not return sample slots as a successful response to a production request. - - -Add this definition to `model.tools`, keeping the quickstart's `endCall` tool. Replace the example Server URL with your handler: - -```json -{ - "type": "function", - "function": { - "name": "lookupAvailability", - "description": "Check available appointment slots. Does not reserve or book a slot.", - "parameters": { - "type": "object", - "properties": { - "date": { - "type": "string", - "description": "Requested calendar date in YYYY-MM-DD format." - }, - "timeZone": { - "type": "string", - "description": "IANA time zone, such as America/New_York." - } - }, - "required": ["date", "timeZone"] - } - }, - "server": { - "url": "https://api.example.com/vapi/tools" - } -} -``` - -You can also create a saved tool and reference its ID through `model.toolIds`. See [Create Tool](/api-reference/tools/create). - - -In the speaker prompt, explain that availability checks are now supported and delegate those requests. Keep booking explicitly unavailable. - -In the reasoner prompt, require a date and time zone, call `lookupAvailability`, and report only returned slots. Explain that availability does not reserve a slot. - - -Publish the updated assistant in the dashboard, or update it with the [Update Assistant endpoint](/api-reference/assistants/update). Preserve its other tools and model settings when preparing the update. - - -Ask for availability with an explicit date and time zone. Confirm that your service receives the request, the returned tool-call ID matches, and the spoken answer agrees with the returned slots. - -Then ask to book a slot. The assistant must explain that booking is not connected. Verify that no appointment record was created. - - - -## Choose a supported tool type - -The reasoner can use these Vapi tool types: - -| Type | Use | -| --- | --- | -| `function` | Call your Server URL with a `tool-calls` message. | -| `apiRequest` | Execute a configured HTTP request. | -| `endCall` | End the conversation on request. | -| `dtmf` | Send outgoing RTP keypad tones on SIP. | -| `transferCall` | Submit a supported blind transfer to a configured destination. | - -Give function tools unique `function.name` values and API request tools unique `name` values. Vapi validates the resolved tool set, including saved tools and tools derived from legacy assistant fields. - -Other tool types are rejected for GPT-Live. To query an external knowledge source, expose the lookup through a supported function or API request tool. See [Configuration and compatibility](/gpt-live/configuration) for transport and transfer restrictions. - -## Handle slow or asynchronous work - -A function may be synchronous or asynchronous. For asynchronous functions, follow Vapi's [function tool contract](/tools/custom-tools); accepting work is not a final result. Keep operation status in your service until completion. - -Return concise, unambiguous information: the requested operation, its status, and the verified outcome. A queued booking should return a pending status, not a booking confirmation. - -Tools may execute concurrently. Make dependent actions explicit: availability, caller confirmation, and booking have prerequisites even when unrelated reads can overlap. Your service should enforce those prerequisites. - -## Add booking safely - -Keep lookup and booking as separate operations. The booking service should validate the selected slot, caller authorization, and current confirmation before creating the record. - -Use an operation identifier to make retries safe. If a request times out, query its status before creating another booking. Report an unknown outcome until your service can establish success or failure. - -A spoken correction is new conversational information. Your backend must determine whether a pending operation can be cancelled, changed, or reconciled. See [Design GPT-Live conversations](/gpt-live/conversation-design). - -## End or transfer the call - -Include `endCall` when the assistant should hang up. Tell the speaker to delegate that request and the reasoner to use the tool. - -For `transferCall`, configure allowed destinations. Carrier acceptance does not prove the destination answered. Avoid describing a transfer as a completed conversation with the recipient until you have evidence of that outcome. - -For background on separating voice interaction from task execution, see OpenAI's [delegation guide](https://developers.openai.com/api/docs/guides/live-delegation). Use Vapi tool definitions for this integration, rather than copying OpenAI session event examples. diff --git a/fern/gpt-live/voices.mdx b/fern/gpt-live/voices.mdx deleted file mode 100644 index d8584d525..000000000 --- a/fern/gpt-live/voices.mdx +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: GPT-Live voices and languages -subtitle: Compare voice previews and configure the spoken language -description: Compare all 22 GPT-Live voices in Vapi, including regional tags and Portuguese previews, then configure voice IDs, language instructions, and pronunciation. -slug: gpt-live/voices ---- - -Choose a voice with `voice.provider: "openai"` and `voice.voiceId`. GPT-Live supports 22 preset voices in Vapi. Use the dashboard's voice previews or the recordings below to compare them. - -## Configure a voice - -Set this object on your assistant: - -```json -{ - "voice": { - "provider": "openai", - "voiceId": "bossa" - } -} -``` - -Publish the updated assistant and start a new call to test the change. This integration does not expose an in-call voice change. - -## Voices with language and regional tags - -The tags describe the voice's language, regional influence, presentation, and source. Regional influence does not guarantee accent fidelity. Bossa and Tempo previews are in Brazilian Portuguese; the other previews are in English. - -| Voice | API ID | Language | Regional influence | Presentation | Source | Preview | -| --- | --- | --- | --- | --- | --- | --- | -| Quartz | `quartz` | English | Australian | Feminine | Generated | [Listen to Quartz](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/quartz.mp3) | -| Ripple | `ripple` | English | Australian | Masculine | Natural | [Listen to Ripple](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/ripple.mp3) | -| Vesper | `vesper` | English | British | Masculine | Natural | [Listen to Vesper](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/vesper.mp3) | -| Willow | `willow` | English | Irish | Feminine | Natural | [Listen to Willow](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/willow.mp3) | -| Stone | `stone` | English | Irish | Masculine | Natural | [Listen to Stone](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/stone.mp3) | -| Gleam | `gleam` | English | North American | Feminine | Natural | [Listen to Gleam](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/gleam.mp3) | -| Meridian | `meridian` | English | North American | Masculine | Natural | [Listen to Meridian](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/meridian.mp3) | -| Bossa | `bossa` | Portuguese | Brazilian | Feminine | Natural | [Listen to Bossa](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/bossa.mp3) | -| Tempo | `tempo` | Portuguese | Brazilian | Masculine | Natural | [Listen to Tempo](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/tempo.mp3) | -| Beacon | `beacon` | English | Filipino | Masculine | Generated | [Listen to Beacon](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/beacon.mp3) | -| Delta | `delta` | English | Southern U.S. | Feminine | Generated | [Listen to Delta](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/delta.mp3) | -| Cinder | `cinder` | English | Southern U.S. | Masculine | Generated | [Listen to Cinder](https://voices-storage.vapi.ai/gpt-live/gpt-live-1/cinder.mp3) | - -## Additional supported voices - -These voices remain available. Their previews use the existing recordings. - -| Voice | API ID | Preview | -| --- | --- | --- | -| Alloy | `alloy` | [Listen to Alloy](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/alloy.mp3) | -| Ash | `ash` | [Listen to Ash](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/ash.mp3) | -| Ballad | `ballad` | [Listen to Ballad](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/ballad.mp3) | -| Cedar | `cedar` | [Listen to Cedar](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/cedar.mp3) | -| Coral | `coral` | [Listen to Coral](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/coral.mp3) | -| Echo | `echo` | [Listen to Echo](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/echo.mp3) | -| Marin | `marin` | [Listen to Marin](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/marin.mp3) | -| Sage | `sage` | [Listen to Sage](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/sage.mp3) | -| Shimmer | `shimmer` | [Listen to Shimmer](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/shimmer.mp3) | -| Verse | `verse` | [Listen to Verse](https://voices-storage.vapi.ai/gpt-live/marble-alpha-v6/verse.mp3) | - -## Set the spoken language - -A voice choice does not replace language instructions. Write the speaker prompt in the language you want the assistant to use, and test both its greeting and its responses. - -For a Brazilian Portuguese assistant using Bossa or Tempo, this model fragment sets a clear language policy: - -```json -{ - "speaker": { - "instructions": "Você ajuda a pessoa a agendar um atendimento. Fale em português do Brasil, com respostas curtas e claras. Faça uma pergunta por vez. Encaminhe ao backend pedidos que precisem consultar dados ou usar ferramentas. Só confirme um agendamento depois de receber a confirmação do backend." - } -} -``` - -Use the same language for `firstMessage`, such as “Olá! Como posso ajudar hoje?” Configure the reasoner's business rules and tools separately. - -## Test pronunciation and speaking style - -Include real task vocabulary in your tests: names, addresses, dates, and product terms. Add pronunciation guidance to the speaker prompt when needed, then evaluate the spoken result. - -A preview is an example recording, not a promise that every response has identical pacing or expression. Compare voices using the same prompt and representative calls. - -See OpenAI's [voice options](https://developers.openai.com/api/docs/guides/live-conversations#voice-options) for the source metadata and [language guidance](https://developers.openai.com/api/docs/guides/live-prompting#language-and-pronunciation) for prompting recommendations. Vapi's supported catalog is the list on this page; upstream custom-voice capabilities do not imply support in this integration. diff --git a/fern/providers/gpt-live.mdx b/fern/providers/gpt-live.mdx deleted file mode 100644 index 2a34ba9ad..000000000 --- a/fern/providers/gpt-live.mdx +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: GPT-Live -subtitle: Connect a full-duplex voice model to reasoning and tools in Vapi -description: Connect GPT-Live in Vapi, configure the OpenAI speaker and reasoner, and find the quickstart, voice catalog, tool guide, and compatibility reference. -slug: providers/gpt-live ---- - -## What is GPT-Live? - -GPT-Live is an OpenAI voice model that listens and speaks at the same time. In Vapi, the speaker handles conversation while a separate reasoner handles delegated tasks and tools. See [GPT-Live architecture](/gpt-live/overview) for the responsibilities of each component. - -## Connect GPT-Live - -Your organization needs GPT-Live enabled. If you bring an OpenAI key, it must have access to GPT-Live and the selected reasoning model. - -Follow the [GPT-Live quickstart](/gpt-live/quickstart) to create an assistant and make a web call. Add [tools](/gpt-live/tools) when you are ready to connect your services. - -## Settings reference - -| Setting | Configuration | -| --- | --- | -| Voice model | `model.provider: "openai"` and `model.model: "gpt-live-1"` | -| Speaker behavior | `model.speaker.instructions` | -| Reasoner behavior | `model.reasoner.instructions` | -| Reasoner selection | `model.reasoner.model` and `model.reasoner.reasoningEffort` | -| Voice | `voice.provider: "openai"` and a [supported voice ID](/gpt-live/voices) | - -Use the [configuration and compatibility reference](/gpt-live/configuration) for defaults, transports, tool restrictions, and settings that do not apply. Learn to [design conversations](/gpt-live/conversation-design) and [test a migration](/gpt-live/testing) before moving an existing assistant.