Python: DevUI audio input and response playback

python/packages/devui/README.md

@@ -196,6 +196,26 @@ response2 = client.responses.create(
 
 **How it works:** DevUI automatically retrieves the conversation's message history from the stored thread and passes it to the agent. You don't need to manually manage message history - just provide the same `conversation` ID for follow-up requests.
 
+### Voice Input and Audio Responses
+
+DevUI chat supports microphone input in browsers that allow recording from `localhost` or HTTPS pages. Use the
+microphone button to start recording, then stop to preview or cancel. You can also hold the microphone button to
+record and release it to attach the recording. Recordings are converted to mono WAV, with defaults of 16 kHz,
+60 seconds, and 10 MB. The recording bar shows elapsed time and animated live microphone levels.
+
+DevUI sends the audio to the selected agent without transcribing it. The selected agent or model must be able to work
+with audio. Uploaded files keep the existing attachment behavior and original format, including multiple audio
+files such as WAV, MP3, M4A, and OGG. Support for those formats depends on the selected agent.
+
+Recordings and agent audio share a waveform player with play/pause, seeking, and duration. Agent audio includes a
+download action. Audio returned by the conversation-history API or a resumed response uses the same controls.
+Playback uses the browser's audio element and does not depend on waveform decoding or downloading remote audio first.
+
+Settings > General > Audio options groups capture limits, sample rate, preview, and browser processing hints under
+Input. Auto-stop on silence optionally attaches click-to-record audio after a quiet pause and does not interrupt
+push-to-talk. Output provides optional agent-audio autoplay, which is disabled by default and remains subject to
+browser restrictions. Restored responses do not autoplay.
+
 ### OpenAI Proxy Mode
 
 DevUI provides an **OpenAI Proxy** feature for testing OpenAI models directly through the interface without creating custom agents. Enable via Settings → OpenAI Proxy tab.

python/packages/devui/agent_framework_devui/_audio.py

@@ -0,0 +1,24 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Conversion of Responses input_audio parts to framework audio content."""
+
+from agent_framework import Content
+from openai.types.responses import ResponseInputAudioParam
+from pydantic import TypeAdapter, ValidationError
+
+_RESPONSE_INPUT_AUDIO_ADAPTER = TypeAdapter(ResponseInputAudioParam)
+
+
+def input_audio_to_content(value: object) -> Content:
+    """Validate an input_audio part and preserve its audio data without transcription."""
+    try:
+        audio = _RESPONSE_INPUT_AUDIO_ADAPTER.validate_python(value)["input_audio"]
+    except ValidationError as exc:
+        raise ValueError("Invalid input_audio: expected audio data with wav or mp3 format") from exc
+    if not audio["data"]:
+        raise ValueError("Invalid input_audio: audio data must not be empty")
+
+    media_type = "audio/wav" if audio["format"] == "wav" else "audio/mpeg"
+    data = audio["data"]
+    uri = data if data.startswith("data:") else f"data:{media_type};base64,{data}"
+    return Content.from_uri(uri=uri, media_type=media_type, name=f"recording.{audio['format']}")

python/packages/devui/agent_framework_devui/_conversations.py

@@ -29,6 +29,7 @@
     ResponseOutputRefusal,
 )
 
+from ._audio import input_audio_to_content
 from ._utils import infer_media_type
 
 _MODEL_OUTPUT_KIND_KEY = "model_output_kind"
@@ -577,6 +578,9 @@ def _to_agent_content(content: dict[str, Any]) -> Content | None:
                 )
             return None
 
+        if content_type == "input_audio":
+            return input_audio_to_content(content)
+
         if content_type == "input_file":
             filename = content.get("filename")
             filename = filename if isinstance(filename, str) else ""

python/packages/devui/agent_framework_devui/_executor.py

@@ -11,6 +11,7 @@
 
 from agent_framework import Content, SupportsAgentRun, Workflow
 
+from ._audio import input_audio_to_content
 from ._conversations import ConversationStore, InMemoryConversationStore
 from ._discovery import EntityDiscovery
 from ._mapper import MessageMapper
@@ -751,6 +752,12 @@ def _convert_openai_input_to_chat_message(
                                             )
                                         )
 
+                                elif content_type == "input_audio":
+                                    try:
+                                        contents.append(input_audio_to_content(content_dict))
+                                    except ValueError as exc:
+                                        raise InputConversionError(str(exc)) from exc
+
                                 elif content_type == "function_approval_response":
                                     # Handle function approval response with server-side validation
                                     try:

python/packages/devui/agent_framework_devui/_mapper.py

@@ -52,6 +52,7 @@
     ResponseUsage,
     ResponseWorkflowEventComplete,
 )
+from .models._openai_custom import MediaResponse, MediaResponseEvent
 
 logger = logging.getLogger(__name__)
 _MODEL_OUTPUT_KIND_KEY = "model_output_kind"
@@ -62,6 +63,7 @@
     ResponseStreamEvent,
     ResponseWorkflowEventComplete,
     ResponseOutputItemAddedEvent,
+    CustomResponseOutputItemAddedEvent,
     ResponseTraceEventComplete,
 ]
 
@@ -330,6 +332,7 @@ async def aggregate_to_response(self, events: Sequence[Any], request: AgentFrame
 
             # Track complete message items that do not use the text-delta lifecycle.
             complete_messages: dict[str, ResponseOutputMessage] = {}
+            custom_output_items: list[Any] = []
 
             for event in events:
                 event_type = getattr(event, "type", None)
@@ -394,6 +397,8 @@ async def aggregate_to_response(self, events: Sequence[Any], request: AgentFrame
                                     message_order.append(item.id)
                                 if item.content:
                                     complete_messages[item.id] = item
+                        elif item_type in {"output_image", "output_file", "output_data"}:
+                            custom_output_items.append(item)
 
                 # Handle function call arguments delta - accumulate arguments
                 elif event_type == "response.function_call_arguments.delta":
@@ -458,6 +463,8 @@ async def aggregate_to_response(self, events: Sequence[Any], request: AgentFrame
                 )
                 output_items.append(response_output_message)
 
+            output_items.extend(custom_output_items)
+
             # If no output items at all, create an empty message
             if not output_items and failed_response is None:
                 response_output_text = ResponseOutputText(type="output_text", text="", annotations=[])
@@ -476,7 +483,8 @@ async def aggregate_to_response(self, events: Sequence[Any], request: AgentFrame
 
             usage = _response_usage(usage_data) if usage_data is not None else None
 
-            return OpenAIResponse(
+            response_type = MediaResponse if custom_output_items else OpenAIResponse
+            return response_type(
                 id=context["response_id"],
                 object="response",
                 created_at=datetime.now().timestamp(),
@@ -503,7 +511,7 @@ async def aggregate_to_response(self, events: Sequence[Any], request: AgentFrame
 
     async def finalize_stream(
         self, events: Sequence[Any], request: AgentFrameworkRequest
-    ) -> ResponseCompletedEvent | ResponseFailedEvent:
+    ) -> ResponseCompletedEvent | ResponseFailedEvent | MediaResponseEvent:
         """Create the single terminal event for a mapped response stream."""
         final_response = await self.aggregate_to_response(events, request)
 
@@ -516,6 +524,13 @@ async def finalize_stream(
                 last_sequence_number = sequence_number
                 break
 
+        if isinstance(final_response, MediaResponse):
+            return MediaResponseEvent(
+                type="response.failed" if final_response.status == "failed" else "response.completed",
+                response=final_response,
+                sequence_number=last_sequence_number + 1,
+            )
+
         if final_response.status == "failed":
             return ResponseFailedEvent(
                 type="response.failed",
@@ -1622,7 +1637,7 @@ async def _map_usage_content(self, content: Any, context: dict[str, Any]) -> Non
 
     async def _map_data_content(
         self, content: Any, context: dict[str, Any]
-    ) -> ResponseOutputItemAddedEvent | ResponseTraceEventComplete:
+    ) -> CustomResponseOutputItemAddedEvent | ResponseTraceEventComplete:
         """Map DataContent to proper output item (image/file/data) or fallback to trace.
 
         Maps Agent Framework DataContent to appropriate output types:
@@ -1631,7 +1646,9 @@ async def _map_data_content(
         - Generic data → ResponseOutputData
         - Unknown/debugging content → ResponseTraceEventComplete (fallback)
         """
-        mime_type = getattr(content, "mime_type", "application/octet-stream")
+        mime_type = (
+            getattr(content, "media_type", None) or getattr(content, "mime_type", None) or "application/octet-stream"
+        )
         item_id = f"item_{uuid.uuid4().hex[:16]}"
 
         # Extract data/uri
@@ -1663,9 +1680,9 @@ async def _map_data_content(
                     sequence_number=self._next_sequence(context),
                 )
 
-            return ResponseOutputItemAddedEvent(
+            return CustomResponseOutputItemAddedEvent(
                 type="response.output_item.added",
-                item=ResponseOutputImage(  # type: ignore[arg-type]
+                item=ResponseOutputImage(
                     id=item_id,
                     type="output_image",
                     image_url=image_url,
@@ -1720,9 +1737,9 @@ async def _map_data_content(
                     sequence_number=self._next_sequence(context),
                 )
 
-            return ResponseOutputItemAddedEvent(
+            return CustomResponseOutputItemAddedEvent(
                 type="response.output_item.added",
-                item=ResponseOutputFile(  # type: ignore[arg-type]
+                item=ResponseOutputFile(
                     id=item_id,
                     type="output_file",
                     filename=filename,
@@ -1750,9 +1767,9 @@ async def _map_data_content(
             else:
                 data_str = str(data_value)
 
-        return ResponseOutputItemAddedEvent(
+        return CustomResponseOutputItemAddedEvent(
             type="response.output_item.added",
-            item=ResponseOutputData(  # type: ignore[arg-type]
+            item=ResponseOutputData(
                 id=item_id,
                 type="output_data",
                 data=data_str,
@@ -1765,15 +1782,15 @@ async def _map_data_content(
 
     async def _map_uri_content(
         self, content: Any, context: dict[str, Any]
-    ) -> ResponseOutputItemAddedEvent | ResponseTraceEventComplete:
+    ) -> CustomResponseOutputItemAddedEvent | ResponseTraceEventComplete:
         """Map UriContent to proper output item (image/file) based on MIME type.
 
         UriContent has a URI and MIME type, so we can create appropriate output items:
         - Images → ResponseOutputImage
         - Common files → ResponseOutputFile
         - Other URIs → ResponseTraceEventComplete (fallback for debugging)
         """
-        mime_type = getattr(content, "mime_type", "text/plain")
+        mime_type = getattr(content, "media_type", None) or getattr(content, "mime_type", None) or "text/plain"
         uri = getattr(content, "uri", "")
         item_id = f"item_{uuid.uuid4().hex[:16]}"
 
@@ -1790,9 +1807,9 @@ async def _map_uri_content(
 
         # Handle images
         if mime_type.startswith("image/"):
-            return ResponseOutputItemAddedEvent(
+            return CustomResponseOutputItemAddedEvent(
                 type="response.output_item.added",
-                item=ResponseOutputImage(  # type: ignore[arg-type]
+                item=ResponseOutputImage(
                     id=item_id,
                     type="output_image",
                     image_url=uri,
@@ -1819,9 +1836,9 @@ async def _map_uri_content(
             # Extract filename from URI or use generic name
             filename = uri.split("/")[-1] if "/" in uri else f"output.{mime_type.split('/')[-1]}"
 
-            return ResponseOutputItemAddedEvent(
+            return CustomResponseOutputItemAddedEvent(
                 type="response.output_item.added",
-                item=ResponseOutputFile(  # type: ignore[arg-type]
+                item=ResponseOutputFile(
                     id=item_id,
                     type="output_file",
                     filename=filename,

python/packages/devui/agent_framework_devui/models/_openai_custom.py

@@ -11,6 +11,7 @@
 from dataclasses import dataclass
 from typing import Any, Literal
 
+from openai.types.responses import Response, ResponseOutputItem
 from pydantic import BaseModel, ConfigDict
 
 # Custom Agent Framework OpenAI event types for structured data
@@ -56,8 +57,8 @@ class ExecutorActionItem(BaseModel):
 class CustomResponseOutputItemAddedEvent(BaseModel):
     """Custom version of ResponseOutputItemAddedEvent that accepts any item type.
 
-    This allows us to emit executor action items while maintaining the same
-    event structure as OpenAI's standard.
+    This supports executor actions and media output items while maintaining
+    the same event structure as OpenAI's standard.
     """
 
     type: Literal["response.output_item.added"] = "response.output_item.added"
@@ -274,6 +275,21 @@ class ResponseOutputData(BaseModel):
     """Optional description of the data."""
 
 
+class MediaResponse(Response):
+    """Responses envelope with validated DevUI media output extensions."""
+
+    # The SDK's output union excludes the media items already supported by DevUI.
+    output: list[ResponseOutputItem | ResponseOutputImage | ResponseOutputFile | ResponseOutputData]  # pyright: ignore[reportIncompatibleVariableOverride]
+
+
+class MediaResponseEvent(BaseModel):
+    """Terminal event retaining the media response's extended serialization schema."""
+
+    type: Literal["response.completed", "response.failed"]
+    sequence_number: int
+    response: MediaResponse
+
+
 # Agent Framework extension fields
 class AgentFrameworkExtraBody(BaseModel):
     """Agent Framework specific routing fields for OpenAI requests."""

python/packages/devui/frontend/package.json

@@ -8,6 +8,7 @@
     "build": "tsc -b && vite build",
     "lint": "eslint .",
     "test:tracing": "node --test test/trace-polling.test.mjs",
+    "test:audio": "node --test test/response-media.test.mjs",
     "preview": "vite preview"
   },
   "dependencies": {