feat: add native audio input encoding to provider formats

docs/reference/provider-api.md

@@ -74,20 +74,51 @@ response = await model.chat([
 print(response.text)
 ```
 
-Image and video inputs use paths, URLs, data URLs, or bytes. For raw bytes, supply `media_type=`. Select a format and model that accept the media; for example, a Gemini model with video support:
+### Multimodal content
+
+User messages accept text and `Image`, `Audio`, and `Video` parts. The public content shape is independent of the selected provider:
+
+| Part | Constructor | Source fields |
+| --- | --- | --- |
+| `Image` | `republic.image(source, media_type=...)` | `media_type`, `data` or `url` |
+| `Audio` | `republic.audio(source, media_type=...)` | `media_type`, `data` or `url` |
+| `Video` | `republic.video(source, media_type=...)` | `media_type`, `data` or `url` |
+
+`media_type` is a MIME type such as `audio/mpeg`, rather than a protocol's format label such as `mp3`. `data` holds raw bytes; `url` holds a remote reference. The loader helpers handle sources consistently:
+
+- A local path or `Path` loads bytes and infers the MIME type from the filename unless `media_type=` is supplied.
+- An HTTP(S) or `gs://` URL keeps a reference without downloading it. MIME inference uses the URL path; query strings do not change its type. Supply `media_type=` for URLs without a recognizable filename.
+- A base64 data URL decodes bytes and uses the MIME type declared in that URL.
+- Raw bytes require an explicit `media_type=`.
+
+Use the parts directly in a user message. Each API format owns its wire encoding:
 
 ```python
 model = republic.get_model("google:MODEL_ID")
-response = await model.chat([
+response = await model.chat(
     republic.user(
         "Describe these inputs.",
         republic.image("path/to/image.png"),
+        republic.audio("path/to/audio.wav"),
         republic.video("path/to/video.mp4"),
-    ),
-])
+    )
+)
 print(response.text)
 ```
 
+The same message works with `model.stream()`. The following table describes Republic's input encoders; the chosen service and model must also support that input.
+
+| API format | Images | Audio | Video |
+| --- | --- | --- | --- |
+| Chat Completions | Data or URL | Inline WAV or MP3 | Data or URL, when the service supports `video_url` |
+| Responses | Data or URL | Rejected | Rejected |
+| Messages | Data or URL | Rejected | Rejected |
+| Gemini | Inline data or file reference | Inline data or file reference | Inline data or file reference |
+
+Provider dialects may extend a format. For example, OpenRouter's Chat Completions encoder also maps OGG, FLAC, and its other supported MIME types to their protocol format labels. Custom Chat Completions dialects can override `audio_content()` or extend `AUDIO_FORMATS`. Application code still passes the same `Audio` part.
+
+Unsupported audio combinations raise `errors.UnsupportedFeatureError` before sending a request. Republic does not transcode audio or fetch remote references to make them inline.
+
 After executing tool calls from a response, return the results with the full assistant message. Here, `results` contains `republic.tool_result(call, output)` values created by the application:
 
 ```python

src/republic/__init__.py

@@ -5,6 +5,7 @@
 
 from republic import auth, decisions, errors, events, formats, history, providers, tools
 from republic._content import (
+    Audio,
     Image,
     Message,
     ProviderData,
@@ -15,6 +16,7 @@
     ToolResult,
     Video,
     assistant,
+    audio,
     image,
     system,
     tool_result,
@@ -42,6 +44,7 @@
 )
 
 __all__ = [
+    "Audio",
     "BuiltinToolCall",
     "ChatModel",
     "ChatOptions",
@@ -67,6 +70,7 @@
     "Video",
     "all_providers",
     "assistant",
+    "audio",
     "auth",
     "decisions",
     "errors",

src/republic/_content.py

@@ -9,6 +9,7 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar
+from urllib.parse import urlsplit
 
 if TYPE_CHECKING:
     from PIL.Image import Image as PILImage
@@ -72,6 +73,13 @@ def to_pil(self) -> PILImage:
         return PILImageModule.open(BytesIO(self.data))
 
 
+@dataclass(frozen=True)
+class Audio(_Media):
+    """Audio input, represented by MIME type and inline bytes or a remote URL."""
+
+    kind: ClassVar[str] = "audio"
+
+
 @dataclass(frozen=True)
 class Video(_Media):
     kind: ClassVar[str] = "video"
@@ -89,7 +97,7 @@ class ProviderData:
     payload: Mapping[str, Any]
 
 
-Part = Text | Reasoning | Image | Video | ProviderData
+Part = Text | Reasoning | Image | Audio | Video | ProviderData
 
 _MediaT = TypeVar("_MediaT", bound=_Media)
 
@@ -143,7 +151,7 @@ def reasoning(self) -> str:
         return "".join(part.text for part in self.parts if isinstance(part, Reasoning))
 
 
-UserContent = str | Image | Video
+UserContent = str | Image | Audio | Video
 
 
 def _to_part(content: UserContent) -> Part:
@@ -156,7 +164,7 @@ def system(text: str) -> Message:
 
 
 def user(*content: UserContent) -> Message:
-    """Build a user message from text, images, and videos."""
+    """Build a user message from text, images, audio, and videos."""
     return Message("user", tuple(_to_part(item) for item in content))
 
 
@@ -189,6 +197,11 @@ def image(source: str | os.PathLike[str] | bytes, *, media_type: str | None = No
     return _load_media(Image, source, media_type)
 
 
+def audio(source: str | os.PathLike[str] | bytes, *, media_type: str | None = None) -> Audio:
+    """Load audio from a path, a URL, a data URL, or raw bytes."""
+    return _load_media(Audio, source, media_type)
+
+
 def video(source: str | os.PathLike[str] | bytes, *, media_type: str | None = None) -> Video:
     """Load a video from a path, a URL, a data URL, or raw bytes."""
     return _load_media(Video, source, media_type)
@@ -218,7 +231,8 @@ def media_from_data_url(media_class: type[_MediaT], url: str) -> _MediaT:
 
 
 def _guess_media_type(media_class: type[_Media], name: str) -> str:
-    guessed, _ = mimetypes.guess_type(name)
+    path = urlsplit(name).path if name.startswith(_REMOTE_PREFIXES) else name
+    guessed, _ = mimetypes.guess_type(path)
     if guessed is None:
         raise ValueError(f"Cannot guess the media type of {name!r}; pass media_type explicitly")
     return guessed

src/republic/formats/chat.py

@@ -4,9 +4,9 @@
 
 import json
 from collections.abc import Iterable, Mapping
-from typing import Any
+from typing import Any, ClassVar
 
-from republic._content import Image, Message, Text, Tool, Video, media_from_data_url
+from republic._content import Audio, Image, Message, Text, Tool, Video, media_from_data_url
 from republic._options import ReasoningEffort
 from republic._response import Citation, FinishReason
 from republic.errors import APIResponseError
@@ -25,6 +25,7 @@
     approximate_location,
     deep_merge,
     strict_schema,
+    unsupported_media,
     unsupported_tool,
 )
 
@@ -39,6 +40,7 @@
 
 class ChatFormat(ChatApiFormat):
     name = "chat"
+    AUDIO_FORMATS: ClassVar[Mapping[str, str]] = {"audio/wav": "wav", "audio/x-wav": "wav", "audio/mpeg": "mp3"}
 
     def chat_request(self, request: ChatRequest, *, stream: bool) -> HttpRequest:
         body: dict[str, Any] = {
@@ -136,6 +138,15 @@ def content_text(self, message: Mapping[str, Any]) -> str | None:
         """Answer text in a message or delta; override for servers that return content parts."""
         return message.get("content")
 
+    def audio_content(self, audio: Audio) -> dict[str, Any]:
+        """Encode inline audio; provider dialects can extend ``AUDIO_FORMATS``."""
+        if audio.data is None:
+            raise unsupported_media(self.name, "remote audio")
+        audio_format = self.AUDIO_FORMATS.get(audio.media_type)
+        if audio_format is None:
+            raise unsupported_media(self.name, audio.media_type)
+        return {"type": "input_audio", "input_audio": {"data": audio.base64_data, "format": audio_format}}
+
     def assistant_fields(self, message: Message) -> dict[str, Any]:
         """Fields added to an assistant message sent back, such as reasoning some servers require."""
         return {}
@@ -180,7 +191,7 @@ def _message_entries(api_format: ChatFormat, message: Message) -> list[dict[str,
         case "system":
             return [{"role": "system", "content": message.text}]
         case "user":
-            return [{"role": "user", "content": _user_content(message)}]
+            return [{"role": "user", "content": _user_content(api_format, message)}]
         case "assistant":
             entry: dict[str, Any] = {"role": "assistant", "content": message.text or None}
             if message.tool_calls:
@@ -197,7 +208,7 @@ def _message_entries(api_format: ChatFormat, message: Message) -> list[dict[str,
             ]
 
 
-def _user_content(message: Message) -> str | list[dict[str, Any]]:
+def _user_content(api_format: ChatFormat, message: Message) -> str | list[dict[str, Any]]:
     if all(isinstance(part, Text) for part in message.parts):
         return message.text
     content: list[dict[str, Any]] = []
@@ -207,6 +218,8 @@ def _user_content(message: Message) -> str | list[dict[str, Any]]:
                 content.append({"type": "text", "text": text})
             case Image():
                 content.append({"type": "image_url", "image_url": {"url": part.data_url}})
+            case Audio():
+                content.append(api_format.audio_content(part))
             case Video():
                 content.append({"type": "video_url", "video_url": {"url": part.data_url}})
     return content

src/republic/formats/gemini.py

@@ -9,7 +9,7 @@
 from collections.abc import Iterable, Mapping
 from typing import Any
 
-from republic._content import Image, Message, ProviderData, Text, Tool, ToolResult, Video, _Media
+from republic._content import Audio, Image, Message, ProviderData, Text, Tool, ToolResult, Video, _Media
 from republic._options import ReasoningEffort, ToolChoice
 from republic._response import BuiltinToolCall, Citation, FinishReason
 from republic.errors import APIResponseError, UnsupportedFeatureError
@@ -31,6 +31,10 @@
     unsupported_tool,
 )
 
+# The standard library reports the legacy aliases for `.wav` and `.aiff` files, and Google
+# accepts only the canonical names they stand for.
+_CANONICAL_MIME_TYPES = {"audio/x-wav": "audio/wav", "audio/x-aiff": "audio/aiff"}
+
 # Gemini only sends call ids for some models. Keep the original so a generated
 # placeholder id is never sent back.
 _CALL_ID = "gemini_call_id"
@@ -254,7 +258,7 @@ def _content(message: Message) -> dict[str, Any]:
     if message.role == "tool":
         return {"role": "user", "parts": [_function_response(result) for result in message.tool_results]}
     parts: list[Mapping[str, Any]] = provider_payloads(message, GeminiFormat.name)
-    parts.extend(_part(part) for part in message.parts if isinstance(part, Text | Image | Video))
+    parts.extend(_part(part) for part in message.parts if isinstance(part, Text | Image | Audio | Video))
     for call in message.tool_calls:
         function_call: dict[str, Any] = {"name": call.name, "args": call.args}
         if call_id := call.metadata.get(_CALL_ID):
@@ -279,6 +283,7 @@ def _function_response(result: ToolResult) -> dict[str, Any]:
 def _part(part: Text | _Media) -> dict[str, Any]:
     if isinstance(part, Text):
         return {"text": part.text}
+    media_type = _CANONICAL_MIME_TYPES.get(part.media_type, part.media_type)
     if part.url is not None:
-        return {"fileData": {"mimeType": part.media_type, "fileUri": part.url}}
-    return {"inlineData": {"mimeType": part.media_type, "data": part.base64_data}}
+        return {"fileData": {"mimeType": media_type, "fileUri": part.url}}
+    return {"inlineData": {"mimeType": media_type, "data": part.base64_data}}

src/republic/formats/messages.py

@@ -6,7 +6,7 @@
 from collections.abc import Iterable, Mapping
 from typing import Any, ClassVar
 
-from republic._content import Image, Message, ProviderData, Text, Tool, Video
+from republic._content import Audio, Image, Message, ProviderData, Text, Tool, Video
 from republic._options import ReasoningEffort, ToolChoice
 from republic._response import BuiltinToolCall, Citation, FinishReason
 from republic.errors import APIResponseError
@@ -330,6 +330,8 @@ def _user_block(part: object) -> dict[str, Any]:
                 "type": "image",
                 "source": {"type": "base64", "media_type": part.media_type, "data": part.base64_data},
             }
+        case Audio():
+            raise unsupported_media(MessagesFormat.name, "audio")
         case Video():
             raise unsupported_media(MessagesFormat.name, "video")
     raise TypeError(f"Unexpected user content: {part!r}")

src/republic/formats/responses.py

@@ -7,7 +7,7 @@
 from collections.abc import Iterable, Mapping
 from typing import Any
 
-from republic._content import Image, Message, ProviderData, Text, Tool, Video
+from republic._content import Audio, Image, Message, ProviderData, Text, Tool, Video
 from republic._options import ReasoningEffort
 from republic._response import BuiltinToolCall, Citation, FinishReason
 from republic.errors import APIResponseError
@@ -184,6 +184,8 @@ def _user_part(part: object) -> dict[str, Any]:
             return {"type": "input_text", "text": text}
         case Image():
             return {"type": "input_image", "image_url": part.data_url}
+        case Audio():
+            raise unsupported_media(ResponsesFormat.name, "audio")
         case Video():
             raise unsupported_media(ResponsesFormat.name, "video")
     raise TypeError(f"Unexpected user content: {part!r}")

src/republic/providers/openrouter.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from collections.abc import Awaitable, Callable
-from typing import Any, Self
+from collections.abc import Awaitable, Callable, Mapping
+from typing import Any, ClassVar, Self
 
 import httpx2
 from authlib.common.security import generate_token
@@ -68,6 +68,18 @@ async def login(cls, *, on_authorize: Callable[[str], Awaitable[str]]) -> Self:
 class OpenRouterChatFormat(ChatFormat):
     """OpenRouter's chat completions, which take reasoning options as one ``reasoning`` object."""
 
+    AUDIO_FORMATS: ClassVar[Mapping[str, str]] = {
+        **ChatFormat.AUDIO_FORMATS,
+        "audio/aiff": "aiff",
+        "audio/x-aiff": "aiff",
+        "audio/aac": "aac",
+        "audio/ogg": "ogg",
+        "audio/flac": "flac",
+        "audio/x-flac": "flac",
+        "audio/mp4": "m4a",
+        "audio/x-m4a": "m4a",
+    }
+
     def reasoning_fields(self, effort: ReasoningEffort | None, *, include_reasoning: bool) -> dict[str, Any]:
         reasoning: dict[str, Any] = {}
         if effort is not None: