diff --git a/docs/reference/provider-api.md b/docs/reference/provider-api.md index 4a4aa40..126d3a2 100644 --- a/docs/reference/provider-api.md +++ b/docs/reference/provider-api.md @@ -74,20 +74,51 @@ response = await model.chat([ print(response.text) ``` -Image and video inputs use paths, URLs, data URLs, or bytes. For raw bytes, supply `media_type=`. Select a format and model that accept the media; for example, a Gemini model with video support: +### Multimodal content + +User messages accept text and `Image`, `Audio`, and `Video` parts. The public content shape is independent of the selected provider: + +| Part | Constructor | Source fields | +| --- | --- | --- | +| `Image` | `republic.image(source, media_type=...)` | `media_type`, `data` or `url` | +| `Audio` | `republic.audio(source, media_type=...)` | `media_type`, `data` or `url` | +| `Video` | `republic.video(source, media_type=...)` | `media_type`, `data` or `url` | + +`media_type` is a MIME type such as `audio/mpeg`, rather than a protocol's format label such as `mp3`. `data` holds raw bytes; `url` holds a remote reference. The loader helpers handle sources consistently: + +- A local path or `Path` loads bytes and infers the MIME type from the filename unless `media_type=` is supplied. +- An HTTP(S) or `gs://` URL keeps a reference without downloading it. MIME inference uses the URL path; query strings do not change its type. Supply `media_type=` for URLs without a recognizable filename. +- A base64 data URL decodes bytes and uses the MIME type declared in that URL. +- Raw bytes require an explicit `media_type=`. + +Use the parts directly in a user message. Each API format owns its wire encoding: ```python model = republic.get_model("google:MODEL_ID") -response = await model.chat([ +response = await model.chat( republic.user( "Describe these inputs.", republic.image("path/to/image.png"), + republic.audio("path/to/audio.wav"), republic.video("path/to/video.mp4"), - ), -]) + ) +) print(response.text) ``` +The same message works with `model.stream()`. The following table describes Republic's input encoders; the chosen service and model must also support that input. + +| API format | Images | Audio | Video | +| --- | --- | --- | --- | +| Chat Completions | Data or URL | Inline WAV or MP3 | Data or URL, when the service supports `video_url` | +| Responses | Data or URL | Rejected | Rejected | +| Messages | Data or URL | Rejected | Rejected | +| Gemini | Inline data or file reference | Inline data or file reference | Inline data or file reference | + +Provider dialects may extend a format. For example, OpenRouter's Chat Completions encoder also maps OGG, FLAC, and its other supported MIME types to their protocol format labels. Custom Chat Completions dialects can override `audio_content()` or extend `AUDIO_FORMATS`. Application code still passes the same `Audio` part. + +Unsupported audio combinations raise `errors.UnsupportedFeatureError` before sending a request. Republic does not transcode audio or fetch remote references to make them inline. + After executing tool calls from a response, return the results with the full assistant message. Here, `results` contains `republic.tool_result(call, output)` values created by the application: ```python diff --git a/src/republic/__init__.py b/src/republic/__init__.py index cca8639..f36b678 100644 --- a/src/republic/__init__.py +++ b/src/republic/__init__.py @@ -5,6 +5,7 @@ from republic import auth, decisions, errors, events, formats, history, providers, tools from republic._content import ( + Audio, Image, Message, ProviderData, @@ -15,6 +16,7 @@ ToolResult, Video, assistant, + audio, image, system, tool_result, @@ -42,6 +44,7 @@ ) __all__ = [ + "Audio", "BuiltinToolCall", "ChatModel", "ChatOptions", @@ -67,6 +70,7 @@ "Video", "all_providers", "assistant", + "audio", "auth", "decisions", "errors", diff --git a/src/republic/_content.py b/src/republic/_content.py index 8ab9e8f..6029f75 100644 --- a/src/republic/_content.py +++ b/src/republic/_content.py @@ -9,6 +9,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar +from urllib.parse import urlsplit if TYPE_CHECKING: from PIL.Image import Image as PILImage @@ -72,6 +73,13 @@ def to_pil(self) -> PILImage: return PILImageModule.open(BytesIO(self.data)) +@dataclass(frozen=True) +class Audio(_Media): + """Audio input, represented by MIME type and inline bytes or a remote URL.""" + + kind: ClassVar[str] = "audio" + + @dataclass(frozen=True) class Video(_Media): kind: ClassVar[str] = "video" @@ -89,7 +97,7 @@ class ProviderData: payload: Mapping[str, Any] -Part = Text | Reasoning | Image | Video | ProviderData +Part = Text | Reasoning | Image | Audio | Video | ProviderData _MediaT = TypeVar("_MediaT", bound=_Media) @@ -143,7 +151,7 @@ def reasoning(self) -> str: return "".join(part.text for part in self.parts if isinstance(part, Reasoning)) -UserContent = str | Image | Video +UserContent = str | Image | Audio | Video def _to_part(content: UserContent) -> Part: @@ -156,7 +164,7 @@ def system(text: str) -> Message: def user(*content: UserContent) -> Message: - """Build a user message from text, images, and videos.""" + """Build a user message from text, images, audio, and videos.""" return Message("user", tuple(_to_part(item) for item in content)) @@ -189,6 +197,11 @@ def image(source: str | os.PathLike[str] | bytes, *, media_type: str | None = No return _load_media(Image, source, media_type) +def audio(source: str | os.PathLike[str] | bytes, *, media_type: str | None = None) -> Audio: + """Load audio from a path, a URL, a data URL, or raw bytes.""" + return _load_media(Audio, source, media_type) + + def video(source: str | os.PathLike[str] | bytes, *, media_type: str | None = None) -> Video: """Load a video from a path, a URL, a data URL, or raw bytes.""" return _load_media(Video, source, media_type) @@ -218,7 +231,8 @@ def media_from_data_url(media_class: type[_MediaT], url: str) -> _MediaT: def _guess_media_type(media_class: type[_Media], name: str) -> str: - guessed, _ = mimetypes.guess_type(name) + path = urlsplit(name).path if name.startswith(_REMOTE_PREFIXES) else name + guessed, _ = mimetypes.guess_type(path) if guessed is None: raise ValueError(f"Cannot guess the media type of {name!r}; pass media_type explicitly") return guessed diff --git a/src/republic/formats/chat.py b/src/republic/formats/chat.py index a5d7e17..ba7192c 100644 --- a/src/republic/formats/chat.py +++ b/src/republic/formats/chat.py @@ -4,9 +4,9 @@ import json from collections.abc import Iterable, Mapping -from typing import Any +from typing import Any, ClassVar -from republic._content import Image, Message, Text, Tool, Video, media_from_data_url +from republic._content import Audio, Image, Message, Text, Tool, Video, media_from_data_url from republic._options import ReasoningEffort from republic._response import Citation, FinishReason from republic.errors import APIResponseError @@ -25,6 +25,7 @@ approximate_location, deep_merge, strict_schema, + unsupported_media, unsupported_tool, ) @@ -39,6 +40,7 @@ class ChatFormat(ChatApiFormat): name = "chat" + AUDIO_FORMATS: ClassVar[Mapping[str, str]] = {"audio/wav": "wav", "audio/x-wav": "wav", "audio/mpeg": "mp3"} def chat_request(self, request: ChatRequest, *, stream: bool) -> HttpRequest: body: dict[str, Any] = { @@ -136,6 +138,15 @@ def content_text(self, message: Mapping[str, Any]) -> str | None: """Answer text in a message or delta; override for servers that return content parts.""" return message.get("content") + def audio_content(self, audio: Audio) -> dict[str, Any]: + """Encode inline audio; provider dialects can extend ``AUDIO_FORMATS``.""" + if audio.data is None: + raise unsupported_media(self.name, "remote audio") + audio_format = self.AUDIO_FORMATS.get(audio.media_type) + if audio_format is None: + raise unsupported_media(self.name, audio.media_type) + return {"type": "input_audio", "input_audio": {"data": audio.base64_data, "format": audio_format}} + def assistant_fields(self, message: Message) -> dict[str, Any]: """Fields added to an assistant message sent back, such as reasoning some servers require.""" return {} @@ -180,7 +191,7 @@ def _message_entries(api_format: ChatFormat, message: Message) -> list[dict[str, case "system": return [{"role": "system", "content": message.text}] case "user": - return [{"role": "user", "content": _user_content(message)}] + return [{"role": "user", "content": _user_content(api_format, message)}] case "assistant": entry: dict[str, Any] = {"role": "assistant", "content": message.text or None} if message.tool_calls: @@ -197,7 +208,7 @@ def _message_entries(api_format: ChatFormat, message: Message) -> list[dict[str, ] -def _user_content(message: Message) -> str | list[dict[str, Any]]: +def _user_content(api_format: ChatFormat, message: Message) -> str | list[dict[str, Any]]: if all(isinstance(part, Text) for part in message.parts): return message.text content: list[dict[str, Any]] = [] @@ -207,6 +218,8 @@ def _user_content(message: Message) -> str | list[dict[str, Any]]: content.append({"type": "text", "text": text}) case Image(): content.append({"type": "image_url", "image_url": {"url": part.data_url}}) + case Audio(): + content.append(api_format.audio_content(part)) case Video(): content.append({"type": "video_url", "video_url": {"url": part.data_url}}) return content diff --git a/src/republic/formats/gemini.py b/src/republic/formats/gemini.py index 7525f93..8cf685b 100644 --- a/src/republic/formats/gemini.py +++ b/src/republic/formats/gemini.py @@ -9,7 +9,7 @@ from collections.abc import Iterable, Mapping from typing import Any -from republic._content import Image, Message, ProviderData, Text, Tool, ToolResult, Video, _Media +from republic._content import Audio, Image, Message, ProviderData, Text, Tool, ToolResult, Video, _Media from republic._options import ReasoningEffort, ToolChoice from republic._response import BuiltinToolCall, Citation, FinishReason from republic.errors import APIResponseError, UnsupportedFeatureError @@ -31,6 +31,10 @@ unsupported_tool, ) +# The standard library reports the legacy aliases for `.wav` and `.aiff` files, and Google +# accepts only the canonical names they stand for. +_CANONICAL_MIME_TYPES = {"audio/x-wav": "audio/wav", "audio/x-aiff": "audio/aiff"} + # Gemini only sends call ids for some models. Keep the original so a generated # placeholder id is never sent back. _CALL_ID = "gemini_call_id" @@ -254,7 +258,7 @@ def _content(message: Message) -> dict[str, Any]: if message.role == "tool": return {"role": "user", "parts": [_function_response(result) for result in message.tool_results]} parts: list[Mapping[str, Any]] = provider_payloads(message, GeminiFormat.name) - parts.extend(_part(part) for part in message.parts if isinstance(part, Text | Image | Video)) + parts.extend(_part(part) for part in message.parts if isinstance(part, Text | Image | Audio | Video)) for call in message.tool_calls: function_call: dict[str, Any] = {"name": call.name, "args": call.args} if call_id := call.metadata.get(_CALL_ID): @@ -279,6 +283,7 @@ def _function_response(result: ToolResult) -> dict[str, Any]: def _part(part: Text | _Media) -> dict[str, Any]: if isinstance(part, Text): return {"text": part.text} + media_type = _CANONICAL_MIME_TYPES.get(part.media_type, part.media_type) if part.url is not None: - return {"fileData": {"mimeType": part.media_type, "fileUri": part.url}} - return {"inlineData": {"mimeType": part.media_type, "data": part.base64_data}} + return {"fileData": {"mimeType": media_type, "fileUri": part.url}} + return {"inlineData": {"mimeType": media_type, "data": part.base64_data}} diff --git a/src/republic/formats/messages.py b/src/republic/formats/messages.py index 593a12a..587a1ba 100644 --- a/src/republic/formats/messages.py +++ b/src/republic/formats/messages.py @@ -6,7 +6,7 @@ from collections.abc import Iterable, Mapping from typing import Any, ClassVar -from republic._content import Image, Message, ProviderData, Text, Tool, Video +from republic._content import Audio, Image, Message, ProviderData, Text, Tool, Video from republic._options import ReasoningEffort, ToolChoice from republic._response import BuiltinToolCall, Citation, FinishReason from republic.errors import APIResponseError @@ -330,6 +330,8 @@ def _user_block(part: object) -> dict[str, Any]: "type": "image", "source": {"type": "base64", "media_type": part.media_type, "data": part.base64_data}, } + case Audio(): + raise unsupported_media(MessagesFormat.name, "audio") case Video(): raise unsupported_media(MessagesFormat.name, "video") raise TypeError(f"Unexpected user content: {part!r}") diff --git a/src/republic/formats/responses.py b/src/republic/formats/responses.py index a381cd5..de7804a 100644 --- a/src/republic/formats/responses.py +++ b/src/republic/formats/responses.py @@ -7,7 +7,7 @@ from collections.abc import Iterable, Mapping from typing import Any -from republic._content import Image, Message, ProviderData, Text, Tool, Video +from republic._content import Audio, Image, Message, ProviderData, Text, Tool, Video from republic._options import ReasoningEffort from republic._response import BuiltinToolCall, Citation, FinishReason from republic.errors import APIResponseError @@ -184,6 +184,8 @@ def _user_part(part: object) -> dict[str, Any]: return {"type": "input_text", "text": text} case Image(): return {"type": "input_image", "image_url": part.data_url} + case Audio(): + raise unsupported_media(ResponsesFormat.name, "audio") case Video(): raise unsupported_media(ResponsesFormat.name, "video") raise TypeError(f"Unexpected user content: {part!r}") diff --git a/src/republic/providers/openrouter.py b/src/republic/providers/openrouter.py index 7cd60bc..52ed493 100644 --- a/src/republic/providers/openrouter.py +++ b/src/republic/providers/openrouter.py @@ -1,7 +1,7 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable -from typing import Any, Self +from collections.abc import Awaitable, Callable, Mapping +from typing import Any, ClassVar, Self import httpx2 from authlib.common.security import generate_token @@ -68,6 +68,18 @@ async def login(cls, *, on_authorize: Callable[[str], Awaitable[str]]) -> Self: class OpenRouterChatFormat(ChatFormat): """OpenRouter's chat completions, which take reasoning options as one ``reasoning`` object.""" + AUDIO_FORMATS: ClassVar[Mapping[str, str]] = { + **ChatFormat.AUDIO_FORMATS, + "audio/aiff": "aiff", + "audio/x-aiff": "aiff", + "audio/aac": "aac", + "audio/ogg": "ogg", + "audio/flac": "flac", + "audio/x-flac": "flac", + "audio/mp4": "m4a", + "audio/x-m4a": "m4a", + } + def reasoning_fields(self, effort: ReasoningEffort | None, *, include_reasoning: bool) -> dict[str, Any]: reasoning: dict[str, Any] = {} if effort is not None: diff --git a/src/republic/py.typed b/src/republic/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_audio.py b/tests/test_audio.py new file mode 100644 index 0000000..21fe3ea --- /dev/null +++ b/tests/test_audio.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +import republic +from republic.errors import UnsupportedFeatureError +from tests.conftest import FakeService + + +@pytest.mark.parametrize("source_kind", ["path", "data-url", "bytes"]) +async def test_audio_sources_reach_the_provider(service: FakeService, tmp_path: Path, source_kind: str) -> None: + path = tmp_path / "voice.wav" + path.write_bytes(b"voice") + sources = { + "path": path, + "data-url": "data:audio/wav;base64,dm9pY2U=", + "bytes": b"voice", + } + service.reply_json({"choices": [{"message": {"content": "heard"}}]}) + model = republic.get_model("openai:test", api_format="chat", http_client=service.client()) + response = await model.chat( + republic.user( + republic.audio(sources[source_kind], **({"media_type": "audio/wav"} if source_kind == "bytes" else {})) + ) + ) + assert response.text == "heard" + assert service.body()["messages"][0]["content"] == [ + {"type": "input_audio", "input_audio": {"data": "dm9pY2U=", "format": "wav"}} + ] + + +def test_audio_bytes_require_a_media_type() -> None: + with pytest.raises(ValueError, match="media_type"): + republic.audio(b"voice") + + +@pytest.mark.parametrize("source", [b"voice", "https://example.test/voice.ogg"]) +async def test_gemini_encodes_audio(service: FakeService, source: bytes | str) -> None: + service.reply_json({"candidates": [{"content": {"parts": [{"text": "heard"}]}, "finishReason": "STOP"}]}) + audio = republic.audio(source, media_type="audio/ogg") + response = await republic.get_model("google:test", http_client=service.client()).chat( + republic.user("Listen", audio) + ) + expected = ( + {"inlineData": {"mimeType": "audio/ogg", "data": "dm9pY2U="}} + if isinstance(source, bytes) + else {"fileData": {"mimeType": "audio/ogg", "fileUri": source}} + ) + assert service.body()["contents"][0]["parts"] == [{"text": "Listen"}, expected] + assert response.text == "heard" + + +@pytest.mark.parametrize( + "suffix,alias,canonical", [("wav", "audio/x-wav", "audio/wav"), ("aiff", "audio/x-aiff", "audio/aiff")] +) +@pytest.mark.parametrize("source_kind", ["path", "url", "data-url"]) +async def test_gemini_encodes_wav_and_aiff_with_the_canonical_mime_type( + service: FakeService, + tmp_path: Path, + source_kind: str, + suffix: str, + alias: str, + canonical: str, +) -> None: + """Google accepts only the canonical names, not the aliases the standard library infers for these suffixes.""" + path = tmp_path / f"voice.{suffix}" + path.write_bytes(b"voice") + sources = { + "path": path, + "url": f"https://media.example.test/voice.{suffix}", + "data-url": f"data:{alias};base64,dm9pY2U=", + } + service.reply_json({"candidates": [{"content": {"parts": [{"text": "heard"}]}, "finishReason": "STOP"}]}) + response = await republic.get_model("google:test", http_client=service.client()).chat( + republic.user("Listen", republic.audio(sources[source_kind])) + ) + expected = ( + {"fileData": {"mimeType": canonical, "fileUri": sources["url"]}} + if source_kind == "url" + else {"inlineData": {"mimeType": canonical, "data": "dm9pY2U="}} + ) + assert service.body()["contents"][0]["parts"] == [{"text": "Listen"}, expected] + assert response.text == "heard" + + +@pytest.mark.parametrize("mime,wire_format", [("audio/wav", "wav"), ("audio/x-wav", "wav"), ("audio/mpeg", "mp3")]) +async def test_chat_encodes_audio(service: FakeService, mime: str, wire_format: str) -> None: + service.reply_json({"choices": [{"message": {"content": "heard"}}]}) + model = republic.get_model("openai:test", api_format="chat", http_client=service.client()) + await model.chat(republic.user(republic.audio(b"voice", media_type=mime))) + assert service.body()["messages"][0]["content"] == [ + {"type": "input_audio", "input_audio": {"data": "dm9pY2U=", "format": wire_format}} + ] + + +@pytest.mark.parametrize( + "provider,api_format", [("openai", "responses"), ("anthropic", "messages"), ("openai", "chat")] +) +async def test_unsupported_audio_fails_before_request(service: FakeService, provider: str, api_format: str) -> None: + model = republic.get_model(f"{provider}:test", api_format=api_format, http_client=service.client()) + with pytest.raises(UnsupportedFeatureError): + await model.chat(republic.user(republic.audio("https://example.test/voice.ogg", media_type="audio/ogg"))) + assert not service.requests + + +async def test_openrouter_encodes_ogg_without_transcoding(service: FakeService) -> None: + service.reply_json({"choices": [{"message": {"content": "heard"}}]}) + model = republic.get_model("openrouter:test", api_format="chat", http_client=service.client()) + await model.chat(republic.user(republic.audio(b"voice", media_type="audio/ogg"))) + assert service.body()["messages"][0]["content"] == [ + {"type": "input_audio", "input_audio": {"data": "dm9pY2U=", "format": "ogg"}} + ] + + +async def test_openai_rejects_ogg_before_sending(service: FakeService) -> None: + model = republic.get_model("openai:test", api_format="chat", http_client=service.client()) + with pytest.raises(UnsupportedFeatureError, match="audio/ogg"): + await model.chat(republic.user(republic.audio(b"voice", media_type="audio/ogg"))) + assert not service.requests + + +@pytest.mark.parametrize( + "loader,mime,suffix", + [(republic.image, "image/png", "png"), (republic.audio, "audio/ogg", "ogg"), (republic.video, "video/mp4", "mp4")], +) +async def test_signed_media_urls_reach_the_provider(service: FakeService, loader, mime: str, suffix: str) -> None: + url = f"https://media.example.test/file.{suffix}?signature=opaque#fragment" + service.reply_json({"candidates": [{"content": {"parts": [{"text": "received"}]}, "finishReason": "STOP"}]}) + model = republic.get_model("google:test", http_client=service.client()) + response = await model.chat(republic.user("Inspect", loader(url))) + assert response.text == "received" + assert service.body()["contents"][0]["parts"] == [ + {"text": "Inspect"}, + {"fileData": {"mimeType": mime, "fileUri": url}}, + ] + + +@pytest.mark.parametrize( + "provider,api_format", + [ + ("openai", "responses"), + ("codex", "responses"), + ("anthropic", "messages"), + ("github-copilot", "messages"), + ], +) +async def test_unsupported_inline_audio_is_a_format_error( + service: FakeService, + provider: str, + api_format: str, +) -> None: + model = republic.get_model(f"{provider}:test", api_format=api_format, api_key="test", http_client=service.client()) + with pytest.raises(UnsupportedFeatureError, match="audio"): + await model.chat(republic.user("Listen", republic.Audio("audio/wav", data=b"voice"))) + assert not service.requests + + +async def test_streaming_sends_audio_and_returns_text(service: FakeService) -> None: + service.reply_events([ + {"choices": [{"delta": {"content": "heard"}, "finish_reason": "stop"}]}, + "[DONE]", + ]) + model = republic.get_model("openai:test", api_format="chat", http_client=service.client()) + audio = republic.Audio("audio/wav", data=b"voice") + async with model.stream(republic.user("Listen", audio)) as stream: + _ = [event async for event in stream] + assert stream.response.text == "heard" + assert service.body()["messages"][0]["content"] == [ + {"type": "text", "text": "Listen"}, + {"type": "input_audio", "input_audio": {"data": "dm9pY2U=", "format": "wav"}}, + ] + + +async def test_follow_up_request_replays_audio(service: FakeService) -> None: + from republic.history import InMemoryHistory + + for _ in range(2): + service.reply_json({"choices": [{"message": {"content": "heard"}}]}) + history = InMemoryHistory() + model = republic.get_model("openai:test", api_format="chat", http_client=service.client(), history=history) + await model.chat(republic.user("Listen", republic.audio(b"voice", media_type="audio/wav"))) + await model.chat("What did you hear?") + assert service.body()["messages"][0]["content"] == [ + {"type": "text", "text": "Listen"}, + {"type": "input_audio", "input_audio": {"data": "dm9pY2U=", "format": "wav"}}, + ] + assert service.body()["messages"][1:] == [ + {"role": "assistant", "content": "heard"}, + {"role": "user", "content": "What did you hear?"}, + ] + + +@pytest.mark.parametrize("mime", ["audio/pcm16", "audio/pcm24", "audio/webm"]) +async def test_openrouter_rejects_unsupported_audio_mime_types(service: FakeService, mime: str) -> None: + model = republic.get_model("openrouter:test", api_format="chat", http_client=service.client()) + with pytest.raises(UnsupportedFeatureError): + await model.chat(republic.user(republic.audio(b"voice", media_type=mime))) + assert not service.requests