diff --git a/python/packages/bedrock/agent_framework_bedrock/_chat_client.py b/python/packages/bedrock/agent_framework_bedrock/_chat_client.py index 6d2d222d1dd..c536b09d227 100644 --- a/python/packages/bedrock/agent_framework_bedrock/_chat_client.py +++ b/python/packages/bedrock/agent_framework_bedrock/_chat_client.py @@ -32,6 +32,7 @@ ) from agent_framework._settings import SecretString, load_settings from agent_framework._telemetry import get_user_agent, mark_feature_used +from agent_framework._types import _get_data_bytes from agent_framework.exceptions import ChatClientInvalidResponseException from agent_framework.observability import ChatTelemetryLayer from boto3.session import Session as Boto3Session @@ -540,7 +541,7 @@ def _convert_message_to_content_blocks(self, message: Message) -> list[dict[str, blocks: list[dict[str, Any]] = [] for content in message.contents: block = self._convert_content_to_bedrock_block(content) - if block is None: + if block is None or ("image" in block and message.role != "user"): logger.debug("Skipping unsupported content type for Bedrock: %s", type(content)) continue blocks.append(block) @@ -550,6 +551,13 @@ def _convert_content_to_bedrock_block(self, content: Content) -> dict[str, Any] match content.type: case "text": return {"text": content.text} + case "data" if content.has_top_level_media_type("image"): + image_format = content.media_type.partition("/")[2].lower() + image_format = "jpeg" if image_format == "jpg" else image_format + if image_format not in ("gif", "jpeg", "png", "webp"): + logger.warning("Skipping %s image: Bedrock accepts gif, jpeg, png or webp.", content.media_type) + return None + return {"image": {"format": image_format, "source": {"bytes": _get_data_bytes(content)}}} case "function_call": arguments = content.parse_arguments() or {} return { diff --git a/python/packages/bedrock/tests/test_bedrock_client.py b/python/packages/bedrock/tests/test_bedrock_client.py index ee17483e84d..b2a099f1dcb 100644 --- a/python/packages/bedrock/tests/test_bedrock_client.py +++ b/python/packages/bedrock/tests/test_bedrock_client.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging from collections import deque from collections.abc import MutableMapping from typing import Any, cast @@ -416,6 +417,69 @@ def test_prepare_bedrock_messages_skips_unsupported_content_and_unmatched_tool_r assert conversation == [{"role": "user", "content": [{"text": "hello"}]}] +@pytest.mark.parametrize( + ("media_type", "image_format"), + [("image/png", "png"), ("image/jpeg", "jpeg"), ("image/jpg", "jpeg"), ("IMAGE/PNG", "png")], +) +async def test_get_response_sends_user_images_as_image_blocks(media_type: str, image_format: str) -> None: + """Image data in a user message should be sent as a Converse image block.""" + stub = _StubBedrockRuntime() + client = BedrockChatClient( + model="us.openai.gpt-6-sol", + region="us-east-1", + client=stub, # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] # pyright: ignore[reportArgumentType] + ) + image_bytes = b"fake-image-bytes" + message = Message( + role="user", + contents=[ + Content.from_text(text="What color is this image?"), + Content.from_data(data=image_bytes, media_type=media_type), + ], + ) + + await client.get_response([message]) + + assert stub.calls[0]["messages"][0]["content"] == [ + {"text": "What color is this image?"}, + {"image": {"format": image_format, "source": {"bytes": image_bytes}}}, + ] + + +def test_prepare_bedrock_messages_skips_images_outside_user_messages() -> None: + """Image data in assistant messages should still be skipped.""" + client = _make_client() + messages = [ + Message(role="user", contents=[Content.from_text(text="Draw a square.")]), + Message( + role="assistant", + contents=[Content.from_text(text="Here it is."), Content.from_data(data=b"x", media_type="image/png")], + ), + ] + + _, conversation = client._prepare_bedrock_messages(messages) + + assert conversation[1] == {"role": "assistant", "content": [{"text": "Here it is."}]} + + +@pytest.mark.parametrize("media_type", ["image/bmp", "image/svg+xml"]) +def test_prepare_bedrock_messages_skips_unsupported_image_formats( + media_type: str, caplog: pytest.LogCaptureFixture +) -> None: + """Images in formats Converse rejects should be skipped with a warning so the rest of the request still works.""" + client = _make_client() + message = Message( + role="user", + contents=[Content.from_text(text="Describe this."), Content.from_data(data=b"x", media_type=media_type)], + ) + + with caplog.at_level(logging.WARNING, logger="agent_framework.bedrock"): + _, conversation = client._prepare_bedrock_messages([message]) + + assert conversation == [{"role": "user", "content": [{"text": "Describe this."}]}] + assert media_type in caplog.text + + def test_align_tool_results_handles_pending_edge_cases() -> None: """Tool result alignment should preserve valid blocks and drop invalid or extra results.""" client = _make_client()