diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 1d4d684e8ac..32c0879f2dc 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -2511,7 +2511,10 @@ async def handle( elif content.type == "image_generation_tool_result" and content.outputs is not None: for event in self._close(): yield event - async for event in self._stream.output_item_image_gen_call(str(content.outputs)): + result = _image_generation_result(content.outputs) + if result is None: + logger.warning("Image generation result has no base64 image; emitting it with an empty result.") + async for event in self._stream.output_item_image_gen_call(result or ""): yield event elif content.type == "mcp_server_tool_call": @@ -3534,6 +3537,38 @@ def _json_safe_to_str(value: Any | None) -> str: return json.dumps(str(value)) +def _image_generation_result(outputs: Any) -> str | None: + """Return the base64 image of an image generation tool result, as the Responses API expects. + + Only a non-empty, valid base64 string or the valid base64 payload of a base64 data URI qualifies. Other URIs + (https, file, blob), data URIs without the base64 marker and malformed payloads are not base64, so they give + None. + """ + if isinstance(outputs, str): + return outputs if _is_base64(outputs) else None + items: Sequence[Any] = [outputs] + if isinstance(outputs, Sequence) and not isinstance(outputs, (bytes, bytearray)): + items = cast(Sequence[Any], outputs) + for item in items: + if isinstance(item, Content) and item.uri and item.uri.startswith("data:"): + # RFC 2397: only the metadata before the first comma can carry the marker, as its last parameter. + metadata, _, payload = item.uri.partition(",") + if metadata.endswith(";base64") and _is_base64(payload): + return payload + return None + + +def _is_base64(value: str) -> bool: + """Check whether a string is non-empty, valid base64.""" + if not value: + return False + try: + base64.b64decode(value, validate=True) + except ValueError: + return False + return True + + def _reasoning_encrypted_content(content: Content) -> str | None: """Return the opaque reasoning payload used for stateless replay.""" encrypted_content = content.protected_data or content.additional_properties.get("encrypted_content") diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index ae971c82199..f1e4c20c93b 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -618,6 +618,82 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() ] +@pytest.mark.parametrize( + ("outputs", "expected"), + [ + (Content.from_uri("data:image/png;base64,iVBORw0KGgo="), "iVBORw0KGgo="), + ([Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), + ((Content.from_uri("data:image/png;base64,iVBORw0KGgo="),), "iVBORw0KGgo="), + ([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), + (["caption", Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), + ("iVBORw0KGgo=", "iVBORw0KGgo="), + (Content.from_uri("data:image/png;name=cat.png;base64,iVBORw0KGgo="), "iVBORw0KGgo="), + ( + [ + Content.from_uri("data:image/png;base64,not-base64!"), + Content.from_uri("data:image/png;base64,iVBORw0KGgo="), + ], + "iVBORw0KGgo=", + ), + (Content.from_uri("https://example.com/cat.png", media_type="image/png"), ""), + (Content.from_uri("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"), ""), + (Content.from_uri("data:image/png;base64,not-base64!"), ""), + (Content.from_uri("data:image/png;base64,iVBORw0KGgo"), ""), + (Content.from_uri("data:image/png;base64,iVBOR\nw0KGgo="), ""), + (Content.from_uri("data:image/png;base64,"), ""), + (Content.from_uri("data:text/plain,x;base64,iVBORw0KGgo="), ""), + (Content.from_hosted_file("file_1"), ""), + ("https://example.com/cat.png", ""), + ("not-base64!", ""), + ("", ""), + (b"\x89PNG\r\n\x1a\n", ""), + ({"result": "iVBORw0KGgo="}, ""), + ], + ids=[ + "data-uri-content", + "content-list", + "content-tuple", + "image-after-text", + "image-after-string", + "base64-string", + "data-uri-with-parameter", + "image-after-malformed-payload", + "https-uri-is-not-base64", + "data-uri-without-base64", + "data-uri-malformed-payload", + "data-uri-missing-padding", + "data-uri-whitespace-in-payload", + "data-uri-empty-payload", + "base64-marker-inside-payload", + "hosted-file", + "url-string-is-not-base64", + "malformed-string", + "empty-string", + "bytes", + "mapping", + ], +) +async def test_output_item_tracker_emits_image_generation_result_as_base64( + outputs: Any, expected: str, caplog: pytest.LogCaptureFixture +) -> None: + stream = ResponseEventStream(response_id="resp_image") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + events: list[Any] = [] + + with caplog.at_level(logging.WARNING): + async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)): + events.append(event) + events.extend(tracker.close()) + + done = [e for e in events if (e.get("type") if isinstance(e, Mapping) else e.type) == "response.output_item.done"] + assert len(done) == 1 + assert done[0]["item"]["type"] == "image_generation_call" + assert done[0]["item"]["result"] == expected + assert ("has no base64 image" in caplog.text) is (expected == "") + + async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message() -> None: stream = ResponseEventStream(response_id="resp_mixed") stream.emit_created()