From a292abf492377571c3c1b1babb7548c1756d378a Mon Sep 17 00:00:00 2001 From: Shanto Date: Tue, 13 Oct 2026 22:32:42 +0600 Subject: [PATCH 1/4] Send the base64 image as the foundry_hosting image_generation_call result --- .../_responses.py | 16 +++++++++- .../foundry_hosting/tests/test_responses.py | 29 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 1d4d684e8ac..2fe2dee4d02 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -2511,7 +2511,8 @@ async def handle( elif content.type == "image_generation_tool_result" and content.outputs is not None: for event in self._close(): yield event - async for event in self._stream.output_item_image_gen_call(str(content.outputs)): + result = _image_generation_result(content.outputs) or str(content.outputs) + async for event in self._stream.output_item_image_gen_call(result): yield event elif content.type == "mcp_server_tool_call": @@ -3534,6 +3535,19 @@ def _json_safe_to_str(value: Any | None) -> str: return json.dumps(str(value)) +def _image_generation_result(outputs: Any) -> str | None: + """Return the base64 image of an image generation tool result, as the Responses API expects.""" + items: Sequence[Any] = [outputs] + if isinstance(outputs, Sequence) and not isinstance(outputs, (str, bytes, bytearray)): + items = cast(Sequence[Any], outputs) + for item in items: + if isinstance(item, Content) and item.uri: + return item.uri.split(";base64,", 1)[1] if ";base64," in item.uri else item.uri + if isinstance(item, str): + return item + return None + + def _reasoning_encrypted_content(content: Content) -> str | None: """Return the opaque reasoning payload used for stateless replay.""" encrypted_content = content.protected_data or content.additional_properties.get("encrypted_content") diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index ae971c82199..00f25fcc36a 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -618,6 +618,35 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() ] +@pytest.mark.parametrize( + ("outputs", "expected"), + [ + (Content.from_uri("data:image/png;base64,iVBORw0KGgo="), "iVBORw0KGgo="), + ([Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), + ((Content.from_uri("data:image/png;base64,iVBORw0KGgo="),), "iVBORw0KGgo="), + ([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), + (Content.from_uri("https://example.com/cat.png", media_type="image/png"), "https://example.com/cat.png"), + ("iVBORw0KGgo=", "iVBORw0KGgo="), + ], + ids=["data-uri-content", "content-list", "content-tuple", "image-after-text", "https-uri", "base64-string"], +) +async def test_output_item_tracker_emits_image_generation_result_as_base64(outputs: Any, expected: str) -> None: + stream = ResponseEventStream(response_id="resp_image") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + events: list[Any] = [] + + async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)): + events.append(event) + events.extend(tracker.close()) + + done = [e for e in events if (e.get("type") if isinstance(e, Mapping) else e.type) == "response.output_item.done"] + assert len(done) == 1 + assert done[0]["item"]["type"] == "image_generation_call" + assert done[0]["item"]["result"] == expected + + async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message() -> None: stream = ResponseEventStream(response_id="resp_mixed") stream.emit_created() From 4b48493c52b51287a15449d981ab83dd56b78397 Mon Sep 17 00:00:00 2001 From: Shanto Date: Tue, 13 Oct 2026 22:49:29 +0600 Subject: [PATCH 2/4] Ignore plain strings inside image generation outputs like hosting-responses --- .../agent_framework_foundry_hosting/_responses.py | 6 +++--- .../packages/foundry_hosting/tests/test_responses.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 2fe2dee4d02..2ca2c86c279 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -3537,14 +3537,14 @@ def _json_safe_to_str(value: Any | None) -> str: def _image_generation_result(outputs: Any) -> str | None: """Return the base64 image of an image generation tool result, as the Responses API expects.""" + if isinstance(outputs, str): + return outputs items: Sequence[Any] = [outputs] - if isinstance(outputs, Sequence) and not isinstance(outputs, (str, bytes, bytearray)): + if isinstance(outputs, Sequence) and not isinstance(outputs, (bytes, bytearray)): items = cast(Sequence[Any], outputs) for item in items: if isinstance(item, Content) and item.uri: return item.uri.split(";base64,", 1)[1] if ";base64," in item.uri else item.uri - if isinstance(item, str): - return item return None diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 00f25fcc36a..bd99dbe0dba 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -625,10 +625,19 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() ([Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), ((Content.from_uri("data:image/png;base64,iVBORw0KGgo="),), "iVBORw0KGgo="), ([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), + (["caption", Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), (Content.from_uri("https://example.com/cat.png", media_type="image/png"), "https://example.com/cat.png"), ("iVBORw0KGgo=", "iVBORw0KGgo="), ], - ids=["data-uri-content", "content-list", "content-tuple", "image-after-text", "https-uri", "base64-string"], + ids=[ + "data-uri-content", + "content-list", + "content-tuple", + "image-after-text", + "image-after-string", + "https-uri", + "base64-string", + ], ) async def test_output_item_tracker_emits_image_generation_result_as_base64(outputs: Any, expected: str) -> None: stream = ResponseEventStream(response_id="resp_image") From 57bb65d8a6d5d443794186c120d5a15a9a087e2d Mon Sep 17 00:00:00 2001 From: Shanto Date: Thu, 15 Oct 2026 14:54:10 +0600 Subject: [PATCH 3/4] Only send base64 in the foundry_hosting image_generation_call result --- .../_responses.py | 27 ++++++++++++++----- .../foundry_hosting/tests/test_responses.py | 10 +++++-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 2ca2c86c279..94edd6fa069 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -2511,8 +2511,10 @@ async def handle( elif content.type == "image_generation_tool_result" and content.outputs is not None: for event in self._close(): yield event - result = _image_generation_result(content.outputs) or str(content.outputs) - async for event in self._stream.output_item_image_gen_call(result): + result = _image_generation_result(content.outputs) + if result is None: + logger.warning("Image generation result has no base64 image; emitting it with an empty result.") + async for event in self._stream.output_item_image_gen_call(result or ""): yield event elif content.type == "mcp_server_tool_call": @@ -3536,18 +3538,31 @@ def _json_safe_to_str(value: Any | None) -> str: def _image_generation_result(outputs: Any) -> str | None: - """Return the base64 image of an image generation tool result, as the Responses API expects.""" + """Return the base64 image of an image generation tool result, as the Responses API expects. + + Only a base64 string or the payload of a base64 data URI qualifies. Other URIs (https, file, blob) are not + base64, so they give None. + """ if isinstance(outputs, str): - return outputs + return outputs if _is_base64(outputs) else None items: Sequence[Any] = [outputs] if isinstance(outputs, Sequence) and not isinstance(outputs, (bytes, bytearray)): items = cast(Sequence[Any], outputs) for item in items: - if isinstance(item, Content) and item.uri: - return item.uri.split(";base64,", 1)[1] if ";base64," in item.uri else item.uri + if isinstance(item, Content) and item.uri and item.uri.startswith("data:") and ";base64," in item.uri: + return item.uri.split(";base64,", 1)[1] return None +def _is_base64(value: str) -> bool: + """Check whether a string is valid base64.""" + try: + base64.b64decode(value, validate=True) + except ValueError: + return False + return True + + def _reasoning_encrypted_content(content: Content) -> str | None: """Return the opaque reasoning payload used for stateless replay.""" encrypted_content = content.protected_data or content.additional_properties.get("encrypted_content") diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index bd99dbe0dba..2098a75bc2f 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -626,8 +626,11 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() ((Content.from_uri("data:image/png;base64,iVBORw0KGgo="),), "iVBORw0KGgo="), ([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), (["caption", Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), - (Content.from_uri("https://example.com/cat.png", media_type="image/png"), "https://example.com/cat.png"), ("iVBORw0KGgo=", "iVBORw0KGgo="), + (Content.from_uri("https://example.com/cat.png", media_type="image/png"), ""), + (Content.from_uri("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"), ""), + (Content.from_hosted_file("file_1"), ""), + ("https://example.com/cat.png", ""), ], ids=[ "data-uri-content", @@ -635,8 +638,11 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() "content-tuple", "image-after-text", "image-after-string", - "https-uri", "base64-string", + "https-uri-is-not-base64", + "data-uri-without-base64", + "hosted-file", + "url-string-is-not-base64", ], ) async def test_output_item_tracker_emits_image_generation_result_as_base64(outputs: Any, expected: str) -> None: From 68ff1b71b44a412bf3d0b8f3644fc83584f5878d Mon Sep 17 00:00:00 2001 From: Shanto Date: Fri, 16 Oct 2026 14:20:47 +0600 Subject: [PATCH 4/4] Validate the base64 payload of data URIs in the foundry_hosting image result --- .../_responses.py | 16 +++++--- .../foundry_hosting/tests/test_responses.py | 38 +++++++++++++++++-- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 94edd6fa069..32c0879f2dc 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -3540,8 +3540,9 @@ def _json_safe_to_str(value: Any | None) -> str: def _image_generation_result(outputs: Any) -> str | None: """Return the base64 image of an image generation tool result, as the Responses API expects. - Only a base64 string or the payload of a base64 data URI qualifies. Other URIs (https, file, blob) are not - base64, so they give None. + Only a non-empty, valid base64 string or the valid base64 payload of a base64 data URI qualifies. Other URIs + (https, file, blob), data URIs without the base64 marker and malformed payloads are not base64, so they give + None. """ if isinstance(outputs, str): return outputs if _is_base64(outputs) else None @@ -3549,13 +3550,18 @@ def _image_generation_result(outputs: Any) -> str | None: if isinstance(outputs, Sequence) and not isinstance(outputs, (bytes, bytearray)): items = cast(Sequence[Any], outputs) for item in items: - if isinstance(item, Content) and item.uri and item.uri.startswith("data:") and ";base64," in item.uri: - return item.uri.split(";base64,", 1)[1] + if isinstance(item, Content) and item.uri and item.uri.startswith("data:"): + # RFC 2397: only the metadata before the first comma can carry the marker, as its last parameter. + metadata, _, payload = item.uri.partition(",") + if metadata.endswith(";base64") and _is_base64(payload): + return payload return None def _is_base64(value: str) -> bool: - """Check whether a string is valid base64.""" + """Check whether a string is non-empty, valid base64.""" + if not value: + return False try: base64.b64decode(value, validate=True) except ValueError: diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 2098a75bc2f..f1e4c20c93b 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -627,10 +627,27 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() ([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), (["caption", Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="), ("iVBORw0KGgo=", "iVBORw0KGgo="), + (Content.from_uri("data:image/png;name=cat.png;base64,iVBORw0KGgo="), "iVBORw0KGgo="), + ( + [ + Content.from_uri("data:image/png;base64,not-base64!"), + Content.from_uri("data:image/png;base64,iVBORw0KGgo="), + ], + "iVBORw0KGgo=", + ), (Content.from_uri("https://example.com/cat.png", media_type="image/png"), ""), (Content.from_uri("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"), ""), + (Content.from_uri("data:image/png;base64,not-base64!"), ""), + (Content.from_uri("data:image/png;base64,iVBORw0KGgo"), ""), + (Content.from_uri("data:image/png;base64,iVBOR\nw0KGgo="), ""), + (Content.from_uri("data:image/png;base64,"), ""), + (Content.from_uri("data:text/plain,x;base64,iVBORw0KGgo="), ""), (Content.from_hosted_file("file_1"), ""), ("https://example.com/cat.png", ""), + ("not-base64!", ""), + ("", ""), + (b"\x89PNG\r\n\x1a\n", ""), + ({"result": "iVBORw0KGgo="}, ""), ], ids=[ "data-uri-content", @@ -639,27 +656,42 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text() "image-after-text", "image-after-string", "base64-string", + "data-uri-with-parameter", + "image-after-malformed-payload", "https-uri-is-not-base64", "data-uri-without-base64", + "data-uri-malformed-payload", + "data-uri-missing-padding", + "data-uri-whitespace-in-payload", + "data-uri-empty-payload", + "base64-marker-inside-payload", "hosted-file", "url-string-is-not-base64", + "malformed-string", + "empty-string", + "bytes", + "mapping", ], ) -async def test_output_item_tracker_emits_image_generation_result_as_base64(outputs: Any, expected: str) -> None: +async def test_output_item_tracker_emits_image_generation_result_as_base64( + outputs: Any, expected: str, caplog: pytest.LogCaptureFixture +) -> None: stream = ResponseEventStream(response_id="resp_image") stream.emit_created() stream.emit_in_progress() tracker = _OutputItemTracker(stream) events: list[Any] = [] - async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)): - events.append(event) + with caplog.at_level(logging.WARNING): + async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)): + events.append(event) events.extend(tracker.close()) done = [e for e in events if (e.get("type") if isinstance(e, Mapping) else e.type) == "response.output_item.done"] assert len(done) == 1 assert done[0]["item"]["type"] == "image_generation_call" assert done[0]["item"]["result"] == expected + assert ("has no base64 image" in caplog.text) is (expected == "") async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message() -> None: