Python: fix(foundry_hosting): send the base64 image as the image_generation_call result

python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py

@@ -2511,7 +2511,10 @@ async def handle(
         elif content.type == "image_generation_tool_result" and content.outputs is not None:
             for event in self._close():
                 yield event
-            async for event in self._stream.output_item_image_gen_call(str(content.outputs)):
+            result = _image_generation_result(content.outputs)
+            if result is None:
+                logger.warning("Image generation result has no base64 image; emitting it with an empty result.")
+            async for event in self._stream.output_item_image_gen_call(result or ""):
                 yield event
 
         elif content.type == "mcp_server_tool_call":
@@ -3534,6 +3537,38 @@ def _json_safe_to_str(value: Any | None) -> str:
         return json.dumps(str(value))
 
 
+def _image_generation_result(outputs: Any) -> str | None:
+    """Return the base64 image of an image generation tool result, as the Responses API expects.
+
+    Only a non-empty, valid base64 string or the valid base64 payload of a base64 data URI qualifies. Other URIs
+    (https, file, blob), data URIs without the base64 marker and malformed payloads are not base64, so they give
+    None.
+    """
+    if isinstance(outputs, str):
+        return outputs if _is_base64(outputs) else None
+    items: Sequence[Any] = [outputs]
+    if isinstance(outputs, Sequence) and not isinstance(outputs, (bytes, bytearray)):
+        items = cast(Sequence[Any], outputs)
+    for item in items:
+        if isinstance(item, Content) and item.uri and item.uri.startswith("data:"):
+            # RFC 2397: only the metadata before the first comma can carry the marker, as its last parameter.
+            metadata, _, payload = item.uri.partition(",")
+            if metadata.endswith(";base64") and _is_base64(payload):
+                return payload
+    return None
+
+
+def _is_base64(value: str) -> bool:
+    """Check whether a string is non-empty, valid base64."""
+    if not value:
+        return False
+    try:
+        base64.b64decode(value, validate=True)
+    except ValueError:
+        return False
+    return True
+
+
 def _reasoning_encrypted_content(content: Content) -> str | None:
     """Return the opaque reasoning payload used for stateless replay."""
     encrypted_content = content.protected_data or content.additional_properties.get("encrypted_content")

python/packages/foundry_hosting/tests/test_responses.py

@@ -618,6 +618,82 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text()
     ]
 
 
+@pytest.mark.parametrize(
+    ("outputs", "expected"),
+    [
+        (Content.from_uri("data:image/png;base64,iVBORw0KGgo="), "iVBORw0KGgo="),
+        ([Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="),
+        ((Content.from_uri("data:image/png;base64,iVBORw0KGgo="),), "iVBORw0KGgo="),
+        ([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="),
+        (["caption", Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="),
+        ("iVBORw0KGgo=", "iVBORw0KGgo="),
+        (Content.from_uri("data:image/png;name=cat.png;base64,iVBORw0KGgo="), "iVBORw0KGgo="),
+        (
+            [
+                Content.from_uri("data:image/png;base64,not-base64!"),
+                Content.from_uri("data:image/png;base64,iVBORw0KGgo="),
+            ],
+            "iVBORw0KGgo=",
+        ),
+        (Content.from_uri("https://example.com/cat.png", media_type="image/png"), ""),
+        (Content.from_uri("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"), ""),
+        (Content.from_uri("data:image/png;base64,not-base64!"), ""),
+        (Content.from_uri("data:image/png;base64,iVBORw0KGgo"), ""),
+        (Content.from_uri("data:image/png;base64,iVBOR\nw0KGgo="), ""),
+        (Content.from_uri("data:image/png;base64,"), ""),
+        (Content.from_uri("data:text/plain,x;base64,iVBORw0KGgo="), ""),
+        (Content.from_hosted_file("file_1"), ""),
+        ("https://example.com/cat.png", ""),
+        ("not-base64!", ""),
+        ("", ""),
+        (b"\x89PNG\r\n\x1a\n", ""),
+        ({"result": "iVBORw0KGgo="}, ""),
+    ],
+    ids=[
+        "data-uri-content",
+        "content-list",
+        "content-tuple",
+        "image-after-text",
+        "image-after-string",
+        "base64-string",
+        "data-uri-with-parameter",
+        "image-after-malformed-payload",
+        "https-uri-is-not-base64",
+        "data-uri-without-base64",
+        "data-uri-malformed-payload",
+        "data-uri-missing-padding",
+        "data-uri-whitespace-in-payload",
+        "data-uri-empty-payload",
+        "base64-marker-inside-payload",
+        "hosted-file",
+        "url-string-is-not-base64",
+        "malformed-string",
+        "empty-string",
+        "bytes",
+        "mapping",
+    ],
+)
+async def test_output_item_tracker_emits_image_generation_result_as_base64(
+    outputs: Any, expected: str, caplog: pytest.LogCaptureFixture
+) -> None:
+    stream = ResponseEventStream(response_id="resp_image")
+    stream.emit_created()
+    stream.emit_in_progress()
+    tracker = _OutputItemTracker(stream)
+    events: list[Any] = []
+
+    with caplog.at_level(logging.WARNING):
+        async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)):
+            events.append(event)
+    events.extend(tracker.close())
+
+    done = [e for e in events if (e.get("type") if isinstance(e, Mapping) else e.type) == "response.output_item.done"]
+    assert len(done) == 1
+    assert done[0]["item"]["type"] == "image_generation_call"
+    assert done[0]["item"]["result"] == expected
+    assert ("has no base64 image" in caplog.text) is (expected == "")
+
+
 async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message() -> None:
     stream = ResponseEventStream(response_id="resp_mixed")
     stream.emit_created()