diff --git a/docs/specs/004-python-function-calling-loop.md b/docs/specs/004-python-function-calling-loop.md index bdf2fc8ac23..c70feb4ba85 100644 --- a/docs/specs/004-python-function-calling-loop.md +++ b/docs/specs/004-python-function-calling-loop.md @@ -432,7 +432,8 @@ that manually replay messages own the equivalent rule: do not resend an approval or a shell call marked with a local environment, paired with a configured local executor, can enter the local function loop and its approval boundary. Stateless replay preserves provider shell call/output items in their original shape and fails explicitly when that provider representation is unavailable or inconsistent. Locally - generated shell outputs are sent in both service-side and stateless continuation modes. + generated shell outputs preserve stdout, stderr, truncation notices, non-zero exits, and timeout outcomes and are + sent in both service-side and stateless continuation modes. ### Reasoning-bound calls @@ -588,7 +589,7 @@ that manually replay messages own the equivalent rule: do not resend an approval | Parallel calls | Results retain the corresponding call ids and execution count. | `test_max_function_calls_limits_parallel_invocations`, `test_streaming_multiple_function_calls_parallel_execution` | | Sequential calls | Calls execute one at a time in model order, including after approval replay; mixed local/Host-owned siblings retain that order, approval-only batches wait for every decision before executing, each call has isolated context, and middleware termination takes effect after the accepted batch finishes. | `packages/core/tests/core/test_tools.py::test_sequential_function_invocation_runs_calls_in_model_order`, `test_sequential_function_invocation_isolates_context`, `test_sequential_function_invocation_finishes_batch_after_termination`, `packages/core/tests/core/test_harness_tool_approval.py::test_mixed_batch_hides_already_approved_request_until_approval_replay`, `test_sequential_approval_replay_preserves_model_order_when_responses_are_reversed`, `test_sequential_approval_replay_waits_for_the_complete_batch`, `packages/core/tests/core/test_function_invocation_logic.py::test_sequential_mixed_batch_preserves_approval_and_host_model_order`, `packages/core/tests/workflow/test_agent_executor_tool_calls.py::test_workflow_cancels_host_member_of_mixed_batch_with_terminal_result`, `test_workflow_host_cancellation_preserves_occurrence_with_reused_call_id` | | Informational-only call | The call is returned but not executed or approved. | `test_informational_only_function_call_is_not_invoked`, `test_informational_only_function_call_does_not_request_approval`, `test_streaming_informational_only_function_call_is_not_invoked` | -| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell items or shell calls marked with a local environment can execute, local execution preserves its configured approval mode, stateless loops preserve the complete provider shell transcript or fail explicitly, and locally generated shell outputs are sent in every continuation mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_parse_chunk_from_openai_local_environment_shell_call_done_emits_command`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_stateless_shell_transcript_without_provider_item_fails`, `test_prepare_messages_keeps_local_shell_output_under_storage`, `test_malformed_local_environment_shell_call_is_not_executable`, `test_response_content_creation_with_local_shell_call_maps_to_function_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | +| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell items or shell calls marked with a local environment can execute, local execution preserves its configured approval mode, stateless loops preserve the complete provider shell transcript or fail explicitly, and locally generated shell outputs preserve stdout, stderr, truncation notices, non-zero exits, and timeout outcomes in every continuation mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_parse_chunk_from_openai_local_environment_shell_call_done_emits_command`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_stateless_shell_transcript_without_provider_item_fails`, `test_prepare_messages_keeps_local_shell_output_under_storage`, `test_shell_output_payloads_use_structured_function_result_items`, `test_malformed_local_environment_shell_call_is_not_executable`, `test_response_content_creation_with_local_shell_call_maps_to_function_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | | Declaration-only call | The call is surfaced as user input and is not executed; streaming arguments appear once while finalized request metadata remains available. | `test_declaration_only_tool`, `test_streaming_declaration_only_tool_preserves_metadata_without_duplicate_arguments` | | Function invocation disabled | The client bypasses the invocation loop without losing invocation kwargs. | `test_function_invocation_config_enabled_false`, `test_function_invocation_config_enabled_false_preserves_invocation_kwargs`, `test_streaming_function_invocation_config_enabled_false` | | Runtime tool changes | Added tools become available on the next iteration and retain approval behavior. | `test_add_tools_available_next_iteration`, `test_add_tools_with_approval_required_tool` | diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index a8a1d037916..bc534fa1825 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -67,6 +67,7 @@ FunctionShellAction, FunctionShellCallOutputContent, FunctionShellCallOutputExitOutcome, + FunctionShellCallOutputTimeoutOutcome, Item, ItemReasoningItem, LocalEnvironmentResource, @@ -2349,14 +2350,19 @@ async def handle( if content.outputs: for out in content.outputs: exit_code = getattr(out, "exit_code", None) + outcome = ( + FunctionShellCallOutputTimeoutOutcome(type="timeout") + if getattr(out, "timed_out", False) + else FunctionShellCallOutputExitOutcome( + type="exit", + exit_code=exit_code if exit_code is not None else 0, + ) + ) output_items.append( FunctionShellCallOutputContent( stdout=getattr(out, "stdout", "") or "", stderr=getattr(out, "stderr", "") or "", - outcome=FunctionShellCallOutputExitOutcome( - type="exit", - exit_code=exit_code if exit_code is not None else 0, - ), + outcome=outcome, ) ) async for event in self._stream.output_item_function_shell_call_output( @@ -2770,6 +2776,17 @@ def _computer_screenshot_to_output(screenshot: Content) -> dict[str, Any]: return output +def _shell_command_output_to_content(output: Mapping[str, Any]) -> Content: + outcome = output["outcome"] + outcome_type = outcome.get("type") + return Content.from_shell_command_output( + stdout=output.get("stdout") or "", + stderr=output.get("stderr") or "", + exit_code=outcome.get("exit_code") if outcome_type == "exit" else None, + timed_out=True if outcome_type == "timeout" else False if outcome_type == "exit" else None, + ) + + async def _item_to_message( item: Item, *, @@ -2887,14 +2904,7 @@ async def _item_to_message( ) if item["type"] == "shell_call_output": - outputs = [ - Content.from_shell_command_output( - stdout=out["stdout"] or "", - stderr=out["stderr"] or "", - exit_code=out["outcome"].get("exit_code"), - ) - for out in (item["output"] or []) - ] + outputs = [_shell_command_output_to_content(out) for out in (item["output"] or [])] return Message( role="tool", contents=[ diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 95c8feb7dfe..6fb67cdc7a3 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -3898,6 +3898,51 @@ async def test_shell_call_preserves_execution_limits(self) -> None: "max_output_length": 4096, } + async def test_shell_result_preserves_outcomes(self) -> None: + agent = _make_agent( + response=AgentResponse( + messages=[ + Message( + role="tool", + contents=[ + Content.from_shell_tool_result( + call_id="shell_1", + outputs=[ + Content.from_shell_command_output( + stdout="ok", + stderr="", + exit_code=0, + timed_out=False, + ), + Content.from_shell_command_output( + stdout="", + stderr="command failed", + exit_code=3, + timed_out=False, + ), + Content.from_shell_command_output( + stdout="partial", + stderr="timeout details", + timed_out=True, + ), + ], + ) + ], + ) + ] + ) + ) + + resp = await _post(_make_server(agent), stream=False) + + assert resp.status_code == 200 + shell_item = next(item for item in resp.json()["output"] if item["type"] == "shell_call_output") + assert shell_item["output"] == [ + {"stdout": "ok", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}, + {"stdout": "", "stderr": "command failed", "outcome": {"type": "exit", "exit_code": 3}}, + {"stdout": "partial", "stderr": "timeout details", "outcome": {"type": "timeout"}}, + ] + async def test_hosted_mcp_call_and_result_persist_as_single_mcp_call(self) -> None: agent = _make_agent( response=AgentResponse( @@ -4865,8 +4910,8 @@ async def test_shell_call_output(self) -> None: output: FunctionShellCallOutputContent = { "stdout": "file.txt", - "stderr": "", - "outcome": cast(FunctionShellCallOutputExitOutcome, {"exit_code": 0}), + "stderr": "warning", + "outcome": cast(FunctionShellCallOutputExitOutcome, {"type": "exit", "exit_code": 3}), } item: OutputItemFunctionShellCallOutput = { "type": "shell_call_output", @@ -4880,6 +4925,40 @@ async def test_shell_call_output(self) -> None: assert msg.role == "tool" assert msg.contents[0].type == "shell_tool_result" assert msg.contents[0].call_id == "call_sc" + assert msg.contents[0].outputs is not None + assert msg.contents[0].outputs[0].stdout == "file.txt" + assert msg.contents[0].outputs[0].stderr == "warning" + assert msg.contents[0].outputs[0].exit_code == 3 + assert msg.contents[0].outputs[0].timed_out is False + + async def test_shell_call_output_preserves_timeout(self) -> None: + from azure.ai.agentserver.responses.models import ( + FunctionShellCallOutputContent, + FunctionShellCallOutputTimeoutOutcome, + OutputItemFunctionShellCallOutput, + ) + + output: FunctionShellCallOutputContent = { + "stdout": "partial", + "stderr": "timeout details", + "outcome": cast(FunctionShellCallOutputTimeoutOutcome, {"type": "timeout"}), + } + item: OutputItemFunctionShellCallOutput = { + "type": "shell_call_output", + "id": "sco-timeout", + "call_id": "call_timeout", + "status": "completed", + "output": [output], + "max_output_length": 1024, + } + + msg = await _output_item_to_message(item) + + assert msg.contents[0].outputs is not None + assert msg.contents[0].outputs[0].stdout == "partial" + assert msg.contents[0].outputs[0].stderr == "timeout details" + assert msg.contents[0].outputs[0].exit_code is None + assert msg.contents[0].outputs[0].timed_out is True async def test_local_shell_call(self) -> None: from azure.ai.agentserver.responses.models import LocalShellExecAction, OutputItemLocalShellToolCall @@ -5463,8 +5542,8 @@ async def test_shell_call_output(self) -> None: output: FunctionShellCallOutputContentParam = { "stdout": "file.txt", - "stderr": "", - "outcome": cast(FunctionShellCallOutputExitOutcomeParam, {"exit_code": 0}), + "stderr": "warning", + "outcome": cast(FunctionShellCallOutputExitOutcomeParam, {"type": "exit", "exit_code": 3}), } item: FunctionShellCallOutputItemParam = { "type": "shell_call_output", @@ -5477,6 +5556,38 @@ async def test_shell_call_output(self) -> None: assert msg.role == "tool" assert msg.contents[0].type == "shell_tool_result" assert msg.contents[0].call_id == "call_sc" + assert msg.contents[0].outputs is not None + assert msg.contents[0].outputs[0].stdout == "file.txt" + assert msg.contents[0].outputs[0].stderr == "warning" + assert msg.contents[0].outputs[0].exit_code == 3 + assert msg.contents[0].outputs[0].timed_out is False + + async def test_shell_call_output_preserves_timeout(self) -> None: + from azure.ai.agentserver.responses.models import ( + FunctionShellCallOutputContentParam, + FunctionShellCallOutputItemParam, + FunctionShellCallOutputTimeoutOutcomeParam, + ) + + output: FunctionShellCallOutputContentParam = { + "stdout": "partial", + "stderr": "timeout details", + "outcome": cast(FunctionShellCallOutputTimeoutOutcomeParam, {"type": "timeout"}), + } + item: FunctionShellCallOutputItemParam = { + "type": "shell_call_output", + "call_id": "call_timeout", + "output": [output], + "max_output_length": 1024, + } + + msg = await _item_to_message(item) + + assert msg.contents[0].outputs is not None + assert msg.contents[0].outputs[0].stdout == "partial" + assert msg.contents[0].outputs[0].stderr == "timeout details" + assert msg.contents[0].outputs[0].exit_code is None + assert msg.contents[0].outputs[0].timed_out is True async def test_local_shell_call(self) -> None: from azure.ai.agentserver.responses.models import ItemLocalShellToolCall, LocalShellExecAction diff --git a/python/packages/openai/agent_framework_openai/_chat_client.py b/python/packages/openai/agent_framework_openai/_chat_client.py index d7b7e0735cb..942b0cc57bd 100644 --- a/python/packages/openai/agent_framework_openai/_chat_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_client.py @@ -2373,57 +2373,109 @@ def _prepare_content_for_openai( logger.debug("Unsupported content type passed (type: %s)", content.type) return {} + @staticmethod + def _shell_output_payloads_from_items(content: Content) -> list[dict[str, Any]] | None: + """Extract structured shell output carried by canonical function-result items.""" + if not content.items: + return None + + if all(item.type == "shell_command_output" for item in content.items): + payloads: list[dict[str, Any]] = [] + for item in content.items: + payload: dict[str, Any] = { + "stdout": item.stdout or "", + "stderr": item.stderr or "", + "timed_out": item.timed_out is True, + } + if item.exit_code is not None: + payload["exit_code"] = item.exit_code + payloads.append(payload) + return payloads + + if len(content.items) != 1 or content.items[0].type != "text": + return None + + properties = content.items[0].additional_properties + stdout = properties.get("stdout") + stderr = properties.get("stderr") + exit_code = properties.get("exit_code") + truncated = properties.get("truncated", False) + timed_out = properties.get("timed_out") + if ( + isinstance(stdout, str) + and isinstance(stderr, str) + and type(exit_code) is int + and type(truncated) is bool + and type(timed_out) is bool + ): + if truncated: + stdout = f"{stdout}\n[output truncated]" if stdout else "[output truncated]" + return [ + { + "stdout": stdout, + "stderr": stderr, + "exit_code": exit_code, + "timed_out": timed_out, + } + ] + return None + @staticmethod def _to_local_shell_output_payload(content: Content) -> str: """Convert function tool output to the local shell JSON payload format.""" payload: dict[str, Any] if isinstance(content.result, Mapping): payload = dict(content.result) # type: ignore[assignment] + elif structured_payloads := RawOpenAIChatClient._shell_output_payloads_from_items(content): + payload = structured_payloads[0] if len(structured_payloads) == 1 else {"output": structured_payloads} else: payload = { "stdout": "" if content.result is None else str(content.result), } - if "exit_code" not in payload: + if "exit_code" not in payload and not bool(payload.get("timed_out", False)): payload["exit_code"] = 1 if content.exception is not None else 0 return json.dumps(payload, ensure_ascii=False) @staticmethod def _to_shell_call_output_payload(content: Content) -> list[dict[str, Any]]: """Convert function tool output to shell_call_output payload format.""" - payload: dict[str, Any] + payloads: list[dict[str, Any]] if isinstance(content.result, Mapping): - payload = dict(content.result) # type: ignore[assignment] + mapped_payload: dict[str, Any] = dict(content.result) # type: ignore[assignment] + payloads = [mapped_payload] + elif structured_payloads := RawOpenAIChatClient._shell_output_payloads_from_items(content): + payloads = structured_payloads else: - payload = { - "stdout": "" if content.result is None else str(content.result), - } + payloads = [{"stdout": "" if content.result is None else str(content.result)}] # Pass through native payload shape when tool already returns shell output entries. - direct_output = payload.get("output") - if isinstance(direct_output, list) and all(isinstance(item, Mapping) for item in direct_output): # type: ignore[reportUnknownMemberType] - return [dict(item) for item in direct_output] # type: ignore[reportUnknownMemberType] - - stdout = str(payload.get("stdout", "")) - stderr = str(payload.get("stderr", "")) - timed_out = bool(payload.get("timed_out", False)) - if timed_out: - outcome: dict[str, Any] = {"type": "timeout"} - else: - exit_code_raw = payload.get("exit_code") - try: - exit_code = ( - int(exit_code_raw) if exit_code_raw is not None else (1 if content.exception is not None else 0) - ) - except (TypeError, ValueError): - exit_code = 1 if content.exception is not None else 0 - outcome = {"type": "exit", "exit_code": exit_code} - return [ - { + if len(payloads) == 1: + direct_output = payloads[0].get("output") + if isinstance(direct_output, list) and all(isinstance(item, Mapping) for item in direct_output): # type: ignore[reportUnknownMemberType] + return [dict(item) for item in direct_output] # type: ignore[reportUnknownMemberType] + + output: list[dict[str, Any]] = [] + for payload in payloads: + stdout = str(payload.get("stdout", "")) + stderr = str(payload.get("stderr", "")) + timed_out = bool(payload.get("timed_out", False)) + if timed_out: + outcome: dict[str, Any] = {"type": "timeout"} + else: + exit_code_raw = payload.get("exit_code") + try: + exit_code = ( + int(exit_code_raw) if exit_code_raw is not None else (1 if content.exception is not None else 0) + ) + except (TypeError, ValueError): + exit_code = 1 if content.exception is not None else 0 + outcome = {"type": "exit", "exit_code": exit_code} + output.append({ "stdout": stdout, "stderr": stderr, "outcome": outcome, - } - ] + }) + return output def _image_generation_item_to_contents(self, item: Any) -> list[Content]: """Convert a completed ``image_generation_call`` output item into framework ``Content`` objects. diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index fcb31c961b3..dc00749b337 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -2262,6 +2262,177 @@ def test_shell_output_payloads_do_not_expose_exception_diagnostics() -> None: assert empty_shell_payload[0]["outcome"] == {"type": "exit", "exit_code": 1} +@pytest.mark.parametrize( + ("properties", "expected_local", "expected_shell"), + [ + pytest.param( + {"stdout": "ok", "stderr": "", "exit_code": 0, "truncated": False, "timed_out": False}, + {"stdout": "ok", "stderr": "", "exit_code": 0, "timed_out": False}, + [{"stdout": "ok", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}], + id="success", + ), + pytest.param( + { + "stdout": "", + "stderr": "command failed", + "exit_code": 3, + "truncated": False, + "timed_out": False, + }, + {"stdout": "", "stderr": "command failed", "exit_code": 3, "timed_out": False}, + [{"stdout": "", "stderr": "command failed", "outcome": {"type": "exit", "exit_code": 3}}], + id="nonzero-exit", + ), + pytest.param( + { + "stdout": "partial", + "stderr": "timeout details", + "exit_code": 124, + "truncated": False, + "timed_out": True, + }, + {"stdout": "partial", "stderr": "timeout details", "exit_code": 124, "timed_out": True}, + [{"stdout": "partial", "stderr": "timeout details", "outcome": {"type": "timeout"}}], + id="timeout", + ), + pytest.param( + { + "stdout": "partial output", + "stderr": "", + "exit_code": 0, + "truncated": True, + "timed_out": False, + }, + { + "stdout": "partial output\n[output truncated]", + "stderr": "", + "exit_code": 0, + "timed_out": False, + }, + [ + { + "stdout": "partial output\n[output truncated]", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + } + ], + id="truncated-stdout", + ), + pytest.param( + { + "stdout": "", + "stderr": "partial error", + "exit_code": 1, + "truncated": True, + "timed_out": False, + }, + { + "stdout": "[output truncated]", + "stderr": "partial error", + "exit_code": 1, + "timed_out": False, + }, + [ + { + "stdout": "[output truncated]", + "stderr": "partial error", + "outcome": {"type": "exit", "exit_code": 1}, + } + ], + id="truncated-empty-stdout", + ), + ], +) +def test_shell_output_payloads_use_structured_function_result_items( + properties: dict[str, Any], + expected_local: dict[str, Any], + expected_shell: list[dict[str, Any]], +) -> None: + display_text = "model-facing shell result" + content = Content.from_function_result( + call_id="call-structured", + result=[Content.from_text(display_text, additional_properties=properties)], + ) + round_tripped = Content.from_dict(content.to_dict()) + + assert round_tripped.result == display_text + assert round_tripped.items is not None + assert round_tripped.items[0].additional_properties == properties + assert json.loads(OpenAIChatClient._to_local_shell_output_payload(round_tripped)) == expected_local + assert OpenAIChatClient._to_shell_call_output_payload(round_tripped) == expected_shell + + +def test_shell_output_payloads_accept_native_shell_command_output() -> None: + content = Content.from_function_result( + call_id="call-native", + result=[ + Content.from_shell_command_output( + stdout="native output", + stderr="native error", + exit_code=9, + timed_out=False, + ) + ], + ) + + assert json.loads(OpenAIChatClient._to_local_shell_output_payload(content)) == { + "stdout": "native output", + "stderr": "native error", + "timed_out": False, + "exit_code": 9, + } + assert OpenAIChatClient._to_shell_call_output_payload(content) == [ + { + "stdout": "native output", + "stderr": "native error", + "outcome": {"type": "exit", "exit_code": 9}, + } + ] + + +def test_prepare_messages_uses_structured_local_shell_result() -> None: + from agent_framework_openai._chat_client import ( + OPENAI_SHELL_OUTPUT_TYPE_KEY, + OPENAI_SHELL_OUTPUT_TYPE_SHELL_CALL, + ) + + client = OpenAIChatClient(model="test-model", api_key="test-key") + shell_result = Content.from_function_result( + call_id="shell-call-1", + result=[ + Content.from_text( + "stderr: command failed\nexit_code: 3", + additional_properties={ + "stdout": "", + "stderr": "command failed", + "exit_code": 3, + "timed_out": False, + }, + ) + ], + additional_properties={OPENAI_SHELL_OUTPUT_TYPE_KEY: OPENAI_SHELL_OUTPUT_TYPE_SHELL_CALL}, + ) + + prepared = client._prepare_message_for_openai( + Message(role="tool", contents=[shell_result]), + request_uses_service_side_storage=False, + ) + + assert prepared == [ + { + "call_id": "shell-call-1", + "type": "shell_call_output", + "output": [ + { + "stdout": "", + "stderr": "command failed", + "outcome": {"type": "exit", "exit_code": 3}, + } + ], + } + ] + + def test_prepared_local_shell_tool_survives_make_tools() -> None: """Regression: the prepared shell tool must be a subscriptable dict. diff --git a/python/packages/tools/agent_framework_tools/shell/_docker.py b/python/packages/tools/agent_framework_tools/shell/_docker.py index 8a7646f77d2..316ba38c15c 100644 --- a/python/packages/tools/agent_framework_tools/shell/_docker.py +++ b/python/packages/tools/agent_framework_tools/shell/_docker.py @@ -60,7 +60,13 @@ from ._policy import ShellPolicy, ShellRequest from ._session import ShellSession from ._truncate import truncate_head_tail as _truncate_bytes -from ._types import ShellCommandError, ShellMode, ShellResult +from ._types import ( + ShellCommandError, + ShellMode, + ShellResult, + _parse_shell_result, # pyright: ignore[reportPrivateUsage] + _shell_result_to_text, # pyright: ignore[reportPrivateUsage] +) logger = logging.getLogger(__name__) @@ -802,7 +808,7 @@ async def _run_shell(command: str) -> str: result = await self.run(command) except ShellCommandError as exc: return str(exc) - return result.format_for_model() + return _shell_result_to_text(result) effective_description = description or _default_description(self._mode) _run_shell.__doc__ = effective_description @@ -812,4 +818,5 @@ async def _run_shell(command: str) -> str: description=effective_description, approval_mode=self._approval_mode, kind=SHELL_TOOL_KIND_VALUE, + result_parser=_parse_shell_result, ) diff --git a/python/packages/tools/agent_framework_tools/shell/_tool.py b/python/packages/tools/agent_framework_tools/shell/_tool.py index 86fb896739c..d85beeebd03 100644 --- a/python/packages/tools/agent_framework_tools/shell/_tool.py +++ b/python/packages/tools/agent_framework_tools/shell/_tool.py @@ -19,7 +19,13 @@ from ._policy import ShellPolicy, ShellRequest from ._resolve import is_powershell, resolve_shell from ._session import ShellSession -from ._types import ShellCommandError, ShellMode, ShellResult +from ._types import ( + ShellCommandError, + ShellMode, + ShellResult, + _parse_shell_result, # pyright: ignore[reportPrivateUsage] + _shell_result_to_text, # pyright: ignore[reportPrivateUsage] +) logger = logging.getLogger(__name__) @@ -311,7 +317,7 @@ async def _run_shell(command: str) -> str: result = await self.run(command) except ShellCommandError as exc: return str(exc) - return result.format_for_model() + return _shell_result_to_text(result) effective_description = description or _default_description(self._mode) _run_shell.__doc__ = effective_description @@ -321,6 +327,7 @@ async def _run_shell(command: str) -> str: description=effective_description, approval_mode=self._approval_mode, kind=SHELL_TOOL_KIND_VALUE, + result_parser=_parse_shell_result, ) # ------------------------------------------------------------------ helpers diff --git a/python/packages/tools/agent_framework_tools/shell/_types.py b/python/packages/tools/agent_framework_tools/shell/_types.py index bb4243c0804..4fbc7dfe5f4 100644 --- a/python/packages/tools/agent_framework_tools/shell/_types.py +++ b/python/packages/tools/agent_framework_tools/shell/_types.py @@ -5,7 +5,9 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Literal +from typing import Any, Literal + +from agent_framework import Content ShellMode = Literal["persistent", "stateless"] @@ -47,6 +49,40 @@ def format_for_model(self) -> str: return "\n".join(parts) +class _ShellResultText(str): + """Model-facing shell text that retains its structured source for normal parsing.""" + + shell_result: ShellResult + + def __new__(cls, result: ShellResult) -> _ShellResultText: + value = super().__new__(cls, result.format_for_model()) + value.shell_result = result + return value + + +def _shell_result_to_text(result: ShellResult) -> str: # pyright: ignore[reportUnusedFunction] + return _ShellResultText(result) + + +def _parse_shell_result(result: Any) -> str | list[Content]: # pyright: ignore[reportUnusedFunction] + """Preserve structured shell fields alongside the model-facing text.""" + if not isinstance(result, _ShellResultText): + return str(result) + shell_result = result.shell_result + return [ + Content.from_text( + result, + additional_properties={ + "stdout": shell_result.stdout, + "stderr": shell_result.stderr, + "exit_code": shell_result.exit_code, + "truncated": shell_result.truncated, + "timed_out": shell_result.timed_out, + }, + ) + ] + + class ShellExecutionError(RuntimeError): """Base class for shell-tool execution failures.""" diff --git a/python/packages/tools/tests/test_docker_shell_tool.py b/python/packages/tools/tests/test_docker_shell_tool.py index 00aed768f2a..b4f001877a2 100644 --- a/python/packages/tools/tests/test_docker_shell_tool.py +++ b/python/packages/tools/tests/test_docker_shell_tool.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio +import json import subprocess import sys from collections.abc import Sequence @@ -461,6 +462,35 @@ def test_as_function_carries_shell_kind(): ) +async def test_as_function_preserves_structured_shell_result() -> None: + shell_result = ShellResult( + stdout="container output", + stderr="container error", + exit_code=7, + duration_ms=25, + ) + tool = DockerShellTool(mode="stateless") + + with patch.object(tool, "run", AsyncMock(return_value=shell_result)): + function = tool.as_function() + result = await function.invoke(arguments={"command": "ignored"}) + raw_result = await function.invoke(arguments={"command": "ignored"}, skip_parsing=True) + + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == shell_result.format_for_model() + assert result[0].additional_properties == { + "stdout": "container output", + "stderr": "container error", + "exit_code": 7, + "truncated": False, + "timed_out": False, + } + assert isinstance(raw_result, str) + assert raw_result == shell_result.format_for_model() + assert json.loads(json.dumps(raw_result)) == shell_result.format_for_model() + + async def test_start_and_close_are_noops_in_stateless_mode() -> None: tool = DockerShellTool(mode="stateless") diff --git a/python/packages/tools/tests/test_local_shell_tool.py b/python/packages/tools/tests/test_local_shell_tool.py index 55df42983e2..f0e0f56362b 100644 --- a/python/packages/tools/tests/test_local_shell_tool.py +++ b/python/packages/tools/tests/test_local_shell_tool.py @@ -18,7 +18,7 @@ ) from agent_framework_tools._feature_usage import FeatureIndex -from agent_framework_tools.shell import LocalShellTool, ShellCommandError, ShellPolicy +from agent_framework_tools.shell import LocalShellTool, ShellCommandError, ShellPolicy, ShellResult from agent_framework_tools.shell._executor import _popen_kwargs_for_group, run_stateless _TEST_SHELL = "agent-framework-test-shell" @@ -396,6 +396,37 @@ async def test_as_function_wires_kind_and_approval() -> None: assert ft.approval_mode == "always_require" +async def test_as_function_preserves_structured_shell_result() -> None: + shell_result = ShellResult( + stdout="partial output", + stderr="command failed", + exit_code=3, + duration_ms=12, + truncated=True, + timed_out=True, + ) + tool = LocalShellTool(mode="stateless", approval_mode="never_require", acknowledge_unsafe=True) + + with patch.object(tool, "run", AsyncMock(return_value=shell_result)): + function = tool.as_function() + result = await function.invoke(arguments={"command": "ignored"}) + raw_result = await function.invoke(arguments={"command": "ignored"}, skip_parsing=True) + + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == shell_result.format_for_model() + assert result[0].additional_properties == { + "stdout": "partial output", + "stderr": "command failed", + "exit_code": 3, + "truncated": True, + "timed_out": True, + } + assert isinstance(raw_result, str) + assert raw_result == shell_result.format_for_model() + assert json.loads(json.dumps(raw_result)) == shell_result.format_for_model() + + async def test_variable_shell_approval_executes_only_the_reviewed_command() -> None: agent, session, request, variable_id, _ = await _request_variable_shell_approval("exact-shell-approval")