diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 1c57b9f72d8..80287b4e762 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -842,6 +842,38 @@ def extract_tools(agent: Any) -> list[dict[str, Any]]: "parameters": tool.parameters(), }) seen.add(tool.name) + for provider in getattr(agent, "context_providers", []): + provider_tools = getattr(provider, "tools", None) + if callable(provider_tools): + try: + provider_tools = provider_tools() + except Exception: + provider_tools = None + elif provider_tools is None and hasattr(provider, "_execute_code_tool"): + execute_code_tool = getattr(provider, "_execute_code_tool", None) + if hasattr(execute_code_tool, "create_run_tool"): + try: + provider_tools = [execute_code_tool.create_run_tool()] + except Exception: + provider_tools = None + elif ( + provider_tools is None + and hasattr(provider, "get_tools") + and callable(getattr(provider, "get_tools", None)) + ): + try: + provider_tools = provider.get_tools() + except Exception: + provider_tools = None + if provider_tools: + for tool in normalize_tools(provider_tools): + if isinstance(tool, FunctionTool) and tool.name not in seen: + tools.append({ + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters(), + }) + seen.add(tool.name) return tools @staticmethod @@ -883,6 +915,34 @@ def _to_eval_item( if isinstance(tool, FunctionTool) and tool.name not in seen: typed_tools.append(tool) seen.add(tool.name) + for provider in getattr(agent, "context_providers", []): + provider_tools = getattr(provider, "tools", None) + if callable(provider_tools): + try: + provider_tools = provider_tools() + except Exception: + provider_tools = None + elif provider_tools is None and hasattr(provider, "_execute_code_tool"): + execute_code_tool = getattr(provider, "_execute_code_tool", None) + if hasattr(execute_code_tool, "create_run_tool"): + try: + provider_tools = [execute_code_tool.create_run_tool()] + except Exception: + provider_tools = None + elif ( + provider_tools is None + and hasattr(provider, "get_tools") + and callable(getattr(provider, "get_tools", None)) + ): + try: + provider_tools = provider.get_tools() + except Exception: + provider_tools = None + if provider_tools: + for tool in normalize_tools(provider_tools): + if isinstance(tool, FunctionTool) and tool.name not in seen: + typed_tools.append(tool) + seen.add(tool.name) return EvalItem( conversation=all_msgs, diff --git a/python/packages/core/agent_framework/_skills.py b/python/packages/core/agent_framework/_skills.py index 3e14844b85a..ca095c9306b 100644 --- a/python/packages/core/agent_framework/_skills.py +++ b/python/packages/core/agent_framework/_skills.py @@ -2857,6 +2857,16 @@ async def _run_script( ), ] + @property + def tools(self) -> list[FunctionTool]: + """Tool definitions exposed by this provider for skill interaction. + + Returns: + The canonical skill tool definitions (``load_skill``, + ``read_skill_resource``, and ``run_skill_script``). + """ + return self._create_tools([]) + @staticmethod def _find_skill(skills: Sequence[Skill], name: str) -> Skill | None: """Find a skill by name (case-insensitive linear scan).""" diff --git a/python/packages/core/tests/core/test_evaluation.py b/python/packages/core/tests/core/test_evaluation.py index f32e216bd9b..c57ed4b8a7a 100644 --- a/python/packages/core/tests/core/test_evaluation.py +++ b/python/packages/core/tests/core/test_evaluation.py @@ -11,6 +11,7 @@ from agent_framework import AgentEvalConverter as ExportedAgentEvalConverter from agent_framework._evaluation import AgentEvalConverter, ConversationSplit, EvalItem, _to_eval_item +from agent_framework._skills import SkillsProvider from agent_framework._tools import FunctionTool from agent_framework._types import AgentResponse, Content, Message @@ -55,6 +56,71 @@ def test_to_eval_item_delegates_to_provider_neutral_builder(self) -> None: assert item.query == "Weather?" assert item.response == "Sunny." + def test_extract_tools_with_context_providers(self) -> None: + provider_tool = FunctionTool(name="provider_tool", description="From provider", func=lambda: "result") + provider = MagicMock() + provider.tools = [provider_tool] + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + tools = AgentEvalConverter.extract_tools(agent) + + assert len(tools) == 1 + assert tools[0]["name"] == "provider_tool" + assert tools[0]["description"] == "From provider" + + def test_extract_tools_with_codeact_provider(self) -> None: + sandbox_helper = FunctionTool(name="sandbox_helper", description="Sandbox helper", func=lambda: "helper") + codeact_tool = FunctionTool(name="execute_code", description="Run code", func=lambda: "ok") + provider = MagicMock() + provider.tools = [codeact_tool] + provider.get_tools = MagicMock(return_value=[sandbox_helper]) + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + tools = AgentEvalConverter.extract_tools(agent) + + tool_names = [t["name"] for t in tools] + assert tool_names == ["execute_code"] + assert "sandbox_helper" not in tool_names + + def test_extract_tools_with_get_tools_only_provider(self) -> None: + custom_tool = FunctionTool(name="custom_tool", description="From get_tools", func=lambda: "ok") + provider = MagicMock(spec=["get_tools"]) + provider.get_tools = MagicMock(return_value=[custom_tool]) + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + tools = AgentEvalConverter.extract_tools(agent) + + tool_names = [t["name"] for t in tools] + assert tool_names == ["custom_tool"] + + def test_extract_tools_with_secure_agent_config(self) -> None: + from agent_framework.security import SecureAgentConfig + + provider = SecureAgentConfig() + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + tools = AgentEvalConverter.extract_tools(agent) + + tool_names = [t["name"] for t in tools] + assert "quarantined_llm" in tool_names + assert "inspect_variable" in tool_names + class TestToEvalItem: def test_string_query(self) -> None: @@ -113,6 +179,140 @@ def test_with_agent_and_mcp_tools(self) -> None: assert item.tools == [agent_tool, mcp_tool] + def test_with_agent_context_providers(self) -> None: + provider_tool = FunctionTool(name="provider_tool", description="From provider", func=lambda: "result") + provider = MagicMock() + provider.tools = [provider_tool] + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Test provider", response=response, agent=agent) + + assert item.tools == [provider_tool] + + def test_with_agent_skills_provider(self) -> None: + provider = SkillsProvider([]) + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Test skills", response=response, agent=agent) + + assert item.tools is not None + tool_names = [tool.name for tool in item.tools] + assert "load_skill" in tool_names + assert "read_skill_resource" in tool_names + assert "run_skill_script" in tool_names + + def test_with_agent_context_providers_deduplication(self) -> None: + agent_tool = FunctionTool(name="common_tool", description="Agent version", func=lambda: "agent") + provider_tool = FunctionTool(name="common_tool", description="Provider version", func=lambda: "provider") + provider = MagicMock() + provider.tools = [provider_tool] + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Test dedup", response=response, agent=agent) + + assert item.tools == [agent_tool] + + def test_with_agent_codeact_provider(self) -> None: + sandbox_helper = FunctionTool(name="sandbox_helper", description="Sandbox helper", func=lambda: "helper") + codeact_tool = FunctionTool(name="execute_code", description="Run code", func=lambda: "ok") + provider = MagicMock() + provider.tools = [codeact_tool] + provider.get_tools = MagicMock(return_value=[sandbox_helper]) + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Run some code", response=response, agent=agent) + + assert item.tools is not None + tool_names = [tool.name for tool in item.tools] + assert tool_names == ["execute_code"] + assert "sandbox_helper" not in tool_names + + def test_with_agent_codeact_provider_fallback(self) -> None: + execute_code_tool = FunctionTool(name="execute_code", description="Run code", func=lambda: "ok") + mock_execute = MagicMock() + mock_execute.create_run_tool.return_value = execute_code_tool + provider = MagicMock(spec=["_execute_code_tool"]) + provider._execute_code_tool = mock_execute + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Run some code", response=response, agent=agent) + + assert item.tools is not None + tool_names = [tool.name for tool in item.tools] + assert tool_names == ["execute_code"] + + def test_with_real_monty_codeact_provider(self) -> None: + pytest.importorskip("agent_framework_monty") + from agent_framework_monty import MontyCodeActProvider + + sandbox_tool = FunctionTool(name="calculate_sum", description="Sum numbers", func=lambda a, b: a + b) + provider = MontyCodeActProvider(tools=[sandbox_tool]) + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Compute 1+1", response=response, agent=agent) + + assert item.tools is not None + tool_names = [tool.name for tool in item.tools] + assert tool_names == ["execute_code"] + assert "calculate_sum" not in tool_names + + def test_with_agent_get_tools_only_provider(self) -> None: + custom_tool = FunctionTool(name="custom_tool", description="From get_tools", func=lambda: "ok") + provider = MagicMock(spec=["get_tools"]) + provider.get_tools = MagicMock(return_value=[custom_tool]) + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Test get_tools", response=response, agent=agent) + + assert item.tools is not None + tool_names = [tool.name for tool in item.tools] + assert tool_names == ["custom_tool"] + + def test_with_agent_secure_agent_config(self) -> None: + from agent_framework.security import SecureAgentConfig + + provider = SecureAgentConfig() + agent = MagicMock() + agent.default_options = {} + agent.mcp_tools = [] + agent.context_providers = [provider] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Secure test", response=response, agent=agent) + + assert item.tools is not None + tool_names = [tool.name for tool in item.tools] + assert "quarantined_llm" in tool_names + assert "inspect_variable" in tool_names + def test_explicit_tools_override_agent(self) -> None: agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") diff --git a/python/packages/core/tests/core/test_skills.py b/python/packages/core/tests/core/test_skills.py index 944c7d74a18..452dbf42f91 100644 --- a/python/packages/core/tests/core/test_skills.py +++ b/python/packages/core/tests/core/test_skills.py @@ -6245,6 +6245,17 @@ async def test_all_tools_auto_approval_rule(self) -> None: unrelated = Content("function_call", call_id="c1", name="some_other_tool", arguments="{}") assert SkillsProvider.all_tools_auto_approval_rule(unrelated) is False + def test_tools_property_exposes_canonical_skill_tools(self) -> None: + """The provider exposes canonical skill tools via its tools property.""" + provider = SkillsProvider([]) + tools = provider.tools + names = [tool.name for tool in tools] + assert names == [ + SkillsProvider.LOAD_SKILL_TOOL_NAME, + SkillsProvider.READ_SKILL_RESOURCE_TOOL_NAME, + SkillsProvider.RUN_SKILL_SCRIPT_TOOL_NAME, + ] + async def test_code_script_exception_propagates_by_default(self) -> None: """A code script function that raises should propagate by default.""" diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py index 119e7fbcdc4..bd7e6095244 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py @@ -75,6 +75,11 @@ def add_tools( """Add provider-owned sandbox tools.""" self._execute_code_tool.add_tools(tools) + @property + def tools(self) -> list[FunctionTool]: + """Model-facing tool definitions contributed by this provider.""" + return [self._execute_code_tool.create_run_tool()] + def get_tools(self) -> list[FunctionTool]: """Return the provider-owned sandbox tools.""" return self._execute_code_tool.get_tools() diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index fcec3c876e9..523c098113f 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -3495,3 +3495,14 @@ async def _drive_many(tool: HyperlightExecuteCodeTool) -> None: gc.collect() assert drop_violations == [] + + +def test_hyperlight_codeact_provider_tools_property() -> None: + def add(a: int, b: int) -> int: + return a + b + + provider = HyperlightCodeActProvider(tools=[add]) + tools = provider.tools + assert len(tools) == 1 + assert tools[0].name == "execute_code" + assert isinstance(tools[0], HyperlightExecuteCodeTool) diff --git a/python/packages/monty/agent_framework_monty/_provider.py b/python/packages/monty/agent_framework_monty/_provider.py index 81192a038ee..ad046dafb78 100644 --- a/python/packages/monty/agent_framework_monty/_provider.py +++ b/python/packages/monty/agent_framework_monty/_provider.py @@ -63,6 +63,11 @@ def add_tools( """Add provider-owned Monty tools.""" self._execute_code_tool.add_tools(tools) + @property + def tools(self) -> list[FunctionTool]: + """Model-facing tool definitions contributed by this provider.""" + return [self._execute_code_tool.create_run_tool()] + def get_tools(self) -> list[FunctionTool]: """Return the provider-owned Monty tools.""" return self._execute_code_tool.get_tools() diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index d5f5db71cc2..ee19e4052a6 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -1625,3 +1625,14 @@ def limited() -> int: assert payload["exc_type"] == "ToolException" assert "maximum invocation limit" in payload["message"] assert limited.invocation_count == 1 + + +def test_provider_tools_property_exposes_execute_code() -> None: + def add(a: int, b: int) -> int: + return a + b + + provider = MontyCodeActProvider(tools=[add]) + tools = provider.tools + assert len(tools) == 1 + assert tools[0].name == "execute_code" + assert isinstance(tools[0], MontyExecuteCodeTool)