Python: fix(eval): include ContextProvider and SkillsProvider tools in evaluation definitions

python/packages/core/agent_framework/_evaluation.py

@@ -842,6 +842,38 @@ def extract_tools(agent: Any) -> list[dict[str, Any]]:
                         "parameters": tool.parameters(),
                     })
                     seen.add(tool.name)
+        for provider in getattr(agent, "context_providers", []):
+            provider_tools = getattr(provider, "tools", None)
+            if callable(provider_tools):
+                try:
+                    provider_tools = provider_tools()
+                except Exception:
+                    provider_tools = None
+            elif provider_tools is None and hasattr(provider, "_execute_code_tool"):
+                execute_code_tool = getattr(provider, "_execute_code_tool", None)
+                if hasattr(execute_code_tool, "create_run_tool"):
+                    try:
+                        provider_tools = [execute_code_tool.create_run_tool()]
+                    except Exception:
+                        provider_tools = None
+            elif (
+                provider_tools is None
+                and hasattr(provider, "get_tools")
+                and callable(getattr(provider, "get_tools", None))
+            ):
+                try:
+                    provider_tools = provider.get_tools()
+                except Exception:
+                    provider_tools = None
+            if provider_tools:
+                for tool in normalize_tools(provider_tools):
+                    if isinstance(tool, FunctionTool) and tool.name not in seen:
+                        tools.append({
+                            "name": tool.name,
+                            "description": tool.description,
+                            "parameters": tool.parameters(),
+                        })
+                        seen.add(tool.name)
         return tools
 
     @staticmethod
@@ -883,6 +915,34 @@ def _to_eval_item(
                 if isinstance(tool, FunctionTool) and tool.name not in seen:
                     typed_tools.append(tool)
                     seen.add(tool.name)
+        for provider in getattr(agent, "context_providers", []):
+            provider_tools = getattr(provider, "tools", None)
+            if callable(provider_tools):
+                try:
+                    provider_tools = provider_tools()
+                except Exception:
+                    provider_tools = None
+            elif provider_tools is None and hasattr(provider, "_execute_code_tool"):
+                execute_code_tool = getattr(provider, "_execute_code_tool", None)
+                if hasattr(execute_code_tool, "create_run_tool"):
+                    try:
+                        provider_tools = [execute_code_tool.create_run_tool()]
+                    except Exception:
+                        provider_tools = None
+            elif (
+                provider_tools is None
+                and hasattr(provider, "get_tools")
+                and callable(getattr(provider, "get_tools", None))
+            ):
+                try:
+                    provider_tools = provider.get_tools()
+                except Exception:
+                    provider_tools = None
+            if provider_tools:
+                for tool in normalize_tools(provider_tools):
+                    if isinstance(tool, FunctionTool) and tool.name not in seen:
+                        typed_tools.append(tool)
+                        seen.add(tool.name)
 
     return EvalItem(
         conversation=all_msgs,

python/packages/core/agent_framework/_skills.py

@@ -2857,6 +2857,16 @@ async def _run_script(
             ),
         ]
 
+    @property
+    def tools(self) -> list[FunctionTool]:
+        """Tool definitions exposed by this provider for skill interaction.
+
+        Returns:
+            The canonical skill tool definitions (``load_skill``,
+            ``read_skill_resource``, and ``run_skill_script``).
+        """
+        return self._create_tools([])
+
     @staticmethod
     def _find_skill(skills: Sequence[Skill], name: str) -> Skill | None:
         """Find a skill by name (case-insensitive linear scan)."""

python/packages/core/tests/core/test_evaluation.py

@@ -11,6 +11,7 @@
 
 from agent_framework import AgentEvalConverter as ExportedAgentEvalConverter
 from agent_framework._evaluation import AgentEvalConverter, ConversationSplit, EvalItem, _to_eval_item
+from agent_framework._skills import SkillsProvider
 from agent_framework._tools import FunctionTool
 from agent_framework._types import AgentResponse, Content, Message
 
@@ -55,6 +56,71 @@ def test_to_eval_item_delegates_to_provider_neutral_builder(self) -> None:
         assert item.query == "Weather?"
         assert item.response == "Sunny."
 
+    def test_extract_tools_with_context_providers(self) -> None:
+        provider_tool = FunctionTool(name="provider_tool", description="From provider", func=lambda: "result")
+        provider = MagicMock()
+        provider.tools = [provider_tool]
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+
+        with pytest.warns(DeprecationWarning, match="AgentEvalConverter"):
+            tools = AgentEvalConverter.extract_tools(agent)
+
+        assert len(tools) == 1
+        assert tools[0]["name"] == "provider_tool"
+        assert tools[0]["description"] == "From provider"
+
+    def test_extract_tools_with_codeact_provider(self) -> None:
+        sandbox_helper = FunctionTool(name="sandbox_helper", description="Sandbox helper", func=lambda: "helper")
+        codeact_tool = FunctionTool(name="execute_code", description="Run code", func=lambda: "ok")
+        provider = MagicMock()
+        provider.tools = [codeact_tool]
+        provider.get_tools = MagicMock(return_value=[sandbox_helper])
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+
+        with pytest.warns(DeprecationWarning, match="AgentEvalConverter"):
+            tools = AgentEvalConverter.extract_tools(agent)
+
+        tool_names = [t["name"] for t in tools]
+        assert tool_names == ["execute_code"]
+        assert "sandbox_helper" not in tool_names
+
+    def test_extract_tools_with_get_tools_only_provider(self) -> None:
+        custom_tool = FunctionTool(name="custom_tool", description="From get_tools", func=lambda: "ok")
+        provider = MagicMock(spec=["get_tools"])
+        provider.get_tools = MagicMock(return_value=[custom_tool])
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+
+        with pytest.warns(DeprecationWarning, match="AgentEvalConverter"):
+            tools = AgentEvalConverter.extract_tools(agent)
+
+        tool_names = [t["name"] for t in tools]
+        assert tool_names == ["custom_tool"]
+
+    def test_extract_tools_with_secure_agent_config(self) -> None:
+        from agent_framework.security import SecureAgentConfig
+
+        provider = SecureAgentConfig()
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+
+        with pytest.warns(DeprecationWarning, match="AgentEvalConverter"):
+            tools = AgentEvalConverter.extract_tools(agent)
+
+        tool_names = [t["name"] for t in tools]
+        assert "quarantined_llm" in tool_names
+        assert "inspect_variable" in tool_names
+
 
 class TestToEvalItem:
     def test_string_query(self) -> None:
@@ -113,6 +179,140 @@ def test_with_agent_and_mcp_tools(self) -> None:
 
         assert item.tools == [agent_tool, mcp_tool]
 
+    def test_with_agent_context_providers(self) -> None:
+        provider_tool = FunctionTool(name="provider_tool", description="From provider", func=lambda: "result")
+        provider = MagicMock()
+        provider.tools = [provider_tool]
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Test provider", response=response, agent=agent)
+
+        assert item.tools == [provider_tool]
+
+    def test_with_agent_skills_provider(self) -> None:
+        provider = SkillsProvider([])
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Test skills", response=response, agent=agent)
+
+        assert item.tools is not None
+        tool_names = [tool.name for tool in item.tools]
+        assert "load_skill" in tool_names
+        assert "read_skill_resource" in tool_names
+        assert "run_skill_script" in tool_names
+
+    def test_with_agent_context_providers_deduplication(self) -> None:
+        agent_tool = FunctionTool(name="common_tool", description="Agent version", func=lambda: "agent")
+        provider_tool = FunctionTool(name="common_tool", description="Provider version", func=lambda: "provider")
+        provider = MagicMock()
+        provider.tools = [provider_tool]
+        agent = MagicMock()
+        agent.default_options = {"tools": [agent_tool]}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Test dedup", response=response, agent=agent)
+
+        assert item.tools == [agent_tool]
+
+    def test_with_agent_codeact_provider(self) -> None:
+        sandbox_helper = FunctionTool(name="sandbox_helper", description="Sandbox helper", func=lambda: "helper")
+        codeact_tool = FunctionTool(name="execute_code", description="Run code", func=lambda: "ok")
+        provider = MagicMock()
+        provider.tools = [codeact_tool]
+        provider.get_tools = MagicMock(return_value=[sandbox_helper])
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Run some code", response=response, agent=agent)
+
+        assert item.tools is not None
+        tool_names = [tool.name for tool in item.tools]
+        assert tool_names == ["execute_code"]
+        assert "sandbox_helper" not in tool_names
+
+    def test_with_agent_codeact_provider_fallback(self) -> None:
+        execute_code_tool = FunctionTool(name="execute_code", description="Run code", func=lambda: "ok")
+        mock_execute = MagicMock()
+        mock_execute.create_run_tool.return_value = execute_code_tool
+        provider = MagicMock(spec=["_execute_code_tool"])
+        provider._execute_code_tool = mock_execute
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Run some code", response=response, agent=agent)
+
+        assert item.tools is not None
+        tool_names = [tool.name for tool in item.tools]
+        assert tool_names == ["execute_code"]
+
+    def test_with_real_monty_codeact_provider(self) -> None:
+        pytest.importorskip("agent_framework_monty")
+        from agent_framework_monty import MontyCodeActProvider
+
+        sandbox_tool = FunctionTool(name="calculate_sum", description="Sum numbers", func=lambda a, b: a + b)
+        provider = MontyCodeActProvider(tools=[sandbox_tool])
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Compute 1+1", response=response, agent=agent)
+
+        assert item.tools is not None
+        tool_names = [tool.name for tool in item.tools]
+        assert tool_names == ["execute_code"]
+        assert "calculate_sum" not in tool_names
+
+    def test_with_agent_get_tools_only_provider(self) -> None:
+        custom_tool = FunctionTool(name="custom_tool", description="From get_tools", func=lambda: "ok")
+        provider = MagicMock(spec=["get_tools"])
+        provider.get_tools = MagicMock(return_value=[custom_tool])
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Test get_tools", response=response, agent=agent)
+
+        assert item.tools is not None
+        tool_names = [tool.name for tool in item.tools]
+        assert tool_names == ["custom_tool"]
+
+    def test_with_agent_secure_agent_config(self) -> None:
+        from agent_framework.security import SecureAgentConfig
+
+        provider = SecureAgentConfig()
+        agent = MagicMock()
+        agent.default_options = {}
+        agent.mcp_tools = []
+        agent.context_providers = [provider]
+        response = AgentResponse(messages=[Message("assistant", ["Done"])])
+
+        item = _to_eval_item(query="Secure test", response=response, agent=agent)
+
+        assert item.tools is not None
+        tool_names = [tool.name for tool in item.tools]
+        assert "quarantined_llm" in tool_names
+        assert "inspect_variable" in tool_names
+
     def test_explicit_tools_override_agent(self) -> None:
         agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "")
         explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "")

python/packages/core/tests/core/test_skills.py

@@ -6245,6 +6245,17 @@ async def test_all_tools_auto_approval_rule(self) -> None:
         unrelated = Content("function_call", call_id="c1", name="some_other_tool", arguments="{}")
         assert SkillsProvider.all_tools_auto_approval_rule(unrelated) is False
 
+    def test_tools_property_exposes_canonical_skill_tools(self) -> None:
+        """The provider exposes canonical skill tools via its tools property."""
+        provider = SkillsProvider([])
+        tools = provider.tools
+        names = [tool.name for tool in tools]
+        assert names == [
+            SkillsProvider.LOAD_SKILL_TOOL_NAME,
+            SkillsProvider.READ_SKILL_RESOURCE_TOOL_NAME,
+            SkillsProvider.RUN_SKILL_SCRIPT_TOOL_NAME,
+        ]
+
     async def test_code_script_exception_propagates_by_default(self) -> None:
         """A code script function that raises should propagate by default."""
 

python/packages/hyperlight/agent_framework_hyperlight/_provider.py

@@ -75,6 +75,11 @@ def add_tools(
         """Add provider-owned sandbox tools."""
         self._execute_code_tool.add_tools(tools)
 
+    @property
+    def tools(self) -> list[FunctionTool]:
+        """Model-facing tool definitions contributed by this provider."""
+        return [self._execute_code_tool.create_run_tool()]
+
     def get_tools(self) -> list[FunctionTool]:
         """Return the provider-owned sandbox tools."""
         return self._execute_code_tool.get_tools()

python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py

@@ -3495,3 +3495,14 @@ async def _drive_many(tool: HyperlightExecuteCodeTool) -> None:
         gc.collect()
 
     assert drop_violations == []
+
+
+def test_hyperlight_codeact_provider_tools_property() -> None:
+    def add(a: int, b: int) -> int:
+        return a + b
+
+    provider = HyperlightCodeActProvider(tools=[add])
+    tools = provider.tools
+    assert len(tools) == 1
+    assert tools[0].name == "execute_code"
+    assert isinstance(tools[0], HyperlightExecuteCodeTool)

python/packages/monty/agent_framework_monty/_provider.py

@@ -63,6 +63,11 @@ def add_tools(
         """Add provider-owned Monty tools."""
         self._execute_code_tool.add_tools(tools)
 
+    @property
+    def tools(self) -> list[FunctionTool]:
+        """Model-facing tool definitions contributed by this provider."""
+        return [self._execute_code_tool.create_run_tool()]
+
     def get_tools(self) -> list[FunctionTool]:
         """Return the provider-owned Monty tools."""
         return self._execute_code_tool.get_tools()

python/packages/monty/tests/monty/test_monty_codeact.py

@@ -1625,3 +1625,14 @@ def limited() -> int:
             assert payload["exc_type"] == "ToolException"
             assert "maximum invocation limit" in payload["message"]
         assert limited.invocation_count == 1
+
+
+def test_provider_tools_property_exposes_execute_code() -> None:
+    def add(a: int, b: int) -> int:
+        return a + b
+
+    provider = MontyCodeActProvider(tools=[add])
+    tools = provider.tools
+    assert len(tools) == 1
+    assert tools[0].name == "execute_code"
+    assert isinstance(tools[0], MontyExecuteCodeTool)