From 5514b85d0959679614abae531c68270afa43a084 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Thu, 29 Oct 2026 11:10:55 -0700 Subject: [PATCH 1/3] fix(AIC-3573): never send a NativeTool to the model as a function tool The tool wrapper turns every NativeTool into a callable tracking stub, so every handler but Claude Agents (and OpenCode in Python) sent it to the model as a function tool that returns nothing. Add an isFunctionTool / is_function_tool helper that rejects a NativeTool or its stub and warns once per tool key, and use it wherever a handler or native graph builds its tool list. No provider built-in mappings yet. Co-Authored-By: Claude Opus 5.5 --- .../handler.py | 11 ++- .../tests/test_native_tools.py | 54 +++++++++++++ .../api-report/launchdarkly_ai_server.txt | 1 + .../src/launchdarkly_ai_server/__init__.py | 2 + .../src/launchdarkly_ai_server/utils.py | 31 +++++++- packages/client/tests/test_public_surface.py | 1 + packages/client/tests/test_utils.py | 32 ++++++++ .../handler.py | 4 + .../native_graph.py | 3 + .../langchain-agents/tests/test_handler.py | 21 +++-- .../tests/test_native_tools.py | 68 ++++++++++++++++ .../handler.py | 11 ++- .../tests/test_native_tools.py | 54 +++++++++++++ .../launchdarkly_ai_litellm_agents/handler.py | 3 +- .../litellm-agents/tests/test_native_tools.py | 54 +++++++++++++ .../handler.py | 3 +- .../tests/test_native_tools.py | 54 +++++++++++++ .../launchdarkly_ai_openai_agents/handler.py | 5 ++ .../native_graph.py | 4 + .../openai-agents/tests/test_native_tools.py | 78 +++++++++++++++++++ .../handler.py | 11 ++- .../tests/test_native_tools.py | 54 +++++++++++++ .../launchdarkly_ai_vercel_agents/handler.py | 3 +- .../vercel-agents/tests/test_native_tools.py | 54 +++++++++++++ .../handler.py | 5 +- .../tests/test_native_tools.py | 65 ++++++++++++++++ 26 files changed, 662 insertions(+), 24 deletions(-) create mode 100644 packages/claude-messages/tests/test_native_tools.py create mode 100644 packages/langchain-agents/tests/test_native_tools.py create mode 100644 packages/langchain-messages/tests/test_native_tools.py create mode 100644 packages/litellm-agents/tests/test_native_tools.py create mode 100644 packages/litellm-messages/tests/test_native_tools.py create mode 100644 packages/openai-agents/tests/test_native_tools.py create mode 100644 packages/openai-messages/tests/test_native_tools.py create mode 100644 packages/vercel-agents/tests/test_native_tools.py create mode 100644 packages/vercel-messages/tests/test_native_tools.py diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 6adc035c..fd0190d9 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -19,6 +19,7 @@ end_span_once, end_unfinished_spans, is_content_blocks, + is_function_tool, parse_template, report_usage, set_input_content_attributes, @@ -230,7 +231,9 @@ def _forwarded_parameters(config: AiConfigRep) -> dict[str, Any]: ) -def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: +def _build_tools( + config_tools: dict[str, Any], tool_handlers: Mapping[str, Any] +) -> list[dict[str, Any]]: return [ { "name": name, @@ -238,6 +241,8 @@ def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: "input_schema": tool.get("parameters", {}), } for name, tool in config_tools.items() + # A missing handler still reaches the model; a NativeTool never does. + if name not in tool_handlers or is_function_tool(tool_handlers[name], name) ] @@ -380,7 +385,7 @@ async def _run_tool_loop( # Not filtered to the tools that have a registered handler, unlike the TypeScript SDK. That # difference predates this span work and changes what the model is offered, not what the span # reports, so it stays as it is: the catalog recorded below is the catalog actually sent. - tools = _build_tools(config.get("tools") or {}) + tools = _build_tools(config.get("tools") or {}, tool_handlers or {}) extra_params = _forwarded_parameters(config) max_tokens = extra_params.pop("max_tokens", 1024) conversation = list(messages) @@ -720,7 +725,7 @@ async def _stream_gen( config, user_input, variables, include_output_format=False, history=history ) - tools = _build_tools(config.get("tools") or {}) + tools = _build_tools(config.get("tools") or {}, tool_handlers or {}) tool_definitions = to_tool_definitions(tools) extra_params = _forwarded_parameters(config) max_tokens = extra_params.pop("max_tokens", 1024) diff --git a/packages/claude-messages/tests/test_native_tools.py b/packages/claude-messages/tests/test_native_tools.py new file mode 100644 index 00000000..0647815c --- /dev/null +++ b/packages/claude-messages/tests/test_native_tools.py @@ -0,0 +1,54 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-claude-messages" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_claude_messages.handler import _build_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _build_tools(c, h), + lambda r: [t["name"] for t in r], + caplog, + ) diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 607611f2..9cb35d96 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -148,6 +148,7 @@ InputTokenDetails [class, @dataclass] uncached: int = 0 inspect_config(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] is_content_blocks(content: MessageContent) -> bool +is_function_tool(handler: Any, name: str) -> TypeGuard[Callable[..., Any]] Judge [class, @dataclass(frozen=True)] __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = None, ground_truth_context: str | None = None) __post_init__() -> None diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 48355f8c..6e6f2408 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -149,6 +149,7 @@ create_run_usage, end_span_once, end_unfinished_spans, + is_function_tool, lang_chain_span_usage, make_track_data, model_stamps_from_meta, @@ -215,6 +216,7 @@ "create_run_usage", "end_span_once", "end_unfinished_spans", + "is_function_tool", "lang_chain_content_text", "lang_chain_finish_reasons", "lang_chain_span_messages", diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index e1c283a8..472edb34 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -1,16 +1,19 @@ from __future__ import annotations import json +import logging import re -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from typing import Any, Literal +from typing import Any, Literal, TypeGuard from .sdk_usage import report_usage from .types import ( + NATIVE_TOOL_KEY, AiConfigRep, GraphNode, InputTokenDetails, + NativeTool, ProviderHandler, UsageDict, VariationMeta, @@ -18,6 +21,8 @@ _StreamFn, ) +logger = logging.getLogger(__name__) + def create_handler( provides_for: tuple[str, Literal["agent", "messages"]], @@ -896,3 +901,25 @@ def parse_json_with_possible_fences(raw_text: str) -> Any | None: pass return None + + +_warned_native_tools: set[str] = set() + + +def is_function_tool(handler: Any, name: str) -> TypeGuard[Callable[..., Any]]: + """ + Whether a handler may send ``handler`` to the model as a function tool. A ``NativeTool``, + or the tracking stub ``wrap_tool_handlers`` makes from one, is a provider built-in, never a + function tool: a handler with no built-in for it skips it, warning once per tool key. + """ + if isinstance(handler, NativeTool) or isinstance( + getattr(handler, NATIVE_TOOL_KEY, None), NativeTool + ): + if name not in _warned_native_tools: + _warned_native_tools.add(name) + logger.warning( + 'Skipping native tool "%s": this handler has no built-in tool for it.', + name, + ) + return False + return callable(handler) diff --git a/packages/client/tests/test_public_surface.py b/packages/client/tests/test_public_surface.py index 29177a3a..7ebcb5bb 100644 --- a/packages/client/tests/test_public_surface.py +++ b/packages/client/tests/test_public_surface.py @@ -82,6 +82,7 @@ "init_client", "inspect_config", "is_content_blocks", + "is_function_tool", "lang_chain_content_text", "lang_chain_finish_reasons", "lang_chain_span_messages", diff --git a/packages/client/tests/test_utils.py b/packages/client/tests/test_utils.py index dd6035e8..007fcaee 100644 --- a/packages/client/tests/test_utils.py +++ b/packages/client/tests/test_utils.py @@ -434,3 +434,35 @@ def test_never_reads_custom(self) -> None: result = model_parameters(config) assert "secret" not in result assert "custom" not in result + + +# --------------------------------------------------------------------------- +# is_function_tool +# --------------------------------------------------------------------------- + + +def test_is_function_tool_accepts_a_callable_and_rejects_a_missing_handler() -> None: + from launchdarkly_ai_server import is_function_tool + + assert is_function_tool(lambda: 1, "fn") is True + assert is_function_tool(None, "missing") is False + + +def test_is_function_tool_rejects_a_native_tool_and_its_stub_warning_once_per_key( + caplog: pytest.LogCaptureFixture, +) -> None: + from launchdarkly_ai_server import NativeTool, is_function_tool, wrap_tool_handlers + + native = NativeTool("WebSearch") + stub = wrap_tool_handlers({"utils-web": native}, {"kind": "user", "key": "u"}, {})[ + "utils-web" + ] + assert callable(stub) + with caplog.at_level("WARNING"): + assert is_function_tool(stub, "utils-web") is False + assert is_function_tool(native, "utils-web") is False + assert is_function_tool(native, "utils-other") is False + assert [r.getMessage() for r in caplog.records] == [ + 'Skipping native tool "utils-web": this handler has no built-in tool for it.', + 'Skipping native tool "utils-other": this handler has no built-in tool for it.', + ] diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 0abd493d..24acb6be 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -24,6 +24,7 @@ create_run_usage, end_span_once, end_unfinished_spans, + is_function_tool, lang_chain_content_text, lang_chain_span_messages, lang_chain_span_usage, @@ -400,6 +401,9 @@ def _build_agent_tools( result = [] for name, tool_cfg in config_tools.items(): + # A missing handler still reaches the model; a NativeTool never does. + if name in tool_handlers and not is_function_tool(tool_handlers[name], name): + continue schema = tool_cfg.get("parameters") or {} async def _handler(_name: str = name, **kwargs: Any) -> str: diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py index cdae0eb8..f6d7a46e 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py @@ -24,6 +24,7 @@ NativeTool, compose_history, get_client, + is_function_tool, make_track_data, parse_template, report_usage, @@ -163,6 +164,8 @@ def _build_node_tools( result = [] for name, tool_cfg in node.config["tools"].items(): + if name in tool_handlers and not is_function_tool(tool_handlers[name], name): + continue schema = tool_cfg.get("parameters") or {} async def _handler(_name: str = name, **kwargs: Any) -> str: diff --git a/packages/langchain-agents/tests/test_handler.py b/packages/langchain-agents/tests/test_handler.py index a413ba1a..efa79522 100644 --- a/packages/langchain-agents/tests/test_handler.py +++ b/packages/langchain-agents/tests/test_handler.py @@ -431,16 +431,21 @@ def _capture_tool(name: Any, fn: Any = None, **kw: Any) -> Any: await captured_fns[0](key="val") @pytest.mark.asyncio - async def test_native_tool_is_not_called(self) -> None: - # Offline evaluations pass a NativeTool through unwrapped; it is not callable. + async def test_native_tool_is_not_registered(self) -> None: + # Offline evaluations pass a NativeTool through unwrapped; it is never a function tool. from launchdarkly_ai_server import NativeTool - run = _built_tool( - {"web-search": {"description": "d", "parameters": {}}}, - {"web-search": NativeTool("WebSearch")}, - ) - with pytest.raises(ValueError, match="No handler"): - await run(query="q") + mocks = _make_langchain_mock() + mocks["langchain_core.tools"].tool = MagicMock() + with _patch_lc(mocks): + from launchdarkly_ai_langchain_agents.handler import _build_agent_tools + + tools = _build_agent_tools( + {"web-search": {"description": "d", "parameters": {}}}, + {"web-search": NativeTool("WebSearch")}, + ) + assert tools == [] + mocks["langchain_core.tools"].tool.assert_not_called() @pytest.mark.asyncio async def test_tool_handler_throws_propagates(self) -> None: diff --git a/packages/langchain-agents/tests/test_native_tools.py b/packages/langchain-agents/tests/test_native_tools.py new file mode 100644 index 00000000..fa1b7ce2 --- /dev/null +++ b/packages/langchain-agents/tests/test_native_tools.py @@ -0,0 +1,68 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from types import SimpleNamespace +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-langchain-agents" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_langchain_agents.handler import _build_agent_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _build_agent_tools(c, h), + lambda r: [t.name for t in r], + caplog, + ) + + +def test_native_graph_never_sends_a_native_tool( + caplog: pytest.LogCaptureFixture, +) -> None: + from launchdarkly_ai_langchain_agents.native_graph import _build_node_tools + + _assert_skipped( + PREFIX + "native_graph", + lambda c, h: _build_node_tools(SimpleNamespace(config={"tools": c}), h), + lambda r: [t.name for t in r], + caplog, + ) diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index cf6eb6a8..c7919b9a 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -22,6 +22,7 @@ end_unfinished_spans, image_block_to_url, is_content_blocks, + is_function_tool, lang_chain_content_text, lang_chain_finish_reasons, lang_chain_span_messages, @@ -391,7 +392,9 @@ def _rebuild_anthropic_output_config(kwargs: dict[str, Any]) -> dict[str, Any]: } -def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: +def _build_tools( + config_tools: dict[str, Any], tool_handlers: Mapping[str, Any] +) -> list[dict[str, Any]]: # Not filtered to the tools that have a registered handler, unlike the TypeScript SDK's # `buildTools`. That difference predates this span work and changes what the model is offered, # not what the span reports, so it stays as it is: the catalog recorded below (via @@ -406,6 +409,8 @@ def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: }, } for name, tool in config_tools.items() + # A missing handler still reaches the model; a NativeTool never does. + if name not in tool_handlers or is_function_tool(tool_handlers[name], name) ] @@ -835,7 +840,7 @@ async def _call_impl( ) base_model = await _resolve_base_model(config, llm, importlib) - tool_defs = _build_tools(config.get("tools") or {}) + tool_defs = _build_tools(config.get("tools") or {}, tool_handlers or {}) output_format = config.get("outputFormat") # CASE 1: outputFormat only, no tools -> withStructuredOutput (all providers). @@ -1152,7 +1157,7 @@ async def _stream_gen( initial_messages = _build_messages(config, user_input, variables, history) - tool_defs = _build_tools(config.get("tools") or {}) + tool_defs = _build_tools(config.get("tools") or {}, tool_handlers or {}) tool_model = base_model.bind_tools(tool_defs) if tool_defs else base_model tool_definitions = to_tool_definitions(tool_defs) conversation_messages = list(initial_messages) diff --git a/packages/langchain-messages/tests/test_native_tools.py b/packages/langchain-messages/tests/test_native_tools.py new file mode 100644 index 00000000..992b4b16 --- /dev/null +++ b/packages/langchain-messages/tests/test_native_tools.py @@ -0,0 +1,54 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-langchain-messages" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_langchain_messages.handler import _build_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _build_tools(c, h), + lambda r: [t["function"]["name"] for t in r], + caplog, + ) diff --git a/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py b/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py index 99de5d9c..10a5dfc0 100644 --- a/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py +++ b/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py @@ -26,6 +26,7 @@ content_to_text, end_span_once, image_block_to_url, + is_function_tool, parse_template, report_usage, set_input_content_attributes, @@ -273,7 +274,7 @@ def _tools(config_value: AiConfigRep, handlers: Mapping[str, Any]) -> list[Any]: tools: list[Any] = [] for name, definition in (config_value.get("tools") or {}).items(): handler = handlers.get(name) - if not callable(handler): + if not is_function_tool(handler, name): continue async def invoke(_context: Any, args: str, _handler: Any = handler) -> Any: diff --git a/packages/litellm-agents/tests/test_native_tools.py b/packages/litellm-agents/tests/test_native_tools.py new file mode 100644 index 00000000..6e58830b --- /dev/null +++ b/packages/litellm-agents/tests/test_native_tools.py @@ -0,0 +1,54 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-litellm-agents" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_litellm_agents.handler import _tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _tools({"tools": c}, h), + lambda r: [t.name for t in r], + caplog, + ) diff --git a/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py b/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py index 7bdcc58e..fd0f71c6 100644 --- a/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py +++ b/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py @@ -20,6 +20,7 @@ content_to_text, end_span_once, image_block_to_url, + is_function_tool, parse_template, report_usage, set_input_content_attributes, @@ -219,7 +220,7 @@ def _build_tools( }, } for name, tool in (config_value.get("tools") or {}).items() - if callable(tool_handlers.get(name)) + if is_function_tool(tool_handlers.get(name), name) ] diff --git a/packages/litellm-messages/tests/test_native_tools.py b/packages/litellm-messages/tests/test_native_tools.py new file mode 100644 index 00000000..6dd7b2d3 --- /dev/null +++ b/packages/litellm-messages/tests/test_native_tools.py @@ -0,0 +1,54 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-litellm-messages" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_litellm_messages.handler import _build_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _build_tools({"tools": c}, h), + lambda r: [t["function"]["name"] for t in r], + caplog, + ) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index a57e071b..cd01c0bd 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -39,6 +39,7 @@ end_span_once, end_unfinished_spans, image_block_to_url, + is_function_tool, parse_template, report_usage, set_input_content_attributes, @@ -220,6 +221,10 @@ def _build_agent_tools( result = [] for name, tool_cfg in config_tools.items(): + # A missing handler still reaches the model; a NativeTool never does. + if name in tool_handlers and not is_function_tool(tool_handlers[name], name): + continue + # on_invoke_tool receives (ToolContext, json_args_str); decode and dispatch. async def _execute(_ctx: Any, args_str: str, _name: str = name) -> str: handler = tool_handlers.get(_name) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py index 92af5ccb..9a716869 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py @@ -22,6 +22,7 @@ NativeTool, compose_history, get_client, + is_function_tool, make_track_data, parse_template, report_usage, @@ -79,6 +80,9 @@ def _build_node_tools( result = [] for name, tool_cfg in node.config["tools"].items(): + if name in tool_handlers and not is_function_tool(tool_handlers[name], name): + continue + # on_invoke_tool receives (ToolContext, json_args_str); decode and dispatch. async def _execute(_ctx: Any, args_str: str, _name: str = name) -> str: handler = tool_handlers.get(_name) diff --git a/packages/openai-agents/tests/test_native_tools.py b/packages/openai-agents/tests/test_native_tools.py new file mode 100644 index 00000000..425ededb --- /dev/null +++ b/packages/openai-agents/tests/test_native_tools.py @@ -0,0 +1,78 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from types import SimpleNamespace +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-openai-agents" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_openai_agents.handler import _build_agent_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _build_agent_tools(c, h), + lambda r: [t.name for t in r], + caplog, + ) + + +def test_native_graph_never_sends_a_native_tool( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + import agents + + from launchdarkly_ai_openai_agents.native_graph import _build_node_tools + + # The adapter calls ``agents.tool(...)`` as a decorator factory; stub it as the other tests do. + monkeypatch.setattr( + agents, + "tool", + lambda **kw: lambda fn: SimpleNamespace(name=kw["name"]), + raising=False, + ) + + _assert_skipped( + PREFIX + "native_graph", + lambda c, h: _build_node_tools(SimpleNamespace(config={"tools": c}), h), + lambda r: [t.name for t in r], + caplog, + ) diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index 3e616c17..d2d3d21b 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -22,6 +22,7 @@ end_unfinished_spans, image_block_to_url, is_content_blocks, + is_function_tool, parse_template, report_usage, set_input_content_attributes, @@ -215,7 +216,9 @@ def _forwarded_parameters(config: AiConfigRep) -> dict[str, Any]: ) -def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: +def _build_tools( + config_tools: dict[str, Any], tool_handlers: Mapping[str, Any] +) -> list[dict[str, Any]]: # Not filtered to the tools that have a registered handler, unlike the TypeScript SDK. That # difference predates this span work and changes what the model is offered, not what the span # reports, so it stays as it is: the catalog recorded on the span is the catalog actually sent. @@ -228,6 +231,8 @@ def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: "strict": False, } for name, tool in config_tools.items() + # A missing handler still reaches the model; a NativeTool never does. + if name not in tool_handlers or is_function_tool(tool_handlers[name], name) ] @@ -432,7 +437,7 @@ async def _call_impl( run_usage = create_run_usage() try: - tools = _build_tools(config.get("tools") or {}) + tools = _build_tools(config.get("tools") or {}, tool_handlers or {}) input_messages = _build_input_messages(config, user_input, vs, history) tool_definitions = to_tool_definitions(tools) @@ -646,7 +651,7 @@ async def _stream_gen( cancelled = False emitter = TurnEmitter(capture_content=capture_content) try: - tools = _build_tools(config.get("tools") or {}) + tools = _build_tools(config.get("tools") or {}, tool_handlers or {}) input_messages = _build_input_messages(config, user_input, variables, history) tool_definitions = to_tool_definitions(tools) diff --git a/packages/openai-messages/tests/test_native_tools.py b/packages/openai-messages/tests/test_native_tools.py new file mode 100644 index 00000000..940062e4 --- /dev/null +++ b/packages/openai-messages/tests/test_native_tools.py @@ -0,0 +1,54 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-openai-messages" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_openai_messages.handler import _build_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _build_tools(c, h), + lambda r: [t["name"] for t in r], + caplog, + ) diff --git a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py index 03bc1f48..82f7b10d 100644 --- a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py +++ b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py @@ -21,6 +21,7 @@ _config, _create_handler, compose_history, + is_function_tool, parse_template, report_usage, ) @@ -293,7 +294,7 @@ def build_agent_tools( result: list[Any] = [] for name, definition in (definitions or {}).items(): handler = (handlers or {}).get(name) - if not callable(handler) or isinstance(handler, NativeTool): + if not is_function_tool(handler, name): continue async def execute( diff --git a/packages/vercel-agents/tests/test_native_tools.py b/packages/vercel-agents/tests/test_native_tools.py new file mode 100644 index 00000000..5c70785a --- /dev/null +++ b/packages/vercel-agents/tests/test_native_tools.py @@ -0,0 +1,54 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-vercel-agents" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_vercel_agents.handler import build_agent_tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: build_agent_tools(c, h), + lambda r: [t.name for t in r], + caplog, + ) diff --git a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py index f18a0ffd..0712dcd6 100644 --- a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py +++ b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py @@ -21,6 +21,7 @@ _config, _create_handler, compose_history, + is_function_tool, parse_template, report_usage, ) @@ -274,7 +275,7 @@ def _tool_executors( executors: dict[str, Callable[..., Any]] = {} for name in cfg.get("tools") or {}: handler = (handlers or {}).get(name) - if callable(handler) and not isinstance(handler, NativeTool): + if is_function_tool(handler, name): executors[name] = handler return executors @@ -338,7 +339,7 @@ def _tools(cfg: AiConfigRep, handlers: Mapping[str, Any] | None) -> list[Any]: result: list[Any] = [] for name, definition in (cfg.get("tools") or {}).items(): handler = (handlers or {}).get(name) - if not callable(handler) or isinstance(handler, NativeTool): + if not is_function_tool(handler, name): continue async def execute(_handler: Callable[..., Any] = handler, **kwargs: Any) -> Any: diff --git a/packages/vercel-messages/tests/test_native_tools.py b/packages/vercel-messages/tests/test_native_tools.py new file mode 100644 index 00000000..96d476ac --- /dev/null +++ b/packages/vercel-messages/tests/test_native_tools.py @@ -0,0 +1,65 @@ +"""A NativeTool is never sent to the model as a function tool.""" + +from __future__ import annotations + +import logging +from typing import Any + +import pytest + +from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool + +# The warn-once set is process-wide, so each test uses its own key. +PREFIX = "native-web-vercel-messages" + + +def _config_tools(key: str) -> dict[str, Any]: + return { + "search": {"description": "d", "parameters": {"type": "object"}}, + key: {"description": "d", "parameters": {"type": "object"}}, + } + + +def _stub(native: NativeTool) -> Any: + """The tracking stub ``wrap_tool_handlers`` makes from a NativeTool.""" + + def stub() -> None: + return None + + setattr(stub, NATIVE_TOOL_KEY, native) + return stub + + +def _assert_skipped( + key: str, build: Any, names: Any, caplog: pytest.LogCaptureFixture +) -> None: + native = NativeTool("web_search") + tools = _config_tools(key) + with caplog.at_level(logging.WARNING): + first = build(tools, {"search": lambda *a, **k: "x", key: _stub(native)}) + second = build(tools, {"search": lambda *a, **k: "x", key: native}) + assert names(first) == ["search"] + assert names(second) == ["search"] + assert sum(f'"{key}"' in r.getMessage() for r in caplog.records) == 1 + + +def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_vercel_messages.handler import _tools + + _assert_skipped( + PREFIX + "handler", + lambda c, h: _tools({"tools": c}, h), + lambda r: [t.name for t in r], + caplog, + ) + + +def test_executors_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> None: + from launchdarkly_ai_vercel_messages.handler import _tool_executors + + _assert_skipped( + PREFIX + "executors", + lambda c, h: _tool_executors({"tools": c}, h), + lambda r: list(r), + caplog, + ) From ca85f9df3fca7ff7cb1fd6055ab165e3898ce957 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Thu, 29 Oct 2026 11:19:05 -0700 Subject: [PATCH 2/3] fix(AIC-3573): correct a stale NativeTool comment Co-Authored-By: Claude Opus 5.5 --- .../src/launchdarkly_ai_langchain_agents/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 24acb6be..793d3d20 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -408,7 +408,7 @@ def _build_agent_tools( async def _handler(_name: str = name, **kwargs: Any) -> str: fn = tool_handlers.get(_name) - # A NativeTool reaches here unwrapped on the offline evaluations path. + # The builder skips a NativeTool, so this guards only a handler that is not callable. if not callable(fn): raise ValueError(f'No handler registered for tool "{_name}"') # Handlers may be sync or async. Graph ``__handoff_*`` tools stay sync so From 21d61509f299b052fd61c0cc39e015d606bf181d Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Fri, 30 Oct 2026 08:06:38 -0700 Subject: [PATCH 3/3] test(AIC-3573): build native graph tools with the real FunctionTool The native graph now builds node tools with agents.FunctionTool (#251), so the agents.tool stub in this test no longer covers anything. Co-Authored-By: Claude Opus 5.5 --- packages/openai-agents/tests/test_native_tools.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/openai-agents/tests/test_native_tools.py b/packages/openai-agents/tests/test_native_tools.py index 425ededb..5477df3a 100644 --- a/packages/openai-agents/tests/test_native_tools.py +++ b/packages/openai-agents/tests/test_native_tools.py @@ -56,20 +56,10 @@ def test_handler_never_sends_a_native_tool(caplog: pytest.LogCaptureFixture) -> def test_native_graph_never_sends_a_native_tool( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch + caplog: pytest.LogCaptureFixture, ) -> None: - import agents - from launchdarkly_ai_openai_agents.native_graph import _build_node_tools - # The adapter calls ``agents.tool(...)`` as a decorator factory; stub it as the other tests do. - monkeypatch.setattr( - agents, - "tool", - lambda **kw: lambda fn: SimpleNamespace(name=kw["name"]), - raising=False, - ) - _assert_skipped( PREFIX + "native_graph", lambda c, h: _build_node_tools(SimpleNamespace(config={"tools": c}), h),