diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index e7ffb79088d..1c57b9f72d8 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -40,7 +40,7 @@ import json import logging import warnings -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field from enum import Enum from typing import ( @@ -1119,15 +1119,21 @@ def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None for c in msg.contents or []: if c.type == "function_call" and c.name: args: dict[str, Any] | None = None - if isinstance(c.arguments, dict): - args = c.arguments + if c.arguments is None: + args = {} + elif isinstance(c.arguments, Mapping): + args = dict(c.arguments) elif isinstance(c.arguments, str): - try: - parsed = json.loads(c.arguments) - if isinstance(parsed, dict): - args = cast(dict[str, Any], parsed) - except (json.JSONDecodeError, TypeError): - pass + stripped = c.arguments.strip() + if not stripped: + args = {} + else: + try: + parsed = json.loads(stripped) + if isinstance(parsed, dict): + args = cast(dict[str, Any], parsed) + except (json.JSONDecodeError, TypeError): + pass calls.append((c.name, args)) return calls diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index e00e0d6a951..9875b2aeeee 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -6,6 +6,7 @@ import inspect import json +from typing import Any import pytest @@ -529,7 +530,7 @@ def check_b(response: str) -> bool: def _make_tool_call_item( - calls: list[tuple[str, dict | None]], + calls: list[tuple[str, Any]], expected: list[ExpectedToolCall] | None = None, ) -> EvalItem: """Build an EvalItem with tool calls in the conversation.""" @@ -644,6 +645,57 @@ def test_null_argument_requires_key(self, arguments, expected_passed, as_json): assert result.passed is expected_passed + def test_empty_expected_args_matches_none_actual_args(self): + item = _make_tool_call_item( + calls=[("get_weather", None)], + expected=[ExpectedToolCall("get_weather", {})], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_empty_expected_args_matches_empty_actual_args(self): + item = _make_tool_call_item( + calls=[("get_weather", {})], + expected=[ExpectedToolCall("get_weather", {})], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_empty_expected_args_matches_empty_string_actual_args(self): + item = _make_tool_call_item( + calls=[("get_weather", "")], + expected=[ExpectedToolCall("get_weather", {})], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_empty_expected_args_does_not_match_invalid_json(self): + item = _make_tool_call_item( + calls=[("get_weather", "not-json")], + expected=[ExpectedToolCall("get_weather", {})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "args mismatch" in result.reason + + def test_empty_expected_args_does_not_match_non_object_json(self): + item = _make_tool_call_item( + calls=[("get_weather", "[]")], + expected=[ExpectedToolCall("get_weather", {})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "args mismatch" in result.reason + + def test_empty_expected_args_does_not_match_non_object_payload(self): + item = _make_tool_call_item( + calls=[("get_weather", [])], + expected=[ExpectedToolCall("get_weather", {})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "args mismatch" in result.reason + def test_tool_not_called(self): item = _make_tool_call_item( calls=[("get_news", None)],