diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 787cac17bd..f5381656a0 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -59,6 +59,7 @@ from ._types import ( _EVALUATION_TOOLS_METADATA_KEY, # pyright: ignore[reportPrivateUsage] AgentResponse, + Content, Message, ) @@ -128,6 +129,11 @@ class ConversationSplit(str, Enum): Evaluates whether the *whole conversation trajectory* served the original request. + The split also applies to the local tool checks (``tool_called_check``, + ``tool_calls_present`` and ``tool_call_args_match``): they only count tool + calls on the response side. To check tool calls across a whole multi-turn + conversation, use ``FULL``. + For custom splits, pass any callable with the ``ConversationSplitter`` signature. """ @@ -199,7 +205,8 @@ class EvalItem: expected_tool_calls: Expected tool calls for tool-correctness evaluation. See :class:`ExpectedToolCall`. split_strategy: Split strategy controlling how ``query`` and - ``response`` are derived from the conversation. Defaults to + ``response`` are derived from the conversation, and which tool + calls the local tool checks count. Defaults to ``ConversationSplit.LAST_TURN``. """ @@ -1102,10 +1109,12 @@ def _check(item: EvalItem) -> CheckResult: @experimental(feature_id=ExperimentalFeature.EVALS) def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck: - """Check that specific tools were called during the conversation. + """Check that specific tools were called in the agent's response. - Inspects the conversation history for ``tool_calls`` entries matching - the expected tool names. + Inspects the response side of the item's conversation split (see + ``EvalItem.split_messages``) for function calls matching the expected + tool names. Tool calls on the query side, such as calls made in + earlier turns, are not counted. Args: *tool_names: Names of tools that should have been called. @@ -1125,22 +1134,21 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> def _check(item: EvalItem) -> CheckResult: expected = set(tool_names) called: set[str] = set() - for msg in item.conversation: - for c in msg.contents or []: - if c.type == "function_call" and c.name: - called.add(c.name) - if mode == "all" and expected.issubset(called): - return CheckResult( - passed=True, - reason=f"All expected tools called: {sorted(called)}", - check_name="tool_called", - ) - if mode == "any" and expected & called: - return CheckResult( - passed=True, - reason=f"Expected tool found: {sorted(expected & called)}", - check_name="tool_called", - ) + for c in _response_function_calls(item): + if c.name: + called.add(c.name) + if mode == "all" and expected.issubset(called): + return CheckResult( + passed=True, + reason=f"All expected tools called: {sorted(called)}", + check_name="tool_called", + ) + if mode == "any" and expected & called: + return CheckResult( + passed=True, + reason=f"Expected tool found: {sorted(expected & called)}", + check_name="tool_called", + ) if mode == "all": missing = [t for t in tool_names if t not in called] if missing: @@ -1163,29 +1171,39 @@ def _check(item: EvalItem) -> CheckResult: return _check +def _response_function_calls(item: EvalItem) -> list[Content]: + """Return the function calls on the response side of the item's conversation split. + + The local tool checks use this so that they look at the same messages as + ``item.response``: calls on the query side, such as calls from earlier + turns of a per-turn item, are not counted. + """ + _, response_msgs = item.split_messages() + return [c for msg in response_msgs for c in msg.contents or [] if c.type == "function_call"] + + def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]: - """Extract (name, arguments) pairs from the conversation's function calls.""" + """Extract (name, arguments) pairs from function calls on the response side of the split.""" calls: list[tuple[str, dict[str, Any] | None]] = [] - for msg in item.conversation: - for c in msg.contents or []: - if c.type == "function_call" and c.name: - args: dict[str, Any] | None = None - if c.arguments is None: + for c in _response_function_calls(item): + if c.name: + args: dict[str, Any] | None = None + if c.arguments is None: + args = {} + elif isinstance(c.arguments, Mapping): + args = dict(c.arguments) + elif isinstance(c.arguments, str): + stripped = c.arguments.strip() + if not stripped: args = {} - elif isinstance(c.arguments, Mapping): - args = dict(c.arguments) - elif isinstance(c.arguments, str): - stripped = c.arguments.strip() - if not stripped: - args = {} - else: - try: - parsed = json.loads(stripped) - if isinstance(parsed, dict): - args = cast(dict[str, Any], parsed) - except (json.JSONDecodeError, TypeError): - pass - calls.append((c.name, args)) + else: + try: + parsed = json.loads(stripped) + if isinstance(parsed, dict): + args = cast(dict[str, Any], parsed) + except (json.JSONDecodeError, TypeError): + pass + calls.append((c.name, args)) return calls @@ -1194,8 +1212,9 @@ def tool_calls_present(item: EvalItem) -> CheckResult: """Check that all expected tool calls were made (unordered, extras OK). Uses ``item.expected_tool_calls`` — checks that every expected tool name - appears at least once in the conversation. Does not check arguments or - ordering. Extra (unexpected) tool calls are not penalized. + appears at least once in the response side of the item's conversation + split. Does not check arguments or ordering. Extra (unexpected) tool + calls are not penalized. Example: @@ -1235,10 +1254,11 @@ def tool_calls_present(item: EvalItem) -> CheckResult: def tool_call_args_match(item: EvalItem) -> CheckResult: """Check that expected tool calls match on name and arguments. - For each expected tool call, finds matching calls in the conversation - by name. If ``ExpectedToolCall.arguments`` is provided, checks that - the actual arguments contain all expected key-value pairs (subset - match — extra actual arguments are OK). + For each expected tool call, finds matching calls by name in the + response side of the item's conversation split. If + ``ExpectedToolCall.arguments`` is provided, checks that the actual + arguments contain all expected key-value pairs (subset match — extra + actual arguments are OK). Example: diff --git a/python/packages/core/tests/core/conftest.py b/python/packages/core/tests/core/conftest.py index 021a916d16..e0597b5e6a 100644 --- a/python/packages/core/tests/core/conftest.py +++ b/python/packages/core/tests/core/conftest.py @@ -77,6 +77,19 @@ def chat_history() -> list[Message]: return [] +@fixture +def two_turn_conversation() -> list[Message]: + """A two-turn conversation in which only turn 1 calls ``get_weather``.""" + return [ + Message("user", ["Weather in NYC?"]), + Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), + Message("tool", [Content.from_function_result("c1", result="Sunny")]), + Message("assistant", ["Sunny."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["No idea."]), + ] + + # region Tools diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 06f39d634a..83cefa81a3 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -12,6 +12,7 @@ from agent_framework._evaluation import ( CheckResult, + ConversationSplit, EvalItem, EvalItemResult, EvalNotPassedError, @@ -982,6 +983,108 @@ async def test_any_mode_none_called(self): assert "None of expected tools" in result.reason # type: ignore[union-attr] # ty: ignore[unresolved-attribute] +class TestToolChecksUseConversationSplit: + """Tool checks inspect the response side of the split, not the whole conversation.""" + + def test_per_turn_tool_checks_ignore_calls_from_prior_turn(self, two_turn_conversation: list[Message]): + first, second = EvalItem.per_turn_items(two_turn_conversation) + expected = [ExpectedToolCall("get_weather", {"location": "NYC"})] + first.expected_tool_calls = expected + second.expected_tool_calls = expected + + assert tool_called_check("get_weather")(first).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(first).passed is True + assert tool_call_args_match(first).passed is True + + assert tool_called_check("get_weather")(second).passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_called_check("get_weather", mode="any")(second).passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(second).passed is False + assert tool_call_args_match(second).passed is False + + def test_full_split_counts_whole_trajectory(self, two_turn_conversation: list[Message]): + item = EvalItem( + conversation=two_turn_conversation, + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + split_strategy=ConversationSplit.FULL, + ) + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True + assert tool_call_args_match(item).passed is True + + def test_single_turn_last_turn_unchanged(self, two_turn_conversation: list[Message]): + item = EvalItem( + conversation=two_turn_conversation[:4], + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True + assert tool_call_args_match(item).passed is True + + def test_conversation_without_user_message_counts_all_calls(self, two_turn_conversation: list[Message]): + # LAST_TURN treats the whole conversation as the response when there is no user message. + item = EvalItem( + conversation=two_turn_conversation[1:4], + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True + assert tool_call_args_match(item).passed is True + + def test_custom_split_decides_which_calls_count(self, two_turn_conversation: list[Message]): + conversation = two_turn_conversation + expected = [ExpectedToolCall("get_weather", {"location": "NYC"})] + + def call_on_response_side(conv: list[Message]) -> tuple[list[Message], list[Message]]: + return conv[:1], conv[1:] + + def call_on_query_side(conv: list[Message]) -> tuple[list[Message], list[Message]]: + return conv[:4], conv[4:] + + on_response = EvalItem( + conversation=conversation, expected_tool_calls=expected, split_strategy=call_on_response_side + ) + assert tool_called_check("get_weather")(on_response).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(on_response).passed is True + assert tool_call_args_match(on_response).passed is True + + on_query = EvalItem(conversation=conversation, expected_tool_calls=expected, split_strategy=call_on_query_side) + assert tool_called_check("get_weather")(on_query).passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(on_query).passed is False + assert tool_call_args_match(on_query).passed is False + + def test_calls_in_the_current_turn_count_and_match_arguments(self, two_turn_conversation: list[Message]): + conversation = [ + *two_turn_conversation[:4], + Message("user", ["Compare Paris and Rome."]), + Message( + "assistant", + [ + Content.from_function_call("c2", "get_weather", arguments={"location": "Paris"}), + Content.from_function_call("c3", "get_weather", arguments='{"location": "Rome"}'), + ], + ), + Message("tool", [Content.from_function_result("c2", result="Rain")]), + Message("tool", [Content.from_function_result("c3", result="Sunny")]), + Message("assistant", ["Rain in Paris, sun in Rome."]), + ] + _, second = EvalItem.per_turn_items(conversation) + + second.expected_tool_calls = [ + ExpectedToolCall("get_weather", {"location": "Paris"}), + ExpectedToolCall("get_weather", {"location": "Rome"}), + ] + assert tool_called_check("get_weather")(second).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(second).passed is True + assert tool_call_args_match(second).passed is True + + # NYC was only requested in the first turn. + second.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})] + assert tool_call_args_match(second).passed is False + + class TestCoerceResultScoreError: """Tests for _coerce_result handling non-numeric score."""