From f8cb19f3ab7fd816f18e05de9d3e6d34b5593511 Mon Sep 17 00:00:00 2001 From: Qiming Bao Date: Thu, 5 Nov 2026 21:17:59 +0000 Subject: [PATCH 1/3] Python: Scope local tool checks to the response side of the split tool_called_check, tool_calls_present and tool_call_args_match scanned item.conversation, so a tool call made in an earlier turn (on the query side of the split) satisfied the check for a later turn. With EvalItem.per_turn_items(), a turn with no tool call still passed. Inspect item.split_messages()[1] instead, the same response messages that item.response and the Foundry tool evaluators use. For single-turn items with the default LAST_TURN split the result is unchanged; with FULL, the whole trajectory after the first user message still counts. Fixes #10091 --- .../core/agent_framework/_evaluation.py | 30 +++++++------ .../core/tests/core/test_local_eval.py | 43 +++++++++++++++++++ 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index e0a3a30c9d..5ae9368a87 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -1102,10 +1102,12 @@ def _check(item: EvalItem) -> CheckResult: @experimental(feature_id=ExperimentalFeature.EVALS) def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck: - """Check that specific tools were called during the conversation. + """Check that specific tools were called in the agent's response. - Inspects the conversation history for ``tool_calls`` entries matching - the expected tool names. + Inspects the response side of the item's conversation split (see + ``EvalItem.split_messages``) for function calls matching the expected + tool names. Tool calls on the query side, such as calls made in + earlier turns, are not counted. Args: *tool_names: Names of tools that should have been called. @@ -1125,7 +1127,8 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> def _check(item: EvalItem) -> CheckResult: expected = set(tool_names) called: set[str] = set() - for msg in item.conversation: + _, response_msgs = item.split_messages() + for msg in response_msgs: for c in msg.contents or []: if c.type == "function_call" and c.name: called.add(c.name) @@ -1164,9 +1167,10 @@ def _check(item: EvalItem) -> CheckResult: def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]: - """Extract (name, arguments) pairs from the conversation's function calls.""" + """Extract (name, arguments) pairs from function calls on the response side of the split.""" calls: list[tuple[str, dict[str, Any] | None]] = [] - for msg in item.conversation: + _, response_msgs = item.split_messages() + for msg in response_msgs: for c in msg.contents or []: if c.type == "function_call" and c.name: args: dict[str, Any] | None = None @@ -1194,8 +1198,9 @@ def tool_calls_present(item: EvalItem) -> CheckResult: """Check that all expected tool calls were made (unordered, extras OK). Uses ``item.expected_tool_calls`` — checks that every expected tool name - appears at least once in the conversation. Does not check arguments or - ordering. Extra (unexpected) tool calls are not penalized. + appears at least once in the response side of the item's conversation + split. Does not check arguments or ordering. Extra (unexpected) tool + calls are not penalized. Example: @@ -1235,10 +1240,11 @@ def tool_calls_present(item: EvalItem) -> CheckResult: def tool_call_args_match(item: EvalItem) -> CheckResult: """Check that expected tool calls match on name and arguments. - For each expected tool call, finds matching calls in the conversation - by name. If ``ExpectedToolCall.arguments`` is provided, checks that - the actual arguments contain all expected key-value pairs (subset - match — extra actual arguments are OK). + For each expected tool call, finds matching calls by name in the + response side of the item's conversation split. If + ``ExpectedToolCall.arguments`` is provided, checks that the actual + arguments contain all expected key-value pairs (subset match — extra + actual arguments are OK). Example: diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index f086d80b31..5c5f69ba16 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -12,6 +12,7 @@ from agent_framework._evaluation import ( CheckResult, + ConversationSplit, EvalItem, EvalItemResult, EvalNotPassedError, @@ -973,6 +974,48 @@ async def test_any_mode_none_called(self): assert "None of expected tools" in result.reason # type: ignore[union-attr] # ty: ignore[unresolved-attribute] +class TestToolChecksUseConversationSplit: + """Tool checks inspect the response side of the split, not the whole conversation.""" + + @staticmethod + def _two_turn_conversation() -> list[Message]: + # Only turn 1 calls get_weather; turn 2 makes no tool call. + return [ + Message("user", ["Weather in NYC?"]), + Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), + Message("tool", [Content.from_function_result("c1", result="Sunny")]), + Message("assistant", ["Sunny."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["No idea."]), + ] + + def test_per_turn_tool_checks_ignore_calls_from_prior_turn(self): + first, second = EvalItem.per_turn_items(self._two_turn_conversation()) + expected = [ExpectedToolCall("get_weather", {"location": "NYC"})] + first.expected_tool_calls = expected + second.expected_tool_calls = expected + + assert tool_called_check("get_weather")(first).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(first).passed is True + assert tool_call_args_match(first).passed is True + + assert tool_called_check("get_weather")(second).passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_called_check("get_weather", mode="any")(second).passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(second).passed is False + assert tool_call_args_match(second).passed is False + + def test_full_split_counts_whole_trajectory(self): + item = EvalItem( + conversation=self._two_turn_conversation(), + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + split_strategy=ConversationSplit.FULL, + ) + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True + assert tool_call_args_match(item).passed is True + + class TestCoerceResultScoreError: """Tests for _coerce_result handling non-numeric score.""" From fccec16750cd6148facf011a7e25ff8b0daa7fb4 Mon Sep 17 00:00:00 2001 From: Qiming Bao Date: Mon, 9 Nov 2026 13:44:21 +0000 Subject: [PATCH 2/3] Python: Share the response-side helper and cover the planned cases Use one helper for tool_called_check, tool_calls_present and tool_call_args_match, document that tool checks follow the split, and add tests for single-turn items, items without a user message, custom splits and calls with arguments in the current turn. --- .../core/agent_framework/_evaluation.py | 90 +++++++++++-------- .../core/tests/core/test_local_eval.py | 72 +++++++++++++++ 2 files changed, 124 insertions(+), 38 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 5ae9368a87..fb43a18db9 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -59,6 +59,7 @@ from ._types import ( _EVALUATION_TOOLS_METADATA_KEY, # pyright: ignore[reportPrivateUsage] AgentResponse, + Content, Message, ) @@ -128,6 +129,11 @@ class ConversationSplit(str, Enum): Evaluates whether the *whole conversation trajectory* served the original request. + The split also applies to the local tool checks (``tool_called_check``, + ``tool_calls_present`` and ``tool_call_args_match``): they only count tool + calls on the response side. To check tool calls across a whole multi-turn + conversation, use ``FULL``. + For custom splits, pass any callable with the ``ConversationSplitter`` signature. """ @@ -199,7 +205,8 @@ class EvalItem: expected_tool_calls: Expected tool calls for tool-correctness evaluation. See :class:`ExpectedToolCall`. split_strategy: Split strategy controlling how ``query`` and - ``response`` are derived from the conversation. Defaults to + ``response`` are derived from the conversation, and which tool + calls the local tool checks count. Defaults to ``ConversationSplit.LAST_TURN``. """ @@ -1127,23 +1134,21 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> def _check(item: EvalItem) -> CheckResult: expected = set(tool_names) called: set[str] = set() - _, response_msgs = item.split_messages() - for msg in response_msgs: - for c in msg.contents or []: - if c.type == "function_call" and c.name: - called.add(c.name) - if mode == "all" and expected.issubset(called): - return CheckResult( - passed=True, - reason=f"All expected tools called: {sorted(called)}", - check_name="tool_called", - ) - if mode == "any" and expected & called: - return CheckResult( - passed=True, - reason=f"Expected tool found: {sorted(expected & called)}", - check_name="tool_called", - ) + for c in _response_function_calls(item): + if c.name: + called.add(c.name) + if mode == "all" and expected.issubset(called): + return CheckResult( + passed=True, + reason=f"All expected tools called: {sorted(called)}", + check_name="tool_called", + ) + if mode == "any" and expected & called: + return CheckResult( + passed=True, + reason=f"Expected tool found: {sorted(expected & called)}", + check_name="tool_called", + ) if mode == "all": missing = [t for t in tool_names if t not in called] if missing: @@ -1166,30 +1171,39 @@ def _check(item: EvalItem) -> CheckResult: return _check +def _response_function_calls(item: EvalItem) -> list[Content]: + """Return the function calls on the response side of the item's conversation split. + + The local tool checks use this so that they look at the same messages as + ``item.response``: calls on the query side, such as calls from earlier + turns of a per-turn item, are not counted. + """ + _, response_msgs = item.split_messages() + return [c for msg in response_msgs for c in msg.contents or [] if c.type == "function_call"] + + def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]: """Extract (name, arguments) pairs from function calls on the response side of the split.""" calls: list[tuple[str, dict[str, Any] | None]] = [] - _, response_msgs = item.split_messages() - for msg in response_msgs: - for c in msg.contents or []: - if c.type == "function_call" and c.name: - args: dict[str, Any] | None = None - if c.arguments is None: + for c in _response_function_calls(item): + if c.name: + args: dict[str, Any] | None = None + if c.arguments is None: + args = {} + elif isinstance(c.arguments, Mapping): + args = dict(c.arguments) + elif isinstance(c.arguments, str): + stripped = c.arguments.strip() + if not stripped: args = {} - elif isinstance(c.arguments, Mapping): - args = dict(c.arguments) - elif isinstance(c.arguments, str): - stripped = c.arguments.strip() - if not stripped: - args = {} - else: - try: - parsed = json.loads(stripped) - if isinstance(parsed, dict): - args = cast(dict[str, Any], parsed) - except (json.JSONDecodeError, TypeError): - pass - calls.append((c.name, args)) + else: + try: + parsed = json.loads(stripped) + if isinstance(parsed, dict): + args = cast(dict[str, Any], parsed) + except (json.JSONDecodeError, TypeError): + pass + calls.append((c.name, args)) return calls diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 5c5f69ba16..00e3848bee 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -1015,6 +1015,78 @@ def test_full_split_counts_whole_trajectory(self): assert tool_calls_present(item).passed is True assert tool_call_args_match(item).passed is True + def test_single_turn_last_turn_unchanged(self): + item = EvalItem( + conversation=self._two_turn_conversation()[:4], + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True + assert tool_call_args_match(item).passed is True + + def test_conversation_without_user_message_counts_all_calls(self): + # LAST_TURN treats the whole conversation as the response when there is no user message. + item = EvalItem( + conversation=self._two_turn_conversation()[1:4], + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True + assert tool_call_args_match(item).passed is True + + def test_custom_split_decides_which_calls_count(self): + conversation = self._two_turn_conversation() + expected = [ExpectedToolCall("get_weather", {"location": "NYC"})] + + def call_on_response_side(conv: list[Message]) -> tuple[list[Message], list[Message]]: + return conv[:1], conv[1:] + + def call_on_query_side(conv: list[Message]) -> tuple[list[Message], list[Message]]: + return conv[:4], conv[4:] + + on_response = EvalItem( + conversation=conversation, expected_tool_calls=expected, split_strategy=call_on_response_side + ) + assert tool_called_check("get_weather")(on_response).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(on_response).passed is True + assert tool_call_args_match(on_response).passed is True + + on_query = EvalItem(conversation=conversation, expected_tool_calls=expected, split_strategy=call_on_query_side) + assert tool_called_check("get_weather")(on_query).passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(on_query).passed is False + assert tool_call_args_match(on_query).passed is False + + def test_calls_in_the_current_turn_count_and_match_arguments(self): + conversation = [ + *self._two_turn_conversation()[:4], + Message("user", ["Compare Paris and Rome."]), + Message( + "assistant", + [ + Content.from_function_call("c2", "get_weather", arguments={"location": "Paris"}), + Content.from_function_call("c3", "get_weather", arguments='{"location": "Rome"}'), + ], + ), + Message("tool", [Content.from_function_result("c2", result="Rain")]), + Message("tool", [Content.from_function_result("c3", result="Sunny")]), + Message("assistant", ["Rain in Paris, sun in Rome."]), + ] + _, second = EvalItem.per_turn_items(conversation) + + second.expected_tool_calls = [ + ExpectedToolCall("get_weather", {"location": "Paris"}), + ExpectedToolCall("get_weather", {"location": "Rome"}), + ] + assert tool_called_check("get_weather")(second).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(second).passed is True + assert tool_call_args_match(second).passed is True + + # NYC was only requested in the first turn. + second.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})] + assert tool_call_args_match(second).passed is False + class TestCoerceResultScoreError: """Tests for _coerce_result handling non-numeric score.""" From 5ebd65e51dfa2be36501383db1908b9f4c99f02f Mon Sep 17 00:00:00 2001 From: Qiming Bao Date: Tue, 10 Nov 2026 18:06:50 +0000 Subject: [PATCH 3/3] Python: Provide the two-turn conversation through a conftest fixture Follow python/CODING_STANDARD.md: shared setup lives in conftest.py and is consumed through pytest injection. --- python/packages/core/tests/core/conftest.py | 13 +++++++ .../core/tests/core/test_local_eval.py | 36 +++++++------------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/python/packages/core/tests/core/conftest.py b/python/packages/core/tests/core/conftest.py index 021a916d16..e0597b5e6a 100644 --- a/python/packages/core/tests/core/conftest.py +++ b/python/packages/core/tests/core/conftest.py @@ -77,6 +77,19 @@ def chat_history() -> list[Message]: return [] +@fixture +def two_turn_conversation() -> list[Message]: + """A two-turn conversation in which only turn 1 calls ``get_weather``.""" + return [ + Message("user", ["Weather in NYC?"]), + Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), + Message("tool", [Content.from_function_result("c1", result="Sunny")]), + Message("assistant", ["Sunny."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["No idea."]), + ] + + # region Tools diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index d2863c5d5f..83cefa81a3 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -986,20 +986,8 @@ async def test_any_mode_none_called(self): class TestToolChecksUseConversationSplit: """Tool checks inspect the response side of the split, not the whole conversation.""" - @staticmethod - def _two_turn_conversation() -> list[Message]: - # Only turn 1 calls get_weather; turn 2 makes no tool call. - return [ - Message("user", ["Weather in NYC?"]), - Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), - Message("tool", [Content.from_function_result("c1", result="Sunny")]), - Message("assistant", ["Sunny."]), - Message("user", ["And tomorrow?"]), - Message("assistant", ["No idea."]), - ] - - def test_per_turn_tool_checks_ignore_calls_from_prior_turn(self): - first, second = EvalItem.per_turn_items(self._two_turn_conversation()) + def test_per_turn_tool_checks_ignore_calls_from_prior_turn(self, two_turn_conversation: list[Message]): + first, second = EvalItem.per_turn_items(two_turn_conversation) expected = [ExpectedToolCall("get_weather", {"location": "NYC"})] first.expected_tool_calls = expected second.expected_tool_calls = expected @@ -1013,9 +1001,9 @@ def test_per_turn_tool_checks_ignore_calls_from_prior_turn(self): assert tool_calls_present(second).passed is False assert tool_call_args_match(second).passed is False - def test_full_split_counts_whole_trajectory(self): + def test_full_split_counts_whole_trajectory(self, two_turn_conversation: list[Message]): item = EvalItem( - conversation=self._two_turn_conversation(), + conversation=two_turn_conversation, expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], split_strategy=ConversationSplit.FULL, ) @@ -1024,9 +1012,9 @@ def test_full_split_counts_whole_trajectory(self): assert tool_calls_present(item).passed is True assert tool_call_args_match(item).passed is True - def test_single_turn_last_turn_unchanged(self): + def test_single_turn_last_turn_unchanged(self, two_turn_conversation: list[Message]): item = EvalItem( - conversation=self._two_turn_conversation()[:4], + conversation=two_turn_conversation[:4], expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], ) @@ -1034,10 +1022,10 @@ def test_single_turn_last_turn_unchanged(self): assert tool_calls_present(item).passed is True assert tool_call_args_match(item).passed is True - def test_conversation_without_user_message_counts_all_calls(self): + def test_conversation_without_user_message_counts_all_calls(self, two_turn_conversation: list[Message]): # LAST_TURN treats the whole conversation as the response when there is no user message. item = EvalItem( - conversation=self._two_turn_conversation()[1:4], + conversation=two_turn_conversation[1:4], expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], ) @@ -1045,8 +1033,8 @@ def test_conversation_without_user_message_counts_all_calls(self): assert tool_calls_present(item).passed is True assert tool_call_args_match(item).passed is True - def test_custom_split_decides_which_calls_count(self): - conversation = self._two_turn_conversation() + def test_custom_split_decides_which_calls_count(self, two_turn_conversation: list[Message]): + conversation = two_turn_conversation expected = [ExpectedToolCall("get_weather", {"location": "NYC"})] def call_on_response_side(conv: list[Message]) -> tuple[list[Message], list[Message]]: @@ -1067,9 +1055,9 @@ def call_on_query_side(conv: list[Message]) -> tuple[list[Message], list[Message assert tool_calls_present(on_query).passed is False assert tool_call_args_match(on_query).passed is False - def test_calls_in_the_current_turn_count_and_match_arguments(self): + def test_calls_in_the_current_turn_count_and_match_arguments(self, two_turn_conversation: list[Message]): conversation = [ - *self._two_turn_conversation()[:4], + *two_turn_conversation[:4], Message("user", ["Compare Paris and Rome."]), Message( "assistant",