Python: Scope local tool checks to the response side of the conversation split

python/packages/core/agent_framework/_evaluation.py

@@ -59,6 +59,7 @@
 from ._types import (
     _EVALUATION_TOOLS_METADATA_KEY,  # pyright: ignore[reportPrivateUsage]
     AgentResponse,
+    Content,
     Message,
 )
 
@@ -128,6 +129,11 @@ class ConversationSplit(str, Enum):
       Evaluates whether the *whole conversation trajectory* served the
       original request.
 
+    The split also applies to the local tool checks (``tool_called_check``,
+    ``tool_calls_present`` and ``tool_call_args_match``): they only count tool
+    calls on the response side.  To check tool calls across a whole multi-turn
+    conversation, use ``FULL``.
+
     For custom splits, pass any callable with the ``ConversationSplitter``
     signature.
     """
@@ -199,7 +205,8 @@ class EvalItem:
         expected_tool_calls: Expected tool calls for tool-correctness
             evaluation.  See :class:`ExpectedToolCall`.
         split_strategy: Split strategy controlling how ``query`` and
-            ``response`` are derived from the conversation. Defaults to
+            ``response`` are derived from the conversation, and which tool
+            calls the local tool checks count. Defaults to
             ``ConversationSplit.LAST_TURN``.
     """
 
@@ -1102,10 +1109,12 @@ def _check(item: EvalItem) -> CheckResult:
 
 @experimental(feature_id=ExperimentalFeature.EVALS)
 def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck:
-    """Check that specific tools were called during the conversation.
+    """Check that specific tools were called in the agent's response.
 
-    Inspects the conversation history for ``tool_calls`` entries matching
-    the expected tool names.
+    Inspects the response side of the item's conversation split (see
+    ``EvalItem.split_messages``) for function calls matching the expected
+    tool names.  Tool calls on the query side, such as calls made in
+    earlier turns, are not counted.
 
     Args:
         *tool_names: Names of tools that should have been called.
@@ -1125,22 +1134,21 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") ->
     def _check(item: EvalItem) -> CheckResult:
         expected = set(tool_names)
         called: set[str] = set()
-        for msg in item.conversation:
-            for c in msg.contents or []:
-                if c.type == "function_call" and c.name:
-                    called.add(c.name)
-                    if mode == "all" and expected.issubset(called):
-                        return CheckResult(
-                            passed=True,
-                            reason=f"All expected tools called: {sorted(called)}",
-                            check_name="tool_called",
-                        )
-                    if mode == "any" and expected & called:
-                        return CheckResult(
-                            passed=True,
-                            reason=f"Expected tool found: {sorted(expected & called)}",
-                            check_name="tool_called",
-                        )
+        for c in _response_function_calls(item):
+            if c.name:
+                called.add(c.name)
+                if mode == "all" and expected.issubset(called):
+                    return CheckResult(
+                        passed=True,
+                        reason=f"All expected tools called: {sorted(called)}",
+                        check_name="tool_called",
+                    )
+                if mode == "any" and expected & called:
+                    return CheckResult(
+                        passed=True,
+                        reason=f"Expected tool found: {sorted(expected & called)}",
+                        check_name="tool_called",
+                    )
         if mode == "all":
             missing = [t for t in tool_names if t not in called]
             if missing:
@@ -1163,29 +1171,39 @@ def _check(item: EvalItem) -> CheckResult:
     return _check
 
 
+def _response_function_calls(item: EvalItem) -> list[Content]:
+    """Return the function calls on the response side of the item's conversation split.
+
+    The local tool checks use this so that they look at the same messages as
+    ``item.response``: calls on the query side, such as calls from earlier
+    turns of a per-turn item, are not counted.
+    """
+    _, response_msgs = item.split_messages()
+    return [c for msg in response_msgs for c in msg.contents or [] if c.type == "function_call"]
+
+
 def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]:
-    """Extract (name, arguments) pairs from the conversation's function calls."""
+    """Extract (name, arguments) pairs from function calls on the response side of the split."""
     calls: list[tuple[str, dict[str, Any] | None]] = []
-    for msg in item.conversation:
-        for c in msg.contents or []:
-            if c.type == "function_call" and c.name:
-                args: dict[str, Any] | None = None
-                if c.arguments is None:
+    for c in _response_function_calls(item):
+        if c.name:
+            args: dict[str, Any] | None = None
+            if c.arguments is None:
+                args = {}
+            elif isinstance(c.arguments, Mapping):
+                args = dict(c.arguments)
+            elif isinstance(c.arguments, str):
+                stripped = c.arguments.strip()
+                if not stripped:
                     args = {}
-                elif isinstance(c.arguments, Mapping):
-                    args = dict(c.arguments)
-                elif isinstance(c.arguments, str):
-                    stripped = c.arguments.strip()
-                    if not stripped:
-                        args = {}
-                    else:
-                        try:
-                            parsed = json.loads(stripped)
-                            if isinstance(parsed, dict):
-                                args = cast(dict[str, Any], parsed)
-                        except (json.JSONDecodeError, TypeError):
-                            pass
-                calls.append((c.name, args))
+                else:
+                    try:
+                        parsed = json.loads(stripped)
+                        if isinstance(parsed, dict):
+                            args = cast(dict[str, Any], parsed)
+                    except (json.JSONDecodeError, TypeError):
+                        pass
+            calls.append((c.name, args))
     return calls
 
 
@@ -1194,8 +1212,9 @@ def tool_calls_present(item: EvalItem) -> CheckResult:
     """Check that all expected tool calls were made (unordered, extras OK).
 
     Uses ``item.expected_tool_calls`` — checks that every expected tool name
-    appears at least once in the conversation.  Does not check arguments or
-    ordering.  Extra (unexpected) tool calls are not penalized.
+    appears at least once in the response side of the item's conversation
+    split.  Does not check arguments or ordering.  Extra (unexpected) tool
+    calls are not penalized.
 
     Example:
 
@@ -1235,10 +1254,11 @@ def tool_calls_present(item: EvalItem) -> CheckResult:
 def tool_call_args_match(item: EvalItem) -> CheckResult:
     """Check that expected tool calls match on name and arguments.
 
-    For each expected tool call, finds matching calls in the conversation
-    by name.  If ``ExpectedToolCall.arguments`` is provided, checks that
-    the actual arguments contain all expected key-value pairs (subset
-    match — extra actual arguments are OK).
+    For each expected tool call, finds matching calls by name in the
+    response side of the item's conversation split.  If
+    ``ExpectedToolCall.arguments`` is provided, checks that the actual
+    arguments contain all expected key-value pairs (subset match — extra
+    actual arguments are OK).
 
     Example:
 

python/packages/core/tests/core/conftest.py

@@ -77,6 +77,19 @@ def chat_history() -> list[Message]:
     return []
 
 
+@fixture
+def two_turn_conversation() -> list[Message]:
+    """A two-turn conversation in which only turn 1 calls ``get_weather``."""
+    return [
+        Message("user", ["Weather in NYC?"]),
+        Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]),
+        Message("tool", [Content.from_function_result("c1", result="Sunny")]),
+        Message("assistant", ["Sunny."]),
+        Message("user", ["And tomorrow?"]),
+        Message("assistant", ["No idea."]),
+    ]
+
+
 # region Tools
 
 

python/packages/core/tests/core/test_local_eval.py

@@ -12,6 +12,7 @@
 
 from agent_framework._evaluation import (
     CheckResult,
+    ConversationSplit,
     EvalItem,
     EvalItemResult,
     EvalNotPassedError,
@@ -982,6 +983,108 @@ async def test_any_mode_none_called(self):
         assert "None of expected tools" in result.reason  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
 
 
+class TestToolChecksUseConversationSplit:
+    """Tool checks inspect the response side of the split, not the whole conversation."""
+
+    def test_per_turn_tool_checks_ignore_calls_from_prior_turn(self, two_turn_conversation: list[Message]):
+        first, second = EvalItem.per_turn_items(two_turn_conversation)
+        expected = [ExpectedToolCall("get_weather", {"location": "NYC"})]
+        first.expected_tool_calls = expected
+        second.expected_tool_calls = expected
+
+        assert tool_called_check("get_weather")(first).passed is True  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(first).passed is True
+        assert tool_call_args_match(first).passed is True
+
+        assert tool_called_check("get_weather")(second).passed is False  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_called_check("get_weather", mode="any")(second).passed is False  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(second).passed is False
+        assert tool_call_args_match(second).passed is False
+
+    def test_full_split_counts_whole_trajectory(self, two_turn_conversation: list[Message]):
+        item = EvalItem(
+            conversation=two_turn_conversation,
+            expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})],
+            split_strategy=ConversationSplit.FULL,
+        )
+
+        assert tool_called_check("get_weather")(item).passed is True  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(item).passed is True
+        assert tool_call_args_match(item).passed is True
+
+    def test_single_turn_last_turn_unchanged(self, two_turn_conversation: list[Message]):
+        item = EvalItem(
+            conversation=two_turn_conversation[:4],
+            expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})],
+        )
+
+        assert tool_called_check("get_weather")(item).passed is True  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(item).passed is True
+        assert tool_call_args_match(item).passed is True
+
+    def test_conversation_without_user_message_counts_all_calls(self, two_turn_conversation: list[Message]):
+        # LAST_TURN treats the whole conversation as the response when there is no user message.
+        item = EvalItem(
+            conversation=two_turn_conversation[1:4],
+            expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})],
+        )
+
+        assert tool_called_check("get_weather")(item).passed is True  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(item).passed is True
+        assert tool_call_args_match(item).passed is True
+
+    def test_custom_split_decides_which_calls_count(self, two_turn_conversation: list[Message]):
+        conversation = two_turn_conversation
+        expected = [ExpectedToolCall("get_weather", {"location": "NYC"})]
+
+        def call_on_response_side(conv: list[Message]) -> tuple[list[Message], list[Message]]:
+            return conv[:1], conv[1:]
+
+        def call_on_query_side(conv: list[Message]) -> tuple[list[Message], list[Message]]:
+            return conv[:4], conv[4:]
+
+        on_response = EvalItem(
+            conversation=conversation, expected_tool_calls=expected, split_strategy=call_on_response_side
+        )
+        assert tool_called_check("get_weather")(on_response).passed is True  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(on_response).passed is True
+        assert tool_call_args_match(on_response).passed is True
+
+        on_query = EvalItem(conversation=conversation, expected_tool_calls=expected, split_strategy=call_on_query_side)
+        assert tool_called_check("get_weather")(on_query).passed is False  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(on_query).passed is False
+        assert tool_call_args_match(on_query).passed is False
+
+    def test_calls_in_the_current_turn_count_and_match_arguments(self, two_turn_conversation: list[Message]):
+        conversation = [
+            *two_turn_conversation[:4],
+            Message("user", ["Compare Paris and Rome."]),
+            Message(
+                "assistant",
+                [
+                    Content.from_function_call("c2", "get_weather", arguments={"location": "Paris"}),
+                    Content.from_function_call("c3", "get_weather", arguments='{"location": "Rome"}'),
+                ],
+            ),
+            Message("tool", [Content.from_function_result("c2", result="Rain")]),
+            Message("tool", [Content.from_function_result("c3", result="Sunny")]),
+            Message("assistant", ["Rain in Paris, sun in Rome."]),
+        ]
+        _, second = EvalItem.per_turn_items(conversation)
+
+        second.expected_tool_calls = [
+            ExpectedToolCall("get_weather", {"location": "Paris"}),
+            ExpectedToolCall("get_weather", {"location": "Rome"}),
+        ]
+        assert tool_called_check("get_weather")(second).passed is True  # type: ignore[union-attr]  # ty: ignore[unresolved-attribute]
+        assert tool_calls_present(second).passed is True
+        assert tool_call_args_match(second).passed is True
+
+        # NYC was only requested in the first turn.
+        second.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})]
+        assert tool_call_args_match(second).passed is False
+
+
 class TestCoerceResultScoreError:
     """Tests for _coerce_result handling non-numeric score."""