diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 257782f7c34..288e07a2a6e 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -217,6 +217,17 @@ agent_framework/ hosts remain responsible for authorizing and tenant-scoping access to any shared checkpoint adapter. - **Orchestrators**: `SequentialOrchestrator`, `ConcurrentOrchestrator`, `GroupChatOrchestrator`, `MagenticOrchestrator`, `HandoffOrchestrator` +## Evaluation (`_evaluation.py`) + +- Core owns provider-neutral evaluation types, local evaluators and checks, `EvalItem` construction, and the + `evaluate_agent` / `evaluate_workflow` orchestration functions. +- Provider packages own their service-specific evaluator implementations and wire serialization. Core evaluation + code must not emit a provider's request schema. The deprecated `AgentEvalConverter` remains as a temporary + compatibility shim for released Foundry packages whose declared core range still imports it; new code must not use + the shim. +- Core orchestration builds `EvalItem` instances through private helpers; callers needing manual control construct + the public `EvalItem` directly. + ## Built-in Providers ### OpenAI (`openai/`) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index ab71d84a15c..327258b846b 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -39,6 +39,7 @@ import inspect import json import logging +import warnings from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from enum import Enum @@ -54,7 +55,7 @@ ) from ._feature_stage import ExperimentalFeature, experimental -from ._tools import FunctionTool +from ._tools import FunctionTool, normalize_tools from ._types import AgentResponse, Message if TYPE_CHECKING: @@ -726,157 +727,121 @@ async def evaluate( # endregion -# region Converter + +def _warn_agent_eval_converter_deprecated() -> None: + """Warn when the legacy evaluation converter compatibility surface is used.""" + warnings.warn( + "`AgentEvalConverter` is deprecated and will be removed in a future version. " + "Construct `EvalItem` directly or use `evaluate_agent()` / `evaluate_workflow()`; " + "Foundry wire conversion is internal to `agent-framework-foundry`.", + DeprecationWarning, + stacklevel=3, + ) + + +def _convert_legacy_foundry_message(message: Message) -> list[dict[str, Any]]: + """Preserve the legacy Foundry wire conversion for package compatibility.""" + content_items: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + + for content in message.contents or []: + if content.type == "text" and content.text: + content_items.append({"type": "text", "text": content.text}) + elif content.type in ("data", "uri") and content.uri: + image: dict[str, Any] = { + "type": "input_image", + "image_url": content.uri, + } + if content.media_type: + image["detail"] = "auto" + content_items.append(image) + elif content.type == "function_call": + arguments = content.arguments + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except (json.JSONDecodeError, TypeError): + arguments = {"_raw_arguments": "[unparseable]"} + content_items.append({ + "type": "tool_call", + "tool_call_id": content.call_id or "", + "name": content.name or "", + "arguments": arguments if arguments is not None else {}, + }) + elif content.type == "function_result": + result = content.result + if isinstance(result, str): + with contextlib.suppress(json.JSONDecodeError, TypeError): + result = json.loads(result) + tool_results.append({ + "call_id": content.call_id or "", + "result": result, + }) + + if tool_results: + return [ + { + "role": "tool", + "tool_call_id": tool_result["call_id"], + "content": [{"type": "tool_result", "tool_result": tool_result["result"]}], + } + for tool_result in tool_results + ] + if content_items: + return [{"role": message.role, "content": content_items}] + return [ + { + "role": message.role, + "content": [{"type": "text", "text": ""}], + } + ] @experimental(feature_id=ExperimentalFeature.EVALS) class AgentEvalConverter: - """Converts agent-framework types to evaluation format. + """Deprecated compatibility surface for earlier Agent Framework releases. - Handles the type gap between agent-framework's ``Message`` / ``Content`` / - ``FunctionTool`` types and the OpenAI-style agent message schema used by - evaluation providers. All methods are static — no instantiation needed. + New code should construct :class:`EvalItem` directly or use + :func:`evaluate_agent` / :func:`evaluate_workflow`. Foundry-specific wire + serialization is owned by ``agent-framework-foundry``. """ @staticmethod def convert_message(message: Message) -> list[dict[str, Any]]: - """Convert a single ``Message`` to Foundry agent evaluator format. - - Uses typed content lists as required by Foundry evaluators: - - .. code-block:: python - - {"role": "assistant", "content": [{"type": "tool_call", ...}]} - {"role": "user", "content": [{"type": "input_image", ...}]} - - Supported content types: - - * ``text`` → ``{"type": "text", "text": ...}`` - * ``data`` / ``uri`` (images) → ``{"type": "input_image", "image_url": ...}`` - * ``function_call`` → ``{"type": "tool_call", ...}`` - * ``function_result`` → ``{"type": "tool_result", ...}`` - - A single agent-framework ``Message`` with multiple ``function_result`` - contents produces multiple output messages (one per tool result). - - Args: - message: An agent-framework ``Message``. - - Returns: - A list of Foundry-format message dicts. - """ - role = message.role - contents = message.contents or [] - - content_items: list[dict[str, Any]] = [] - tool_results: list[dict[str, Any]] = [] - - for c in contents: - if c.type == "text" and c.text: - content_items.append({"type": "text", "text": c.text}) - elif c.type in ("data", "uri") and c.uri: - # Image / media content → OpenAI input_image format - img: dict[str, Any] = { - "type": "input_image", - "image_url": c.uri, - } - if c.media_type: - img["detail"] = "auto" - content_items.append(img) - elif c.type == "function_call": - args = c.arguments - if isinstance(args, str): - try: - args = json.loads(args) - except (json.JSONDecodeError, TypeError): - # Sanitize to avoid leaking sensitive tool-call arguments - # to external evaluation services. - args = {"_raw_arguments": "[unparseable]"} - tc: dict[str, Any] = { - "type": "tool_call", - "tool_call_id": c.call_id or "", - "name": c.name or "", - } - tc["arguments"] = args if args is not None else {} - content_items.append(tc) - elif c.type == "function_result": - result_val = c.result - if isinstance(result_val, str): - with contextlib.suppress(json.JSONDecodeError, TypeError): - result_val = json.loads(result_val) - tool_results.append({ - "call_id": c.call_id or "", - "result": result_val, - }) - - output: list[dict[str, Any]] = [] - - if tool_results: - for tr in tool_results: - output.append({ - "role": "tool", - "tool_call_id": tr["call_id"], - "content": [{"type": "tool_result", "tool_result": tr["result"]}], - }) - elif content_items: - output.append({"role": role, "content": content_items}) - else: - output.append({ - "role": role, - "content": [{"type": "text", "text": ""}], - }) - - return output + """Convert one message using the legacy Foundry evaluator wire format.""" + _warn_agent_eval_converter_deprecated() + return _convert_legacy_foundry_message(message) @staticmethod def convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: - """Convert a sequence of ``Message`` objects to Foundry evaluator format. - - Args: - messages: Agent-framework messages. - - Returns: - A list of Foundry-format message dicts with typed content lists. - """ - result: list[dict[str, Any]] = [] - for msg in messages: - result.extend(AgentEvalConverter.convert_message(msg)) - return result + """Convert messages using the legacy Foundry evaluator wire format.""" + _warn_agent_eval_converter_deprecated() + return [converted for message in messages for converted in _convert_legacy_foundry_message(message)] @staticmethod def extract_tools(agent: Any) -> list[dict[str, Any]]: - """Extract tool definitions from an agent instance. - - Reads ``agent.default_options["tools"]`` and ``agent.mcp_tools`` - and converts each ``FunctionTool`` to ``{name, description, parameters}``. - - Args: - agent: An agent-framework agent instance. - - Returns: - A list of tool definition dicts. - """ + """Extract legacy evaluator tool-definition dictionaries from an agent.""" + _warn_agent_eval_converter_deprecated() tools: list[dict[str, Any]] = [] seen: set[str] = set() raw_tools = getattr(agent, "default_options", {}).get("tools", []) - for t in raw_tools: - if isinstance(t, FunctionTool) and t.name not in seen: + for tool in raw_tools: + if isinstance(tool, FunctionTool) and tool.name not in seen: tools.append({ - "name": t.name, - "description": t.description, - "parameters": t.parameters(), + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters(), }) - seen.add(t.name) - # Include tools from connected MCP servers + seen.add(tool.name) for mcp in getattr(agent, "mcp_tools", []): - for t in getattr(mcp, "functions", []): - if isinstance(t, FunctionTool) and t.name not in seen: + for tool in getattr(mcp, "functions", []): + if isinstance(tool, FunctionTool) and tool.name not in seen: tools.append({ - "name": t.name, - "description": t.description, - "parameters": t.parameters(), + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters(), }) - seen.add(t.name) + seen.add(tool.name) return tools @staticmethod @@ -885,47 +850,46 @@ def to_eval_item( query: str | Sequence[Message], response: AgentResponse[Any], agent: Any | None = None, - tools: Sequence[FunctionTool] | None = None, + tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, context: str | None = None, ) -> EvalItem: - """Convert a complete agent interaction to an ``EvalItem``. + """Build an ``EvalItem`` through the provider-neutral compatibility path.""" + _warn_agent_eval_converter_deprecated() + return _to_eval_item(query=query, response=response, agent=agent, tools=tools, context=context) - Args: - query: The user query string, or input messages. - response: The agent's response. - agent: Optional agent instance to auto-extract tool definitions. - tools: Explicit tool list (takes precedence over *agent*). - context: Optional context document for groundedness evaluation. - - Returns: - An ``EvalItem`` suitable for passing to any ``Evaluator``. - """ - input_msgs = [Message("user", [query])] if isinstance(query, str) else list(query) - - all_msgs = list(input_msgs) + list(response.messages or []) - - typed_tools: list[FunctionTool] = [] - if tools: - typed_tools = list(tools) - elif agent: - raw_tools = getattr(agent, "default_options", {}).get("tools", []) - typed_tools = [t for t in raw_tools if isinstance(t, FunctionTool)] - # Include tools from connected MCP servers - seen = {t.name for t in typed_tools} - for mcp in getattr(agent, "mcp_tools", []): - for t in getattr(mcp, "functions", []): - if isinstance(t, FunctionTool) and t.name not in seen: - typed_tools.append(t) - seen.add(t.name) - - return EvalItem( - conversation=all_msgs, - tools=typed_tools or None, - context=context, - ) +@experimental(feature_id=ExperimentalFeature.EVALS) +def _to_eval_item( + *, + query: str | Sequence[Message], + response: AgentResponse[Any], + agent: Any | None = None, + tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, + context: str | None = None, +) -> EvalItem: + """Build a provider-neutral ``EvalItem`` from an agent interaction.""" + input_msgs = [Message("user", [query])] if isinstance(query, str) else list(query) + all_msgs = list(input_msgs) + list(response.messages or []) + + typed_tools: list[FunctionTool] = [] + if tools: + typed_tools = [tool for tool in normalize_tools(tools) if isinstance(tool, FunctionTool)] + elif agent: + raw_tools = getattr(agent, "default_options", {}).get("tools", []) + typed_tools = [tool for tool in normalize_tools(raw_tools) if isinstance(tool, FunctionTool)] + seen = {tool.name for tool in typed_tools} + for mcp in getattr(agent, "mcp_tools", []): + for tool in getattr(mcp, "functions", []): + if isinstance(tool, FunctionTool) and tool.name not in seen: + typed_tools.append(tool) + seen.add(tool.name) + + return EvalItem( + conversation=all_msgs, + tools=typed_tools or None, + context=context, + ) -# endregion # region Workflow extraction helpers @@ -1772,7 +1736,7 @@ async def evaluate_agent( raise ValueError(f"Got {len(query_list)} queries but {len(resp_list)} responses.") for q, r in zip(query_list, resp_list): items.append( - AgentEvalConverter.to_eval_item( + _to_eval_item( query=q, response=r, agent=agent, @@ -1792,7 +1756,7 @@ async def evaluate_agent( for query in queries: response = await agent.run([Message("user", [query])]) items.append( - AgentEvalConverter.to_eval_item( + _to_eval_item( query=query, response=response, agent=agent, @@ -1962,7 +1926,7 @@ async def evaluate_workflow( agent_items_by_id: dict[str, list[EvalItem]] = {} for executor_id, agent_data_list in agents_by_id.items(): agent_items_by_id[executor_id] = [ - AgentEvalConverter.to_eval_item( + _to_eval_item( query=ad["query"], response=ad["response"], agent=ad["agent"], @@ -2051,7 +2015,7 @@ def _build_overall_item( messages=[Message("assistant", [str(final_output)])] # type: ignore[reportUnknownArgumentType] ) - return AgentEvalConverter.to_eval_item(query=query, response=overall_response) + return _to_eval_item(query=query, response=overall_response) def _resolve_evaluators( diff --git a/python/packages/core/tests/core/test_evaluation.py b/python/packages/core/tests/core/test_evaluation.py new file mode 100644 index 00000000000..f32e216bd9b --- /dev/null +++ b/python/packages/core/tests/core/test_evaluation.py @@ -0,0 +1,368 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for provider-neutral evaluation item construction and splitting.""" + +from __future__ import annotations + +from typing import Any, cast +from unittest.mock import MagicMock + +import pytest + +from agent_framework import AgentEvalConverter as ExportedAgentEvalConverter +from agent_framework._evaluation import AgentEvalConverter, ConversationSplit, EvalItem, _to_eval_item +from agent_framework._tools import FunctionTool +from agent_framework._types import AgentResponse, Content, Message + + +class TestAgentEvalConverterCompatibility: + def test_root_export_is_legacy_converter(self) -> None: + assert ExportedAgentEvalConverter is AgentEvalConverter + + def test_convert_messages_preserves_legacy_foundry_wire_format(self) -> None: + messages = [ + Message("user", ["What's the weather?"]), + Message( + "assistant", + [Content.from_function_call(call_id="call_1", name="get_weather", arguments={"city": "Seattle"})], + ), + ] + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + converted = AgentEvalConverter.convert_messages(messages) + + assert converted == [ + {"role": "user", "content": [{"type": "text", "text": "What's the weather?"}]}, + { + "role": "assistant", + "content": [ + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "get_weather", + "arguments": {"city": "Seattle"}, + } + ], + }, + ] + + def test_to_eval_item_delegates_to_provider_neutral_builder(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["Sunny."])]) + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + item = AgentEvalConverter.to_eval_item(query="Weather?", response=response) + + assert item.query == "Weather?" + assert item.response == "Sunny." + + +class TestToEvalItem: + def test_string_query(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])]) + item = _to_eval_item(query="What's the weather?", response=response) + + assert item.query == "What's the weather?" + assert item.response == "The weather is sunny." + assert [message.role for message in item.conversation] == ["user", "assistant"] + + def test_message_query(self) -> None: + input_messages = [ + Message("system", ["Be helpful."]), + Message("user", ["Hello"]), + ] + response = AgentResponse(messages=[Message("assistant", ["Hi there!"])]) + + item = _to_eval_item(query=input_messages, response=response) + + assert item.query == "Hello" + assert len(item.conversation) == 3 + + def test_with_context(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["Answer."])]) + + item = _to_eval_item( + query="Question?", + response=response, + context="Some reference document.", + ) + + assert item.context == "Some reference document." + + def test_with_explicit_tools(self) -> None: + def search(query: str) -> str: + """Search the web.""" + return f"Results for {query}" + + response = AgentResponse(messages=[Message("assistant", ["Found it."])]) + + item = _to_eval_item(query="Find info", response=response, tools=[search]) + + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "search" + + def test_with_agent_and_mcp_tools(self) -> None: + agent_tool = FunctionTool(name="calculate", description="Calculate", func=lambda value: str(value)) + mcp_tool = FunctionTool(name="search", description="Search", func=lambda query: query) + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + agent.mcp_tools = [MagicMock(functions=[agent_tool, mcp_tool])] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Research this", response=response, agent=agent) + + assert item.tools == [agent_tool, mcp_tool] + + def test_explicit_tools_override_agent(self) -> None: + agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") + explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item( + query="Test", + response=response, + agent=agent, + tools=[explicit_tool], + ) + + assert item.tools == [explicit_tool] + + +class TestEvalItemSplitting: + def test_split_messages_format(self) -> None: + tool = FunctionTool(name="test", description="Test", func=lambda: "") + item = _to_eval_item( + query="Q", + response=AgentResponse(messages=[Message("assistant", ["Answer"])]), + tools=[tool], + ) + + query_messages, response_messages = item.split_messages() + + assert [message.role for message in query_messages] == ["user"] + assert [message.role for message in response_messages] == ["assistant"] + assert item.tools == [tool] + + def test_multiturn_preserves_interleaving(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", [Content(type="function_call", name="get_forecast")]), + Message("tool", [Content(type="function_result", result="Rain expected")]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages() + + assert [message.role for message in query_messages] == ["user", "assistant", "user"] + assert [message.role for message in response_messages] == ["assistant", "tool", "assistant"] + + def test_full_split(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, ConversationSplit.FULL) + ) + + assert [message.text for message in query_messages] == ["What's the weather?"] + assert [message.role for message in response_messages] == ["assistant", "user", "assistant"] + + def test_full_split_includes_system_message(self) -> None: + conversation = [ + Message("system", ["You are a weather assistant."]), + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny."]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, ConversationSplit.FULL) + ) + + assert [message.role for message in query_messages] == ["system", "user"] + assert [message.role for message in response_messages] == ["assistant"] + + def test_full_split_puts_tool_interactions_in_response(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="62°F")]), + Message("assistant", ["It's 62°F."]), + Message("user", ["Thanks!"]), + Message("assistant", ["You're welcome!"]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, ConversationSplit.FULL) + ) + + assert len(query_messages) == 1 + assert len(response_messages) == 5 + + def test_last_turn_is_default(self) -> None: + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there"]), + Message("user", ["Bye"]), + Message("assistant", ["Goodbye"]), + ] + item = EvalItem(conversation=conversation) + + default_query, default_response = item.split_messages() + explicit_query, explicit_response = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) + + assert default_query == explicit_query + assert default_response == explicit_response + + def test_per_turn_items(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain expected."]), + ] + + items = EvalItem.per_turn_items(conversation) + + assert len(items) == 2 + assert (items[0].query, items[0].response) == ("What's the weather?", "It's 62°F.") + assert (items[1].query, items[1].response) == ("What's the weather? And tomorrow?", "Rain expected.") + assert [len(item.conversation) for item in items] == [2, 4] + + def test_per_turn_items_preserve_tools(self) -> None: + conversation = [ + Message("user", ["Check weather"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="sunny")]), + Message("assistant", ["It's sunny."]), + Message("user", ["Thanks"]), + Message("assistant", ["You're welcome!"]), + ] + tool = FunctionTool(name="get_weather", description="Get weather") + + items = EvalItem.per_turn_items(conversation, tools=[tool]) + + assert len(items) == 2 + assert items[0].tools == [tool] + assert items[0].response == "It's sunny." + assert items[1].response == "You're welcome!" + + def test_per_turn_items_without_user_messages(self) -> None: + assert EvalItem.per_turn_items([Message("assistant", ["Hello"])]) == [] + + def test_per_turn_items_single_turn(self) -> None: + items = EvalItem.per_turn_items([ + Message("user", ["Hi"]), + Message("assistant", ["Hello!"]), + ]) + + assert len(items) == 1 + assert (items[0].query, items[0].response) == ("Hi", "Hello!") + + def test_custom_splitter_callable(self) -> None: + conversation = [ + Message("user", ["Remember my name is Alice"]), + Message("assistant", ["Got it, Alice!"]), + Message("user", ["What's the capital of France?"]), + Message("assistant", [Content(type="function_call", name="retrieve_memory", call_id="m1")]), + Message("tool", [Content(type="function_result", call_id="m1", result="User name: Alice")]), + Message("assistant", ["The capital of France is Paris, Alice!"]), + ] + + def split_before_memory(messages: list[Message]) -> tuple[list[Message], list[Message]]: + for index, message in enumerate(messages): + if any(content.name == "retrieve_memory" for content in message.contents): + return messages[:index], messages[index:] + return EvalItem._split_last_turn_static(messages) + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, split_before_memory) + ) + + assert len(query_messages) == 3 + assert query_messages[-1].role == "user" + assert len(response_messages) == 3 + assert response_messages[0].role == "assistant" + + def test_custom_splitter_fallback(self) -> None: + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there!"]), + ] + + def split_before_memory(messages: list[Message]) -> tuple[list[Message], list[Message]]: + for index, message in enumerate(messages): + if any(content.name == "retrieve_memory" for content in message.contents): + return messages[:index], messages[index:] + return EvalItem._split_last_turn_static(messages) + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, split_before_memory) + ) + + assert [message.role for message in query_messages] == ["user"] + assert [message.role for message in response_messages] == ["assistant"] + + def test_custom_splitter_lambda(self) -> None: + conversation = [ + Message("user", ["A"]), + Message("assistant", ["B"]), + Message("user", ["C"]), + Message("assistant", ["D"]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, lambda messages: (messages[:2], messages[2:])) + ) + + assert len(query_messages) == 2 + assert len(response_messages) == 2 + + def test_item_split_strategy_is_default(self) -> None: + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem(conversation=conversation, split_strategy=cast(Any, ConversationSplit.FULL)) + + query_messages, response_messages = item.split_messages() + + assert [message.text for message in query_messages] == ["First"] + assert len(response_messages) == 3 + + def test_explicit_split_overrides_item_strategy(self) -> None: + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem(conversation=conversation, split_strategy=cast(Any, ConversationSplit.FULL)) + + query_messages, response_messages = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) + + assert len(query_messages) == 3 + assert query_messages[-1].text == "Second" + assert len(response_messages) == 1 + + def test_no_split_defaults_to_last_turn(self) -> None: + item = EvalItem( + conversation=[ + Message("user", ["Hello"]), + Message("assistant", ["Hi"]), + ] + ) + + query_messages, _ = item.split_messages() + + assert item.split_strategy is None + assert [message.role for message in query_messages] == ["user"] diff --git a/python/packages/foundry/README.md b/python/packages/foundry/README.md index 74e37bbfb04..1bc0051fec0 100644 --- a/python/packages/foundry/README.md +++ b/python/packages/foundry/README.md @@ -2,6 +2,32 @@ This package contains the Microsoft Foundry integrations for Microsoft Agent Framework, including Foundry chat clients, preconfigured Foundry agents, Foundry embedding clients, and Foundry memory providers. +## Evaluations + +`FoundryEvals` implements the provider-neutral `Evaluator` protocol with +Microsoft Foundry's built-in and generated evaluators. Core owns `EvalItem`, +local evaluation, and the `evaluate_agent()` / `evaluate_workflow()` +orchestration functions; this package owns the Foundry Evals data mappings, +wire serialization, submission, polling, and result parsing. + +Use `evaluate_agent()` for the common run-and-evaluate path: + +```python +from agent_framework import evaluate_agent +from agent_framework.foundry import FoundryEvals + +results = await evaluate_agent( + agent=agent, + queries=["What's the weather in Seattle?"], + evaluators=FoundryEvals(), +) +``` + +For manual control, construct public `EvalItem` instances and pass them to +`FoundryEvals.evaluate()`. The Foundry wire format is private to this package. +`evaluate_traces()` and `evaluate_foundry_target()` provide Foundry-specific +entry points for existing traces, response IDs, and registered targets. + ## Concurrent reuse A `FoundryChatClient` instance can be shared by concurrent asynchronous calls on the same event loop. Streaming, diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index 25fd3bd817a..37dfb4a7a4a 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -3,8 +3,7 @@ """Microsoft Foundry Evals integration for Microsoft Agent Framework. Provides ``FoundryEvals``, an ``Evaluator`` implementation backed by Azure AI -Foundry's built-in evaluators. See docs/decisions/0018-foundry-evals-integration.md -for the design rationale. +Foundry's built-in evaluators. Example: @@ -27,13 +26,14 @@ from __future__ import annotations import asyncio +import contextlib +import json import logging from collections.abc import Iterable, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast from agent_framework._evaluation import ( - AgentEvalConverter, ConversationSplit, ConversationSplitter, EvalItem, @@ -44,6 +44,7 @@ ) from agent_framework._feature_stage import ExperimentalFeature, experimental from agent_framework._telemetry import mark_feature_used +from agent_framework._types import Message from openai import AsyncOpenAI from ._chat_client import FoundryChatClient @@ -217,6 +218,69 @@ def _resolve_evaluator(name: str) -> str: # --------------------------------------------------------------------------- +def _convert_message(message: Message) -> list[dict[str, Any]]: + """Convert one Agent Framework message to the Foundry Evals wire format.""" + content_items: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + + for content in message.contents or []: + if content.type == "text" and content.text: + content_items.append({"type": "text", "text": content.text}) + elif content.type in ("data", "uri") and content.uri: + image: dict[str, Any] = { + "type": "input_image", + "image_url": content.uri, + } + if content.media_type: + image["detail"] = "auto" + content_items.append(image) + elif content.type == "function_call": + arguments = content.arguments + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except (json.JSONDecodeError, TypeError): + arguments = {"_raw_arguments": "[unparseable]"} + content_items.append({ + "type": "tool_call", + "tool_call_id": content.call_id or "", + "name": content.name or "", + "arguments": arguments if arguments is not None else {}, + }) + elif content.type == "function_result": + result = content.result + if isinstance(result, str): + with contextlib.suppress(json.JSONDecodeError, TypeError): + result = json.loads(result) + tool_results.append({ + "call_id": content.call_id or "", + "result": result, + }) + + if tool_results: + return [ + { + "role": "tool", + "tool_call_id": tool_result["call_id"], + "content": [{"type": "tool_result", "tool_result": tool_result["result"]}], + } + for tool_result in tool_results + ] + if content_items: + return [{"role": message.role, "content": content_items}] + return [ + { + "role": message.role, + "content": [{"type": "text", "text": ""}], + } + ] + + +def _convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert Agent Framework messages to the Foundry Evals wire format.""" + return [converted for message in messages for converted in _convert_message(message)] + + def _build_testing_criteria( evaluators: Sequence[str | GeneratedEvaluatorRef], model: str, @@ -882,7 +946,7 @@ async def evaluate( evaluators and filters tool evaluators for items without tool definitions. Args: - items: Eval data items from ``AgentEvalConverter.to_eval_item()``. + items: Provider-neutral evaluation data items. eval_name: Display name for the evaluation run. Returns: @@ -919,8 +983,8 @@ async def _evaluate_via_dataset( d: dict[str, Any] = { "query": query_text, "response": response_text, - "query_messages": AgentEvalConverter.convert_messages(query_msgs), - "response_messages": AgentEvalConverter.convert_messages(response_msgs), + "query_messages": _convert_messages(query_msgs), + "response_messages": _convert_messages(response_msgs), } if item.tools: d["tool_definitions"] = [ diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py index f2b015e04c4..6832f27b7a2 100644 --- a/python/packages/foundry/tests/test_foundry_evals.py +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. -"""Tests for the AgentEvalConverter, FoundryEvals, and eval helper functions.""" +"""Tests for Foundry Evals wire conversion and evaluation helpers.""" from __future__ import annotations @@ -12,8 +12,6 @@ import pytest from agent_framework import AgentExecutorResponse, AgentResponse, Content, FunctionTool, Message, WorkflowEvent from agent_framework._evaluation import ( - AgentEvalConverter, - ConversationSplit, EvalItem, EvalNotPassedError, EvalResults, @@ -33,6 +31,8 @@ FoundryEvals, _build_item_schema, _build_testing_criteria, + _convert_message, + _convert_messages, _extract_per_evaluator, _extract_result_counts, _extract_rubric_scores, @@ -109,25 +109,25 @@ def test_unknown_raises(self) -> None: # --------------------------------------------------------------------------- -# AgentEvalConverter.convert_message +# Foundry Evals message conversion # --------------------------------------------------------------------------- class TestConvertMessage: def test_user_text_message(self) -> None: msg = Message("user", ["Hello, world!"]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0] == {"role": "user", "content": [{"type": "text", "text": "Hello, world!"}]} def test_system_message(self) -> None: msg = Message("system", ["You are helpful."]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert result[0] == {"role": "system", "content": [{"type": "text", "text": "You are helpful."}]} def test_assistant_text_message(self) -> None: msg = Message("assistant", ["Here is the answer."]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "assistant" assert result[0]["content"] == [{"type": "text", "text": "Here is the answer."}] @@ -144,7 +144,7 @@ def test_assistant_with_tool_call(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "assistant" tc = result[0]["content"][0] @@ -164,7 +164,7 @@ def test_assistant_with_zero_argument_tool_call(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) tc = result[0]["content"][0] assert tc["type"] == "tool_call" assert "arguments" in tc @@ -182,7 +182,7 @@ def test_assistant_text_and_tool_call(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} tc = result[0]["content"][1] @@ -199,7 +199,7 @@ def test_tool_result_message(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "tool" assert result[0]["tool_call_id"] == "call_1" @@ -213,7 +213,7 @@ def test_multiple_tool_results(self) -> None: Content.from_function_result(call_id="call_2", result="r2"), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 2 assert result[0]["tool_call_id"] == "call_1" assert result[1]["tool_call_id"] == "call_2" @@ -228,21 +228,21 @@ def test_non_string_result_kept_as_object(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) tr = result[0]["content"][0] assert tr["type"] == "tool_result" assert tr["tool_result"] == {"temp": 72, "unit": "F"} def test_empty_message(self) -> None: msg = Message("user", []) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert result[0] == {"role": "user", "content": [{"type": "text", "text": ""}]} def test_user_image_from_data(self) -> None: """Image created via Content.from_data() emits input_image.""" img = Content.from_data(data=b"\x89PNG\r\n\x1a\n", media_type="image/png") msg = Message("user", [img]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "user" part = result[0]["content"][0] @@ -254,7 +254,7 @@ def test_user_image_from_uri(self) -> None: """Image created via Content.from_uri() with an external URL.""" img = Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg") msg = Message("user", [img]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 part = result[0]["content"][0] assert part["type"] == "input_image" @@ -265,7 +265,7 @@ def test_user_image_uri_without_media_type(self) -> None: """URI content without media_type still emits input_image (no detail key).""" img = Content("uri", uri="https://example.com/pic.png") msg = Message("user", [img]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) part = result[0]["content"][0] assert part["type"] == "input_image" assert part["image_url"] == "https://example.com/pic.png" @@ -280,7 +280,7 @@ def test_mixed_text_and_image(self) -> None: Content.from_uri("https://example.com/cat.jpg", media_type="image/jpeg"), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert len(result[0]["content"]) == 2 assert result[0]["content"][0] == {"type": "text", "text": "What's in this image?"} @@ -289,7 +289,7 @@ def test_mixed_text_and_image(self) -> None: # --------------------------------------------------------------------------- -# AgentEvalConverter.convert_messages +# Foundry Evals conversation conversion # --------------------------------------------------------------------------- @@ -304,7 +304,7 @@ def test_full_conversation(self) -> None: Message("tool", [Content.from_function_result(call_id="c1", result="Sunny")]), Message("assistant", ["It's sunny in Seattle!"]), ] - result = AgentEvalConverter.convert_messages(messages) + result = _convert_messages(messages) assert len(result) == 4 assert result[0]["role"] == "user" assert result[1]["role"] == "assistant" @@ -327,7 +327,7 @@ def test_multimodal_conversation_preserves_images(self) -> None: ), Message("assistant", ["This is a photo of a sunset over the ocean."]), ] - result = AgentEvalConverter.convert_messages(messages) + result = _convert_messages(messages) assert len(result) == 2 # User message has text + image user_content = result[0]["content"] @@ -339,412 +339,6 @@ def test_multimodal_conversation_preserves_images(self) -> None: assert result[1]["content"] == [{"type": "text", "text": "This is a photo of a sunset over the ocean."}] -# --------------------------------------------------------------------------- -# AgentEvalConverter.extract_tools -# --------------------------------------------------------------------------- - - -class TestExtractTools: - def test_extracts_function_tools(self) -> None: - tool = FunctionTool( - name="get_weather", - description="Get weather for a location", - func=lambda location: f"Sunny in {location}", - ) - agent = MagicMock() - agent.default_options = {"tools": [tool]} - - result = AgentEvalConverter.extract_tools(agent) - assert len(result) == 1 - assert result[0]["name"] == "get_weather" - assert result[0]["description"] == "Get weather for a location" - assert "parameters" in result[0] - - def test_skips_non_function_tools(self) -> None: - agent = MagicMock() - agent.default_options = {"tools": [{"type": "web_search"}, "some_string"]} - - result = AgentEvalConverter.extract_tools(agent) - assert len(result) == 0 - - def test_no_tools(self) -> None: - agent = MagicMock() - agent.default_options = {} - assert AgentEvalConverter.extract_tools(agent) == [] - - def test_no_default_options(self) -> None: - agent = MagicMock(spec=[]) # No attributes - assert AgentEvalConverter.extract_tools(agent) == [] - - -# --------------------------------------------------------------------------- -# AgentEvalConverter.to_eval_item (now returns EvalItem) -# --------------------------------------------------------------------------- - - -class TestToEvalItem: - def test_string_query(self) -> None: - response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])]) - item = AgentEvalConverter.to_eval_item(query="What's the weather?", response=response) - - assert isinstance(item, EvalItem) - assert item.query == "What's the weather?" - assert item.response == "The weather is sunny." - assert len(item.conversation) == 2 - assert item.conversation[0].role == "user" - assert item.conversation[1].role == "assistant" - - def test_message_query(self) -> None: - input_msgs = [ - Message("system", ["Be helpful."]), - Message("user", ["Hello"]), - ] - response = AgentResponse(messages=[Message("assistant", ["Hi there!"])]) - item = AgentEvalConverter.to_eval_item(query=input_msgs, response=response) - - assert item.query == "Hello" # Only user messages - assert len(item.conversation) == 3 # system + user + assistant - - def test_with_context(self) -> None: - response = AgentResponse(messages=[Message("assistant", ["Answer."])]) - item = AgentEvalConverter.to_eval_item( - query="Question?", - response=response, - context="Some reference document.", - ) - assert item.context == "Some reference document." - - def test_with_explicit_tools(self) -> None: - tool = FunctionTool( - name="search", - description="Search the web", - func=lambda q: f"Results for {q}", - ) - response = AgentResponse(messages=[Message("assistant", ["Found it."])]) - item = AgentEvalConverter.to_eval_item( - query="Find info", - response=response, - tools=[tool], - ) - assert item.tools is not None - assert len(item.tools) == 1 - assert item.tools[0].name == "search" - - def test_with_agent_tools(self) -> None: - tool = FunctionTool(name="calc", description="Calculate", func=lambda x: str(x)) - agent = MagicMock() - agent.default_options = {"tools": [tool]} - - response = AgentResponse(messages=[Message("assistant", ["42"])]) - item = AgentEvalConverter.to_eval_item( - query="What is 6*7?", - response=response, - agent=agent, - ) - assert item.tools is not None - assert item.tools[0].name == "calc" - - def test_explicit_tools_override_agent(self) -> None: - agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") - explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") - - agent = MagicMock() - agent.default_options = {"tools": [agent_tool]} - - response = AgentResponse(messages=[Message("assistant", ["Done"])]) - item = AgentEvalConverter.to_eval_item( - query="Test", - response=response, - agent=agent, - tools=[explicit_tool], - ) - assert item.tools is not None - assert len(item.tools) == 1 - assert item.tools[0].name == "explicit_tool" - - def test_split_messages_format(self) -> None: - """split_messages() should split conversation at last user message.""" - response = AgentResponse(messages=[Message("assistant", ["Answer"])]) - item = AgentEvalConverter.to_eval_item( - query="Q", - response=response, - tools=[FunctionTool(name="t", description="d", func=lambda: "")], - ) - query_msgs, response_msgs = item.split_messages() - # Single-turn: query has just the user msg, response has the assistant msg - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - assert len(response_msgs) == 1 - assert response_msgs[0].role == "assistant" - # Tools preserved on item - assert item.tools is not None - assert len(item.tools) == 1 - assert item.tools[0].name == "t" - - def test_split_messages_multiturn_preserves_interleaving(self) -> None: - """Multi-turn split_messages() splits at last user message, preserving interleaving.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", ["It's sunny in Seattle."]), - Message("user", ["And tomorrow?"]), - Message("assistant", [Content(type="function_call", name="get_forecast")]), - Message("tool", [Content(type="function_result", result="Rain expected")]), - Message("assistant", ["Rain is expected tomorrow."]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages() - # query_messages: everything up to and including the last user message - assert len(query_msgs) == 3 # user, assistant, user - assert query_msgs[0].role == "user" - assert query_msgs[1].role == "assistant" # interleaved! - assert query_msgs[2].role == "user" - # response_messages: everything after the last user message - assert len(response_msgs) == 3 # assistant(tool_call), tool, assistant - assert response_msgs[0].role == "assistant" - assert response_msgs[1].role == "tool" - assert response_msgs[2].role == "assistant" - - def test_split_messages_full_split(self) -> None: - """ConversationSplit.FULL splits after the first user message.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", ["It's 62°F in Seattle."]), - Message("user", ["And tomorrow?"]), - Message("assistant", ["Rain is expected tomorrow."]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.FULL)) - # query_messages: just the first user message - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - assert query_msgs[0].text == "What's the weather?" - # response_messages: everything after the first user message - assert len(response_msgs) == 3 - assert response_msgs[0].role == "assistant" - assert response_msgs[1].role == "user" - assert response_msgs[2].role == "assistant" - - def test_split_messages_full_split_with_system(self) -> None: - """FULL split includes system messages before the first user message in query.""" - conversation = [ - Message("system", ["You are a weather assistant."]), - Message("user", ["What's the weather?"]), - Message("assistant", ["It's sunny."]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.FULL)) - # query includes system + first user - assert len(query_msgs) == 2 - assert query_msgs[0].role == "system" - assert query_msgs[1].role == "user" - assert len(response_msgs) == 1 - - def test_split_messages_full_split_with_tools(self) -> None: - """FULL split puts all tool interactions in response_messages.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", [Content(type="function_call", name="get_weather")]), - Message("tool", [Content(type="function_result", result="62°F")]), - Message("assistant", ["It's 62°F."]), - Message("user", ["Thanks!"]), - Message("assistant", ["You're welcome!"]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.FULL)) - assert len(query_msgs) == 1 - assert len(response_msgs) == 5 - - def test_split_messages_last_turn_is_default(self) -> None: - """Default split_messages() uses LAST_TURN split.""" - conversation = [ - Message("user", ["Hello"]), - Message("assistant", ["Hi there"]), - Message("user", ["Bye"]), - Message("assistant", ["Goodbye"]), - ] - item = EvalItem(conversation=conversation) - q_default, r_default = item.split_messages() - q_explicit, r_explicit = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) - assert [m.role for m in q_default] == [m.role for m in q_explicit] - assert [m.text for m in q_default] == [m.text for m in q_explicit] - assert [m.role for m in r_default] == [m.role for m in r_explicit] - assert [m.text for m in r_default] == [m.text for m in r_explicit] - - def test_per_turn_items_simple(self) -> None: - """per_turn_items produces one EvalItem per user message.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", ["It's 62°F."]), - Message("user", ["And tomorrow?"]), - Message("assistant", ["Rain expected."]), - ] - items = EvalItem.per_turn_items(conversation) - assert len(items) == 2 - - # Turn 1 - assert items[0].query == "What's the weather?" - assert items[0].response == "It's 62°F." - assert len(items[0].conversation) == 2 - - # Turn 2 — includes cumulative context; query joins all user texts in query split - assert items[1].query == "What's the weather? And tomorrow?" - assert items[1].response == "Rain expected." - assert len(items[1].conversation) == 4 - - def test_per_turn_items_with_tools(self) -> None: - """per_turn_items handles tool calls within a turn.""" - conversation = [ - Message("user", ["Check weather"]), - Message("assistant", [Content(type="function_call", name="get_weather")]), - Message("tool", [Content(type="function_result", result="sunny")]), - Message("assistant", ["It's sunny."]), - Message("user", ["Thanks"]), - Message("assistant", ["You're welcome!"]), - ] - tool_objs = [_make_tool("get_weather")] - items = EvalItem.per_turn_items(conversation, tools=cast(Any, tool_objs)) - assert len(items) == 2 - - # Turn 1: response includes tool_call, tool_result, and final assistant - assert items[0].response == "It's sunny." - assert items[0].tools == tool_objs - assert len(items[0].conversation) == 4 # user, assistant(tool), tool, assistant - - # Turn 2 - assert items[1].response == "You're welcome!" - assert len(items[1].conversation) == 6 # full conversation - - def test_per_turn_items_empty(self) -> None: - """per_turn_items returns empty list when no user messages.""" - items = EvalItem.per_turn_items([Message("assistant", ["Hello"])]) - assert items == [] - - def test_per_turn_items_single_turn(self) -> None: - """per_turn_items with single turn produces one item.""" - conversation = [ - Message("user", ["Hi"]), - Message("assistant", ["Hello!"]), - ] - items = EvalItem.per_turn_items(conversation) - assert len(items) == 1 - assert items[0].query == "Hi" - assert items[0].response == "Hello!" - - def test_custom_splitter_callable(self) -> None: - """Custom callable splitter is used by split_messages().""" - conversation = [ - Message("user", ["Remember my name is Alice"]), - Message("assistant", ["Got it, Alice!"]), - Message("user", ["What's the capital of France?"]), - Message("assistant", [Content(type="function_call", name="retrieve_memory", call_id="m1")]), - Message("tool", [Content(type="function_result", call_id="m1", result="User name: Alice")]), - Message("assistant", ["The capital of France is Paris, Alice!"]), - ] - - def split_before_memory(conversation): - """Split just before the memory retrieval tool call.""" - for i, msg in enumerate(conversation): - for c in msg.contents: - if c.name == "retrieve_memory": - return conversation[:i], conversation[i:] - return EvalItem._split_last_turn_static(conversation) - - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=split_before_memory) - - # split_before_memory finds "retrieve_memory" at conv[3] (assistant tool_call msg) - # query = conv[:3] = [user, assistant, user] - # response = conv[3:] = [assistant(tool_call), tool, assistant] - assert len(query_msgs) == 3 - assert query_msgs[-1].role == "user" - assert len(response_msgs) == 3 - assert response_msgs[0].role == "assistant" # the tool_call msg - - def test_custom_splitter_with_fallback(self) -> None: - """Custom splitter falls back to _split_last_turn_static when pattern not found.""" - conversation = [ - Message("user", ["Hello"]), - Message("assistant", ["Hi there!"]), - ] - - def split_before_memory(conversation): - for i, msg in enumerate(conversation): - for c in msg.contents: - if c.name == "retrieve_memory": - return conversation[:i], conversation[i:] - return EvalItem._split_last_turn_static(conversation) - - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=split_before_memory) - # Falls back to last-turn split - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - assert len(response_msgs) == 1 - assert response_msgs[0].role == "assistant" - - def test_custom_splitter_lambda(self) -> None: - """A lambda works as a custom splitter.""" - conversation = [ - Message("user", ["A"]), - Message("assistant", ["B"]), - Message("user", ["C"]), - Message("assistant", ["D"]), - ] - # Split at index 2 (arbitrary) - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=lambda conversation: (conversation[:2], conversation[2:])) - assert len(query_msgs) == 2 - assert len(response_msgs) == 2 - - def test_split_strategy_on_item_used_by_split_messages(self) -> None: - """split_strategy field on EvalItem is used as default by split_messages().""" - conversation = [ - Message("user", ["First"]), - Message("assistant", ["Response 1"]), - Message("user", ["Second"]), - Message("assistant", ["Response 2"]), - ] - item = EvalItem( - conversation=conversation, - split_strategy=cast(Any, ConversationSplit.FULL), - ) - # split_messages() with no split arg should use item.split_strategy - query_msgs, response_msgs = item.split_messages() - assert len(query_msgs) == 1 # FULL: just first user msg - assert query_msgs[0].text == "First" - assert len(response_msgs) == 3 - - def test_explicit_split_overrides_item_split_strategy(self) -> None: - """Explicit split= arg to split_messages() overrides item.split_strategy.""" - conversation = [ - Message("user", ["First"]), - Message("assistant", ["Response 1"]), - Message("user", ["Second"]), - Message("assistant", ["Response 2"]), - ] - item = EvalItem( - conversation=conversation, - split_strategy=cast(Any, ConversationSplit.FULL), - ) - # Explicit split= should override split_strategy - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) - assert len(query_msgs) == 3 # LAST_TURN: up to last user - assert query_msgs[-1].text == "Second" - assert len(response_msgs) == 1 - - def test_no_split_defaults_to_last_turn(self) -> None: - """When neither split= nor split_strategy is set, defaults to LAST_TURN.""" - conversation = [ - Message("user", ["Hello"]), - Message("assistant", ["Hi"]), - ] - item = EvalItem(conversation=conversation) - assert item.split_strategy is None - query_msgs, response_msgs = item.split_messages() - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - - # --------------------------------------------------------------------------- # _build_testing_criteria # --------------------------------------------------------------------------- diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/README.md b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md index c423d2b600b..cf33fa28cd0 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/README.md +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md @@ -15,15 +15,24 @@ These samples demonstrate evaluating agent-framework agents using Microsoft Foun ### `evaluate_agent_sample.py` — Dataset Evaluation (Path 3) -The dev inner loop. Two patterns from simplest to most control: - -1. **`evaluate_agent()`** — One call: runs agent → converts → evaluates -2. **`FoundryEvals.evaluate()`** — Run agent yourself, convert with `AgentEvalConverter`, inspect/modify, then evaluate +The dev inner loop. Pass existing responses or let `evaluate_agent()` run the +agent against test queries before submitting provider-neutral `EvalItem` data +to Foundry. ```bash uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py ``` +### `evaluate_tool_calls_sample.py` — Explicit Eval Items + +For more control, run the agent yourself, construct public `EvalItem` instances +from the conversation and typed tools, inspect or modify them, and pass them to +`FoundryEvals.evaluate()`. + +```bash +uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py +``` + ### `evaluate_traces_sample.py` — Trace & Response Evaluation (Path 1) Evaluate what already happened — zero changes to agent code: @@ -71,5 +80,5 @@ Create a `.env` file with configuration as in the `.env.example` file in this fo - **"I want to test my agent during development"** → `evaluate_agent_sample.py`, Pattern 1 - **"I want to evaluate past agent runs"** → `evaluate_traces_sample.py` -- **"I want to inspect/modify eval data before submitting"** → `evaluate_agent_sample.py`, Pattern 2 +- **"I want to inspect/modify eval data before submitting"** → `evaluate_tool_calls_sample.py` - **"I want to score against a custom rubric I created in Foundry"** → `evaluate_with_rubric_sample.py` diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py index d9fcf051ce5..019c0709d79 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py @@ -13,7 +13,7 @@ import asyncio import os -from agent_framework import Agent, AgentEvalConverter +from agent_framework import Agent, EvalItem, Message, tool from agent_framework.foundry import FoundryChatClient, FoundryEvals from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -21,6 +21,7 @@ load_dotenv() +@tool def get_weather(location: str) -> str: """Get the current weather for a location.""" weather_data = { @@ -31,6 +32,7 @@ def get_weather(location: str) -> str: return weather_data.get(location.lower(), f"Weather data not available for {location}") +@tool def get_flight_price(origin: str, destination: str) -> str: """Get the price of a flight between two cities.""" return f"Flights from {origin} to {destination}: $450 round-trip" @@ -65,7 +67,10 @@ async def main() -> None: print(f"Query: {q}") print(f"Response: {response.text[:100]}...") - item = AgentEvalConverter.to_eval_item(query=q, response=response, agent=agent) + item = EvalItem( + conversation=[Message("user", [q]), *response.messages], + tools=[get_weather, get_flight_price], + ) items.append(item) print(f" Has tools: {item.tools is not None}")