From db135abfe85a5b02918947f76d861bec2f1efd2c Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 3 Sep 2026 12:08:48 +0200 Subject: [PATCH 1/5] Python: move Foundry eval serialization out of core Keep provider-neutral EvalItem construction in core while moving the Foundry Evals wire conversion into agent-framework-foundry. Remove the accidental experimental converter export and migrate its tests, sample, and current package documentation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/packages/core/AGENTS.md | 9 + .../packages/core/agent_framework/__init__.py | 2 - .../core/agent_framework/__init__.pyi | 2 - .../core/agent_framework/_evaluation.py | 233 ++------- .../core/tests/core/test_evaluation.py | 323 +++++++++++++ python/packages/foundry/README.md | 26 + .../agent_framework_foundry/_foundry_evals.py | 75 ++- .../foundry/tests/test_foundry_evals.py | 448 +----------------- .../evaluation/foundry_evals/README.md | 19 +- .../evaluate_tool_calls_sample.py | 22 +- 10 files changed, 514 insertions(+), 645 deletions(-) create mode 100644 python/packages/core/tests/core/test_evaluation.py diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index e515e70e2bb..3fd2e9a4f7a 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -210,6 +210,15 @@ agent_framework/ hosts remain responsible for authorizing and tenant-scoping access to any shared checkpoint adapter. - **Orchestrators**: `SequentialOrchestrator`, `ConcurrentOrchestrator`, `GroupChatOrchestrator`, `MagenticOrchestrator`, `HandoffOrchestrator` +## Evaluation (`_evaluation.py`) + +- Core owns provider-neutral evaluation types, local evaluators and checks, `EvalItem` construction, and the + `evaluate_agent` / `evaluate_workflow` orchestration functions. +- Provider packages own their service-specific evaluator implementations and wire serialization. Core evaluation + code must not emit a provider's request schema. +- Core orchestration builds `EvalItem` instances through private helpers; callers needing manual control construct + the public `EvalItem` directly. + ## Built-in Providers ### OpenAI (`openai/`) diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index 77c56fa7559..dde69e1075a 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -85,7 +85,6 @@ "included_token_count", ), "._evaluation": ( - "AgentEvalConverter", "CheckResult", "ConversationSplit", "ConversationSplitter", @@ -389,7 +388,6 @@ "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", "AgentContext", - "AgentEvalConverter", "AgentExecutor", "AgentExecutorRequest", "AgentExecutorResponse", diff --git a/python/packages/core/agent_framework/__init__.pyi b/python/packages/core/agent_framework/__init__.pyi index fa8f6a75ae6..2f64cdd9ada 100644 --- a/python/packages/core/agent_framework/__init__.pyi +++ b/python/packages/core/agent_framework/__init__.pyi @@ -48,7 +48,6 @@ from ._compaction import ( included_token_count, ) from ._evaluation import ( - AgentEvalConverter, CheckResult, ConversationSplit, ConversationSplitter, @@ -353,7 +352,6 @@ __all__ = [ "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", "AgentContext", - "AgentEvalConverter", "AgentExecutor", "AgentExecutorRequest", "AgentExecutorResponse", diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index ab71d84a15c..15c249b9049 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -35,7 +35,6 @@ from __future__ import annotations import asyncio -import contextlib import inspect import json import logging @@ -726,206 +725,38 @@ async def evaluate( # endregion -# region Converter - -@experimental(feature_id=ExperimentalFeature.EVALS) -class AgentEvalConverter: - """Converts agent-framework types to evaluation format. - - Handles the type gap between agent-framework's ``Message`` / ``Content`` / - ``FunctionTool`` types and the OpenAI-style agent message schema used by - evaluation providers. All methods are static — no instantiation needed. - """ - - @staticmethod - def convert_message(message: Message) -> list[dict[str, Any]]: - """Convert a single ``Message`` to Foundry agent evaluator format. - - Uses typed content lists as required by Foundry evaluators: - - .. code-block:: python - - {"role": "assistant", "content": [{"type": "tool_call", ...}]} - {"role": "user", "content": [{"type": "input_image", ...}]} - - Supported content types: - - * ``text`` → ``{"type": "text", "text": ...}`` - * ``data`` / ``uri`` (images) → ``{"type": "input_image", "image_url": ...}`` - * ``function_call`` → ``{"type": "tool_call", ...}`` - * ``function_result`` → ``{"type": "tool_result", ...}`` - - A single agent-framework ``Message`` with multiple ``function_result`` - contents produces multiple output messages (one per tool result). - - Args: - message: An agent-framework ``Message``. - - Returns: - A list of Foundry-format message dicts. - """ - role = message.role - contents = message.contents or [] - - content_items: list[dict[str, Any]] = [] - tool_results: list[dict[str, Any]] = [] - - for c in contents: - if c.type == "text" and c.text: - content_items.append({"type": "text", "text": c.text}) - elif c.type in ("data", "uri") and c.uri: - # Image / media content → OpenAI input_image format - img: dict[str, Any] = { - "type": "input_image", - "image_url": c.uri, - } - if c.media_type: - img["detail"] = "auto" - content_items.append(img) - elif c.type == "function_call": - args = c.arguments - if isinstance(args, str): - try: - args = json.loads(args) - except (json.JSONDecodeError, TypeError): - # Sanitize to avoid leaking sensitive tool-call arguments - # to external evaluation services. - args = {"_raw_arguments": "[unparseable]"} - tc: dict[str, Any] = { - "type": "tool_call", - "tool_call_id": c.call_id or "", - "name": c.name or "", - } - tc["arguments"] = args if args is not None else {} - content_items.append(tc) - elif c.type == "function_result": - result_val = c.result - if isinstance(result_val, str): - with contextlib.suppress(json.JSONDecodeError, TypeError): - result_val = json.loads(result_val) - tool_results.append({ - "call_id": c.call_id or "", - "result": result_val, - }) - - output: list[dict[str, Any]] = [] - - if tool_results: - for tr in tool_results: - output.append({ - "role": "tool", - "tool_call_id": tr["call_id"], - "content": [{"type": "tool_result", "tool_result": tr["result"]}], - }) - elif content_items: - output.append({"role": role, "content": content_items}) - else: - output.append({ - "role": role, - "content": [{"type": "text", "text": ""}], - }) - - return output - - @staticmethod - def convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: - """Convert a sequence of ``Message`` objects to Foundry evaluator format. - - Args: - messages: Agent-framework messages. - - Returns: - A list of Foundry-format message dicts with typed content lists. - """ - result: list[dict[str, Any]] = [] - for msg in messages: - result.extend(AgentEvalConverter.convert_message(msg)) - return result - - @staticmethod - def extract_tools(agent: Any) -> list[dict[str, Any]]: - """Extract tool definitions from an agent instance. - - Reads ``agent.default_options["tools"]`` and ``agent.mcp_tools`` - and converts each ``FunctionTool`` to ``{name, description, parameters}``. - - Args: - agent: An agent-framework agent instance. - - Returns: - A list of tool definition dicts. - """ - tools: list[dict[str, Any]] = [] - seen: set[str] = set() +def _to_eval_item( + *, + query: str | Sequence[Message], + response: AgentResponse[Any], + agent: Any | None = None, + tools: Sequence[FunctionTool] | None = None, + context: str | None = None, +) -> EvalItem: + """Build a provider-neutral ``EvalItem`` from an agent interaction.""" + input_msgs = [Message("user", [query])] if isinstance(query, str) else list(query) + all_msgs = list(input_msgs) + list(response.messages or []) + + typed_tools: list[FunctionTool] = [] + if tools: + typed_tools = list(tools) + elif agent: raw_tools = getattr(agent, "default_options", {}).get("tools", []) - for t in raw_tools: - if isinstance(t, FunctionTool) and t.name not in seen: - tools.append({ - "name": t.name, - "description": t.description, - "parameters": t.parameters(), - }) - seen.add(t.name) - # Include tools from connected MCP servers + typed_tools = [tool for tool in raw_tools if isinstance(tool, FunctionTool)] + seen = {tool.name for tool in typed_tools} for mcp in getattr(agent, "mcp_tools", []): - for t in getattr(mcp, "functions", []): - if isinstance(t, FunctionTool) and t.name not in seen: - tools.append({ - "name": t.name, - "description": t.description, - "parameters": t.parameters(), - }) - seen.add(t.name) - return tools - - @staticmethod - def to_eval_item( - *, - query: str | Sequence[Message], - response: AgentResponse[Any], - agent: Any | None = None, - tools: Sequence[FunctionTool] | None = None, - context: str | None = None, - ) -> EvalItem: - """Convert a complete agent interaction to an ``EvalItem``. - - Args: - query: The user query string, or input messages. - response: The agent's response. - agent: Optional agent instance to auto-extract tool definitions. - tools: Explicit tool list (takes precedence over *agent*). - context: Optional context document for groundedness evaluation. - - Returns: - An ``EvalItem`` suitable for passing to any ``Evaluator``. - """ - input_msgs = [Message("user", [query])] if isinstance(query, str) else list(query) - - all_msgs = list(input_msgs) + list(response.messages or []) - - typed_tools: list[FunctionTool] = [] - if tools: - typed_tools = list(tools) - elif agent: - raw_tools = getattr(agent, "default_options", {}).get("tools", []) - typed_tools = [t for t in raw_tools if isinstance(t, FunctionTool)] - # Include tools from connected MCP servers - seen = {t.name for t in typed_tools} - for mcp in getattr(agent, "mcp_tools", []): - for t in getattr(mcp, "functions", []): - if isinstance(t, FunctionTool) and t.name not in seen: - typed_tools.append(t) - seen.add(t.name) - - return EvalItem( - conversation=all_msgs, - tools=typed_tools or None, - context=context, - ) - + for tool in getattr(mcp, "functions", []): + if isinstance(tool, FunctionTool) and tool.name not in seen: + typed_tools.append(tool) + seen.add(tool.name) + + return EvalItem( + conversation=all_msgs, + tools=typed_tools or None, + context=context, + ) -# endregion # region Workflow extraction helpers @@ -1772,7 +1603,7 @@ async def evaluate_agent( raise ValueError(f"Got {len(query_list)} queries but {len(resp_list)} responses.") for q, r in zip(query_list, resp_list): items.append( - AgentEvalConverter.to_eval_item( + _to_eval_item( query=q, response=r, agent=agent, @@ -1792,7 +1623,7 @@ async def evaluate_agent( for query in queries: response = await agent.run([Message("user", [query])]) items.append( - AgentEvalConverter.to_eval_item( + _to_eval_item( query=query, response=response, agent=agent, @@ -1962,7 +1793,7 @@ async def evaluate_workflow( agent_items_by_id: dict[str, list[EvalItem]] = {} for executor_id, agent_data_list in agents_by_id.items(): agent_items_by_id[executor_id] = [ - AgentEvalConverter.to_eval_item( + _to_eval_item( query=ad["query"], response=ad["response"], agent=ad["agent"], @@ -2051,7 +1882,7 @@ def _build_overall_item( messages=[Message("assistant", [str(final_output)])] # type: ignore[reportUnknownArgumentType] ) - return AgentEvalConverter.to_eval_item(query=query, response=overall_response) + return _to_eval_item(query=query, response=overall_response) def _resolve_evaluators( diff --git a/python/packages/core/tests/core/test_evaluation.py b/python/packages/core/tests/core/test_evaluation.py new file mode 100644 index 00000000000..eda406c04eb --- /dev/null +++ b/python/packages/core/tests/core/test_evaluation.py @@ -0,0 +1,323 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for provider-neutral evaluation item construction and splitting.""" + +from __future__ import annotations + +from typing import Any, cast +from unittest.mock import MagicMock + +from agent_framework._evaluation import ConversationSplit, EvalItem, _to_eval_item +from agent_framework._tools import FunctionTool +from agent_framework._types import AgentResponse, Content, Message + + +class TestToEvalItem: + def test_string_query(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])]) + item = _to_eval_item(query="What's the weather?", response=response) + + assert item.query == "What's the weather?" + assert item.response == "The weather is sunny." + assert [message.role for message in item.conversation] == ["user", "assistant"] + + def test_message_query(self) -> None: + input_messages = [ + Message("system", ["Be helpful."]), + Message("user", ["Hello"]), + ] + response = AgentResponse(messages=[Message("assistant", ["Hi there!"])]) + + item = _to_eval_item(query=input_messages, response=response) + + assert item.query == "Hello" + assert len(item.conversation) == 3 + + def test_with_context(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["Answer."])]) + + item = _to_eval_item( + query="Question?", + response=response, + context="Some reference document.", + ) + + assert item.context == "Some reference document." + + def test_with_explicit_tools(self) -> None: + tool = FunctionTool( + name="search", + description="Search the web", + func=lambda query: f"Results for {query}", + ) + response = AgentResponse(messages=[Message("assistant", ["Found it."])]) + + item = _to_eval_item(query="Find info", response=response, tools=[tool]) + + assert item.tools == [tool] + + def test_with_agent_and_mcp_tools(self) -> None: + agent_tool = FunctionTool(name="calculate", description="Calculate", func=lambda value: str(value)) + mcp_tool = FunctionTool(name="search", description="Search", func=lambda query: query) + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + agent.mcp_tools = [MagicMock(functions=[agent_tool, mcp_tool])] + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item(query="Research this", response=response, agent=agent) + + assert item.tools == [agent_tool, mcp_tool] + + def test_explicit_tools_override_agent(self) -> None: + agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") + explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + + item = _to_eval_item( + query="Test", + response=response, + agent=agent, + tools=[explicit_tool], + ) + + assert item.tools == [explicit_tool] + + +class TestEvalItemSplitting: + def test_split_messages_format(self) -> None: + tool = FunctionTool(name="test", description="Test", func=lambda: "") + item = _to_eval_item( + query="Q", + response=AgentResponse(messages=[Message("assistant", ["Answer"])]), + tools=[tool], + ) + + query_messages, response_messages = item.split_messages() + + assert [message.role for message in query_messages] == ["user"] + assert [message.role for message in response_messages] == ["assistant"] + assert item.tools == [tool] + + def test_multiturn_preserves_interleaving(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", [Content(type="function_call", name="get_forecast")]), + Message("tool", [Content(type="function_result", result="Rain expected")]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages() + + assert [message.role for message in query_messages] == ["user", "assistant", "user"] + assert [message.role for message in response_messages] == ["assistant", "tool", "assistant"] + + def test_full_split(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, ConversationSplit.FULL) + ) + + assert [message.text for message in query_messages] == ["What's the weather?"] + assert [message.role for message in response_messages] == ["assistant", "user", "assistant"] + + def test_full_split_includes_system_message(self) -> None: + conversation = [ + Message("system", ["You are a weather assistant."]), + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny."]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, ConversationSplit.FULL) + ) + + assert [message.role for message in query_messages] == ["system", "user"] + assert [message.role for message in response_messages] == ["assistant"] + + def test_full_split_puts_tool_interactions_in_response(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="62°F")]), + Message("assistant", ["It's 62°F."]), + Message("user", ["Thanks!"]), + Message("assistant", ["You're welcome!"]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, ConversationSplit.FULL) + ) + + assert len(query_messages) == 1 + assert len(response_messages) == 5 + + def test_last_turn_is_default(self) -> None: + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there"]), + Message("user", ["Bye"]), + Message("assistant", ["Goodbye"]), + ] + item = EvalItem(conversation=conversation) + + default_query, default_response = item.split_messages() + explicit_query, explicit_response = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) + + assert default_query == explicit_query + assert default_response == explicit_response + + def test_per_turn_items(self) -> None: + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain expected."]), + ] + + items = EvalItem.per_turn_items(conversation) + + assert len(items) == 2 + assert (items[0].query, items[0].response) == ("What's the weather?", "It's 62°F.") + assert (items[1].query, items[1].response) == ("What's the weather? And tomorrow?", "Rain expected.") + assert [len(item.conversation) for item in items] == [2, 4] + + def test_per_turn_items_preserve_tools(self) -> None: + conversation = [ + Message("user", ["Check weather"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="sunny")]), + Message("assistant", ["It's sunny."]), + Message("user", ["Thanks"]), + Message("assistant", ["You're welcome!"]), + ] + tool = FunctionTool(name="get_weather", description="Get weather") + + items = EvalItem.per_turn_items(conversation, tools=[tool]) + + assert len(items) == 2 + assert items[0].tools == [tool] + assert items[0].response == "It's sunny." + assert items[1].response == "You're welcome!" + + def test_per_turn_items_without_user_messages(self) -> None: + assert EvalItem.per_turn_items([Message("assistant", ["Hello"])]) == [] + + def test_per_turn_items_single_turn(self) -> None: + items = EvalItem.per_turn_items([ + Message("user", ["Hi"]), + Message("assistant", ["Hello!"]), + ]) + + assert len(items) == 1 + assert (items[0].query, items[0].response) == ("Hi", "Hello!") + + def test_custom_splitter_callable(self) -> None: + conversation = [ + Message("user", ["Remember my name is Alice"]), + Message("assistant", ["Got it, Alice!"]), + Message("user", ["What's the capital of France?"]), + Message("assistant", [Content(type="function_call", name="retrieve_memory", call_id="m1")]), + Message("tool", [Content(type="function_result", call_id="m1", result="User name: Alice")]), + Message("assistant", ["The capital of France is Paris, Alice!"]), + ] + + def split_before_memory(messages: list[Message]) -> tuple[list[Message], list[Message]]: + for index, message in enumerate(messages): + if any(content.name == "retrieve_memory" for content in message.contents): + return messages[:index], messages[index:] + return EvalItem._split_last_turn_static(messages) + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, split_before_memory) + ) + + assert len(query_messages) == 3 + assert query_messages[-1].role == "user" + assert len(response_messages) == 3 + assert response_messages[0].role == "assistant" + + def test_custom_splitter_fallback(self) -> None: + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there!"]), + ] + + def split_before_memory(messages: list[Message]) -> tuple[list[Message], list[Message]]: + for index, message in enumerate(messages): + if any(content.name == "retrieve_memory" for content in message.contents): + return messages[:index], messages[index:] + return EvalItem._split_last_turn_static(messages) + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, split_before_memory) + ) + + assert [message.role for message in query_messages] == ["user"] + assert [message.role for message in response_messages] == ["assistant"] + + def test_custom_splitter_lambda(self) -> None: + conversation = [ + Message("user", ["A"]), + Message("assistant", ["B"]), + Message("user", ["C"]), + Message("assistant", ["D"]), + ] + + query_messages, response_messages = EvalItem(conversation=conversation).split_messages( + split=cast(Any, lambda messages: (messages[:2], messages[2:])) + ) + + assert len(query_messages) == 2 + assert len(response_messages) == 2 + + def test_item_split_strategy_is_default(self) -> None: + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem(conversation=conversation, split_strategy=cast(Any, ConversationSplit.FULL)) + + query_messages, response_messages = item.split_messages() + + assert [message.text for message in query_messages] == ["First"] + assert len(response_messages) == 3 + + def test_explicit_split_overrides_item_strategy(self) -> None: + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem(conversation=conversation, split_strategy=cast(Any, ConversationSplit.FULL)) + + query_messages, response_messages = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) + + assert len(query_messages) == 3 + assert query_messages[-1].text == "Second" + assert len(response_messages) == 1 + + def test_no_split_defaults_to_last_turn(self) -> None: + item = EvalItem( + conversation=[ + Message("user", ["Hello"]), + Message("assistant", ["Hi"]), + ] + ) + + query_messages, _ = item.split_messages() + + assert item.split_strategy is None + assert [message.role for message in query_messages] == ["user"] diff --git a/python/packages/foundry/README.md b/python/packages/foundry/README.md index ef2c0bba46e..c61c2996b3a 100644 --- a/python/packages/foundry/README.md +++ b/python/packages/foundry/README.md @@ -2,6 +2,32 @@ This package contains the Microsoft Foundry integrations for Microsoft Agent Framework, including Foundry chat clients, preconfigured Foundry agents, Foundry embedding clients, and Foundry memory providers. +## Evaluations + +`FoundryEvals` implements the provider-neutral `Evaluator` protocol with +Microsoft Foundry's built-in and generated evaluators. Core owns `EvalItem`, +local evaluation, and the `evaluate_agent()` / `evaluate_workflow()` +orchestration functions; this package owns the Foundry Evals data mappings, +wire serialization, submission, polling, and result parsing. + +Use `evaluate_agent()` for the common run-and-evaluate path: + +```python +from agent_framework import evaluate_agent +from agent_framework.foundry import FoundryEvals + +results = await evaluate_agent( + agent=agent, + queries=["What's the weather in Seattle?"], + evaluators=FoundryEvals(), +) +``` + +For manual control, construct public `EvalItem` instances and pass them to +`FoundryEvals.evaluate()`. The Foundry wire format is private to this package. +`evaluate_traces()` and `evaluate_foundry_target()` provide Foundry-specific +entry points for existing traces, response IDs, and registered targets. + ## Toolboxes A *toolbox* is a named, versioned bundle of hosted tool configurations — code interpreter, file search, image generation, MCP, web search, and so on — stored inside a Microsoft Foundry project. Toolboxes let you manage tool configuration once and reuse it across agents. diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index 25fd3bd817a..ba2e291933c 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -3,7 +3,7 @@ """Microsoft Foundry Evals integration for Microsoft Agent Framework. Provides ``FoundryEvals``, an ``Evaluator`` implementation backed by Azure AI -Foundry's built-in evaluators. See docs/decisions/0018-foundry-evals-integration.md +Foundry's built-in evaluators. See docs/decisions/0023-foundry-evals-integration.md for the design rationale. Example: @@ -27,13 +27,14 @@ from __future__ import annotations import asyncio +import contextlib +import json import logging from collections.abc import Iterable, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast from agent_framework._evaluation import ( - AgentEvalConverter, ConversationSplit, ConversationSplitter, EvalItem, @@ -44,6 +45,7 @@ ) from agent_framework._feature_stage import ExperimentalFeature, experimental from agent_framework._telemetry import mark_feature_used +from agent_framework._types import Message from openai import AsyncOpenAI from ._chat_client import FoundryChatClient @@ -217,6 +219,69 @@ def _resolve_evaluator(name: str) -> str: # --------------------------------------------------------------------------- +def _convert_message(message: Message) -> list[dict[str, Any]]: + """Convert one Agent Framework message to the Foundry Evals wire format.""" + content_items: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + + for content in message.contents or []: + if content.type == "text" and content.text: + content_items.append({"type": "text", "text": content.text}) + elif content.type in ("data", "uri") and content.uri: + image: dict[str, Any] = { + "type": "input_image", + "image_url": content.uri, + } + if content.media_type: + image["detail"] = "auto" + content_items.append(image) + elif content.type == "function_call": + arguments = content.arguments + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except (json.JSONDecodeError, TypeError): + arguments = {"_raw_arguments": "[unparseable]"} + content_items.append({ + "type": "tool_call", + "tool_call_id": content.call_id or "", + "name": content.name or "", + "arguments": arguments if arguments is not None else {}, + }) + elif content.type == "function_result": + result = content.result + if isinstance(result, str): + with contextlib.suppress(json.JSONDecodeError, TypeError): + result = json.loads(result) + tool_results.append({ + "call_id": content.call_id or "", + "result": result, + }) + + if tool_results: + return [ + { + "role": "tool", + "tool_call_id": tool_result["call_id"], + "content": [{"type": "tool_result", "tool_result": tool_result["result"]}], + } + for tool_result in tool_results + ] + if content_items: + return [{"role": message.role, "content": content_items}] + return [ + { + "role": message.role, + "content": [{"type": "text", "text": ""}], + } + ] + + +def _convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert Agent Framework messages to the Foundry Evals wire format.""" + return [converted for message in messages for converted in _convert_message(message)] + + def _build_testing_criteria( evaluators: Sequence[str | GeneratedEvaluatorRef], model: str, @@ -882,7 +947,7 @@ async def evaluate( evaluators and filters tool evaluators for items without tool definitions. Args: - items: Eval data items from ``AgentEvalConverter.to_eval_item()``. + items: Provider-neutral evaluation data items. eval_name: Display name for the evaluation run. Returns: @@ -919,8 +984,8 @@ async def _evaluate_via_dataset( d: dict[str, Any] = { "query": query_text, "response": response_text, - "query_messages": AgentEvalConverter.convert_messages(query_msgs), - "response_messages": AgentEvalConverter.convert_messages(response_msgs), + "query_messages": _convert_messages(query_msgs), + "response_messages": _convert_messages(response_msgs), } if item.tools: d["tool_definitions"] = [ diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py index f2b015e04c4..6832f27b7a2 100644 --- a/python/packages/foundry/tests/test_foundry_evals.py +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. -"""Tests for the AgentEvalConverter, FoundryEvals, and eval helper functions.""" +"""Tests for Foundry Evals wire conversion and evaluation helpers.""" from __future__ import annotations @@ -12,8 +12,6 @@ import pytest from agent_framework import AgentExecutorResponse, AgentResponse, Content, FunctionTool, Message, WorkflowEvent from agent_framework._evaluation import ( - AgentEvalConverter, - ConversationSplit, EvalItem, EvalNotPassedError, EvalResults, @@ -33,6 +31,8 @@ FoundryEvals, _build_item_schema, _build_testing_criteria, + _convert_message, + _convert_messages, _extract_per_evaluator, _extract_result_counts, _extract_rubric_scores, @@ -109,25 +109,25 @@ def test_unknown_raises(self) -> None: # --------------------------------------------------------------------------- -# AgentEvalConverter.convert_message +# Foundry Evals message conversion # --------------------------------------------------------------------------- class TestConvertMessage: def test_user_text_message(self) -> None: msg = Message("user", ["Hello, world!"]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0] == {"role": "user", "content": [{"type": "text", "text": "Hello, world!"}]} def test_system_message(self) -> None: msg = Message("system", ["You are helpful."]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert result[0] == {"role": "system", "content": [{"type": "text", "text": "You are helpful."}]} def test_assistant_text_message(self) -> None: msg = Message("assistant", ["Here is the answer."]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "assistant" assert result[0]["content"] == [{"type": "text", "text": "Here is the answer."}] @@ -144,7 +144,7 @@ def test_assistant_with_tool_call(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "assistant" tc = result[0]["content"][0] @@ -164,7 +164,7 @@ def test_assistant_with_zero_argument_tool_call(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) tc = result[0]["content"][0] assert tc["type"] == "tool_call" assert "arguments" in tc @@ -182,7 +182,7 @@ def test_assistant_text_and_tool_call(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} tc = result[0]["content"][1] @@ -199,7 +199,7 @@ def test_tool_result_message(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "tool" assert result[0]["tool_call_id"] == "call_1" @@ -213,7 +213,7 @@ def test_multiple_tool_results(self) -> None: Content.from_function_result(call_id="call_2", result="r2"), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 2 assert result[0]["tool_call_id"] == "call_1" assert result[1]["tool_call_id"] == "call_2" @@ -228,21 +228,21 @@ def test_non_string_result_kept_as_object(self) -> None: ), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) tr = result[0]["content"][0] assert tr["type"] == "tool_result" assert tr["tool_result"] == {"temp": 72, "unit": "F"} def test_empty_message(self) -> None: msg = Message("user", []) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert result[0] == {"role": "user", "content": [{"type": "text", "text": ""}]} def test_user_image_from_data(self) -> None: """Image created via Content.from_data() emits input_image.""" img = Content.from_data(data=b"\x89PNG\r\n\x1a\n", media_type="image/png") msg = Message("user", [img]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert result[0]["role"] == "user" part = result[0]["content"][0] @@ -254,7 +254,7 @@ def test_user_image_from_uri(self) -> None: """Image created via Content.from_uri() with an external URL.""" img = Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg") msg = Message("user", [img]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 part = result[0]["content"][0] assert part["type"] == "input_image" @@ -265,7 +265,7 @@ def test_user_image_uri_without_media_type(self) -> None: """URI content without media_type still emits input_image (no detail key).""" img = Content("uri", uri="https://example.com/pic.png") msg = Message("user", [img]) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) part = result[0]["content"][0] assert part["type"] == "input_image" assert part["image_url"] == "https://example.com/pic.png" @@ -280,7 +280,7 @@ def test_mixed_text_and_image(self) -> None: Content.from_uri("https://example.com/cat.jpg", media_type="image/jpeg"), ], ) - result = AgentEvalConverter.convert_message(msg) + result = _convert_message(msg) assert len(result) == 1 assert len(result[0]["content"]) == 2 assert result[0]["content"][0] == {"type": "text", "text": "What's in this image?"} @@ -289,7 +289,7 @@ def test_mixed_text_and_image(self) -> None: # --------------------------------------------------------------------------- -# AgentEvalConverter.convert_messages +# Foundry Evals conversation conversion # --------------------------------------------------------------------------- @@ -304,7 +304,7 @@ def test_full_conversation(self) -> None: Message("tool", [Content.from_function_result(call_id="c1", result="Sunny")]), Message("assistant", ["It's sunny in Seattle!"]), ] - result = AgentEvalConverter.convert_messages(messages) + result = _convert_messages(messages) assert len(result) == 4 assert result[0]["role"] == "user" assert result[1]["role"] == "assistant" @@ -327,7 +327,7 @@ def test_multimodal_conversation_preserves_images(self) -> None: ), Message("assistant", ["This is a photo of a sunset over the ocean."]), ] - result = AgentEvalConverter.convert_messages(messages) + result = _convert_messages(messages) assert len(result) == 2 # User message has text + image user_content = result[0]["content"] @@ -339,412 +339,6 @@ def test_multimodal_conversation_preserves_images(self) -> None: assert result[1]["content"] == [{"type": "text", "text": "This is a photo of a sunset over the ocean."}] -# --------------------------------------------------------------------------- -# AgentEvalConverter.extract_tools -# --------------------------------------------------------------------------- - - -class TestExtractTools: - def test_extracts_function_tools(self) -> None: - tool = FunctionTool( - name="get_weather", - description="Get weather for a location", - func=lambda location: f"Sunny in {location}", - ) - agent = MagicMock() - agent.default_options = {"tools": [tool]} - - result = AgentEvalConverter.extract_tools(agent) - assert len(result) == 1 - assert result[0]["name"] == "get_weather" - assert result[0]["description"] == "Get weather for a location" - assert "parameters" in result[0] - - def test_skips_non_function_tools(self) -> None: - agent = MagicMock() - agent.default_options = {"tools": [{"type": "web_search"}, "some_string"]} - - result = AgentEvalConverter.extract_tools(agent) - assert len(result) == 0 - - def test_no_tools(self) -> None: - agent = MagicMock() - agent.default_options = {} - assert AgentEvalConverter.extract_tools(agent) == [] - - def test_no_default_options(self) -> None: - agent = MagicMock(spec=[]) # No attributes - assert AgentEvalConverter.extract_tools(agent) == [] - - -# --------------------------------------------------------------------------- -# AgentEvalConverter.to_eval_item (now returns EvalItem) -# --------------------------------------------------------------------------- - - -class TestToEvalItem: - def test_string_query(self) -> None: - response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])]) - item = AgentEvalConverter.to_eval_item(query="What's the weather?", response=response) - - assert isinstance(item, EvalItem) - assert item.query == "What's the weather?" - assert item.response == "The weather is sunny." - assert len(item.conversation) == 2 - assert item.conversation[0].role == "user" - assert item.conversation[1].role == "assistant" - - def test_message_query(self) -> None: - input_msgs = [ - Message("system", ["Be helpful."]), - Message("user", ["Hello"]), - ] - response = AgentResponse(messages=[Message("assistant", ["Hi there!"])]) - item = AgentEvalConverter.to_eval_item(query=input_msgs, response=response) - - assert item.query == "Hello" # Only user messages - assert len(item.conversation) == 3 # system + user + assistant - - def test_with_context(self) -> None: - response = AgentResponse(messages=[Message("assistant", ["Answer."])]) - item = AgentEvalConverter.to_eval_item( - query="Question?", - response=response, - context="Some reference document.", - ) - assert item.context == "Some reference document." - - def test_with_explicit_tools(self) -> None: - tool = FunctionTool( - name="search", - description="Search the web", - func=lambda q: f"Results for {q}", - ) - response = AgentResponse(messages=[Message("assistant", ["Found it."])]) - item = AgentEvalConverter.to_eval_item( - query="Find info", - response=response, - tools=[tool], - ) - assert item.tools is not None - assert len(item.tools) == 1 - assert item.tools[0].name == "search" - - def test_with_agent_tools(self) -> None: - tool = FunctionTool(name="calc", description="Calculate", func=lambda x: str(x)) - agent = MagicMock() - agent.default_options = {"tools": [tool]} - - response = AgentResponse(messages=[Message("assistant", ["42"])]) - item = AgentEvalConverter.to_eval_item( - query="What is 6*7?", - response=response, - agent=agent, - ) - assert item.tools is not None - assert item.tools[0].name == "calc" - - def test_explicit_tools_override_agent(self) -> None: - agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") - explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") - - agent = MagicMock() - agent.default_options = {"tools": [agent_tool]} - - response = AgentResponse(messages=[Message("assistant", ["Done"])]) - item = AgentEvalConverter.to_eval_item( - query="Test", - response=response, - agent=agent, - tools=[explicit_tool], - ) - assert item.tools is not None - assert len(item.tools) == 1 - assert item.tools[0].name == "explicit_tool" - - def test_split_messages_format(self) -> None: - """split_messages() should split conversation at last user message.""" - response = AgentResponse(messages=[Message("assistant", ["Answer"])]) - item = AgentEvalConverter.to_eval_item( - query="Q", - response=response, - tools=[FunctionTool(name="t", description="d", func=lambda: "")], - ) - query_msgs, response_msgs = item.split_messages() - # Single-turn: query has just the user msg, response has the assistant msg - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - assert len(response_msgs) == 1 - assert response_msgs[0].role == "assistant" - # Tools preserved on item - assert item.tools is not None - assert len(item.tools) == 1 - assert item.tools[0].name == "t" - - def test_split_messages_multiturn_preserves_interleaving(self) -> None: - """Multi-turn split_messages() splits at last user message, preserving interleaving.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", ["It's sunny in Seattle."]), - Message("user", ["And tomorrow?"]), - Message("assistant", [Content(type="function_call", name="get_forecast")]), - Message("tool", [Content(type="function_result", result="Rain expected")]), - Message("assistant", ["Rain is expected tomorrow."]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages() - # query_messages: everything up to and including the last user message - assert len(query_msgs) == 3 # user, assistant, user - assert query_msgs[0].role == "user" - assert query_msgs[1].role == "assistant" # interleaved! - assert query_msgs[2].role == "user" - # response_messages: everything after the last user message - assert len(response_msgs) == 3 # assistant(tool_call), tool, assistant - assert response_msgs[0].role == "assistant" - assert response_msgs[1].role == "tool" - assert response_msgs[2].role == "assistant" - - def test_split_messages_full_split(self) -> None: - """ConversationSplit.FULL splits after the first user message.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", ["It's 62°F in Seattle."]), - Message("user", ["And tomorrow?"]), - Message("assistant", ["Rain is expected tomorrow."]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.FULL)) - # query_messages: just the first user message - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - assert query_msgs[0].text == "What's the weather?" - # response_messages: everything after the first user message - assert len(response_msgs) == 3 - assert response_msgs[0].role == "assistant" - assert response_msgs[1].role == "user" - assert response_msgs[2].role == "assistant" - - def test_split_messages_full_split_with_system(self) -> None: - """FULL split includes system messages before the first user message in query.""" - conversation = [ - Message("system", ["You are a weather assistant."]), - Message("user", ["What's the weather?"]), - Message("assistant", ["It's sunny."]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.FULL)) - # query includes system + first user - assert len(query_msgs) == 2 - assert query_msgs[0].role == "system" - assert query_msgs[1].role == "user" - assert len(response_msgs) == 1 - - def test_split_messages_full_split_with_tools(self) -> None: - """FULL split puts all tool interactions in response_messages.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", [Content(type="function_call", name="get_weather")]), - Message("tool", [Content(type="function_result", result="62°F")]), - Message("assistant", ["It's 62°F."]), - Message("user", ["Thanks!"]), - Message("assistant", ["You're welcome!"]), - ] - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.FULL)) - assert len(query_msgs) == 1 - assert len(response_msgs) == 5 - - def test_split_messages_last_turn_is_default(self) -> None: - """Default split_messages() uses LAST_TURN split.""" - conversation = [ - Message("user", ["Hello"]), - Message("assistant", ["Hi there"]), - Message("user", ["Bye"]), - Message("assistant", ["Goodbye"]), - ] - item = EvalItem(conversation=conversation) - q_default, r_default = item.split_messages() - q_explicit, r_explicit = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) - assert [m.role for m in q_default] == [m.role for m in q_explicit] - assert [m.text for m in q_default] == [m.text for m in q_explicit] - assert [m.role for m in r_default] == [m.role for m in r_explicit] - assert [m.text for m in r_default] == [m.text for m in r_explicit] - - def test_per_turn_items_simple(self) -> None: - """per_turn_items produces one EvalItem per user message.""" - conversation = [ - Message("user", ["What's the weather?"]), - Message("assistant", ["It's 62°F."]), - Message("user", ["And tomorrow?"]), - Message("assistant", ["Rain expected."]), - ] - items = EvalItem.per_turn_items(conversation) - assert len(items) == 2 - - # Turn 1 - assert items[0].query == "What's the weather?" - assert items[0].response == "It's 62°F." - assert len(items[0].conversation) == 2 - - # Turn 2 — includes cumulative context; query joins all user texts in query split - assert items[1].query == "What's the weather? And tomorrow?" - assert items[1].response == "Rain expected." - assert len(items[1].conversation) == 4 - - def test_per_turn_items_with_tools(self) -> None: - """per_turn_items handles tool calls within a turn.""" - conversation = [ - Message("user", ["Check weather"]), - Message("assistant", [Content(type="function_call", name="get_weather")]), - Message("tool", [Content(type="function_result", result="sunny")]), - Message("assistant", ["It's sunny."]), - Message("user", ["Thanks"]), - Message("assistant", ["You're welcome!"]), - ] - tool_objs = [_make_tool("get_weather")] - items = EvalItem.per_turn_items(conversation, tools=cast(Any, tool_objs)) - assert len(items) == 2 - - # Turn 1: response includes tool_call, tool_result, and final assistant - assert items[0].response == "It's sunny." - assert items[0].tools == tool_objs - assert len(items[0].conversation) == 4 # user, assistant(tool), tool, assistant - - # Turn 2 - assert items[1].response == "You're welcome!" - assert len(items[1].conversation) == 6 # full conversation - - def test_per_turn_items_empty(self) -> None: - """per_turn_items returns empty list when no user messages.""" - items = EvalItem.per_turn_items([Message("assistant", ["Hello"])]) - assert items == [] - - def test_per_turn_items_single_turn(self) -> None: - """per_turn_items with single turn produces one item.""" - conversation = [ - Message("user", ["Hi"]), - Message("assistant", ["Hello!"]), - ] - items = EvalItem.per_turn_items(conversation) - assert len(items) == 1 - assert items[0].query == "Hi" - assert items[0].response == "Hello!" - - def test_custom_splitter_callable(self) -> None: - """Custom callable splitter is used by split_messages().""" - conversation = [ - Message("user", ["Remember my name is Alice"]), - Message("assistant", ["Got it, Alice!"]), - Message("user", ["What's the capital of France?"]), - Message("assistant", [Content(type="function_call", name="retrieve_memory", call_id="m1")]), - Message("tool", [Content(type="function_result", call_id="m1", result="User name: Alice")]), - Message("assistant", ["The capital of France is Paris, Alice!"]), - ] - - def split_before_memory(conversation): - """Split just before the memory retrieval tool call.""" - for i, msg in enumerate(conversation): - for c in msg.contents: - if c.name == "retrieve_memory": - return conversation[:i], conversation[i:] - return EvalItem._split_last_turn_static(conversation) - - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=split_before_memory) - - # split_before_memory finds "retrieve_memory" at conv[3] (assistant tool_call msg) - # query = conv[:3] = [user, assistant, user] - # response = conv[3:] = [assistant(tool_call), tool, assistant] - assert len(query_msgs) == 3 - assert query_msgs[-1].role == "user" - assert len(response_msgs) == 3 - assert response_msgs[0].role == "assistant" # the tool_call msg - - def test_custom_splitter_with_fallback(self) -> None: - """Custom splitter falls back to _split_last_turn_static when pattern not found.""" - conversation = [ - Message("user", ["Hello"]), - Message("assistant", ["Hi there!"]), - ] - - def split_before_memory(conversation): - for i, msg in enumerate(conversation): - for c in msg.contents: - if c.name == "retrieve_memory": - return conversation[:i], conversation[i:] - return EvalItem._split_last_turn_static(conversation) - - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=split_before_memory) - # Falls back to last-turn split - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - assert len(response_msgs) == 1 - assert response_msgs[0].role == "assistant" - - def test_custom_splitter_lambda(self) -> None: - """A lambda works as a custom splitter.""" - conversation = [ - Message("user", ["A"]), - Message("assistant", ["B"]), - Message("user", ["C"]), - Message("assistant", ["D"]), - ] - # Split at index 2 (arbitrary) - item = EvalItem(conversation=conversation) - query_msgs, response_msgs = item.split_messages(split=lambda conversation: (conversation[:2], conversation[2:])) - assert len(query_msgs) == 2 - assert len(response_msgs) == 2 - - def test_split_strategy_on_item_used_by_split_messages(self) -> None: - """split_strategy field on EvalItem is used as default by split_messages().""" - conversation = [ - Message("user", ["First"]), - Message("assistant", ["Response 1"]), - Message("user", ["Second"]), - Message("assistant", ["Response 2"]), - ] - item = EvalItem( - conversation=conversation, - split_strategy=cast(Any, ConversationSplit.FULL), - ) - # split_messages() with no split arg should use item.split_strategy - query_msgs, response_msgs = item.split_messages() - assert len(query_msgs) == 1 # FULL: just first user msg - assert query_msgs[0].text == "First" - assert len(response_msgs) == 3 - - def test_explicit_split_overrides_item_split_strategy(self) -> None: - """Explicit split= arg to split_messages() overrides item.split_strategy.""" - conversation = [ - Message("user", ["First"]), - Message("assistant", ["Response 1"]), - Message("user", ["Second"]), - Message("assistant", ["Response 2"]), - ] - item = EvalItem( - conversation=conversation, - split_strategy=cast(Any, ConversationSplit.FULL), - ) - # Explicit split= should override split_strategy - query_msgs, response_msgs = item.split_messages(split=cast(Any, ConversationSplit.LAST_TURN)) - assert len(query_msgs) == 3 # LAST_TURN: up to last user - assert query_msgs[-1].text == "Second" - assert len(response_msgs) == 1 - - def test_no_split_defaults_to_last_turn(self) -> None: - """When neither split= nor split_strategy is set, defaults to LAST_TURN.""" - conversation = [ - Message("user", ["Hello"]), - Message("assistant", ["Hi"]), - ] - item = EvalItem(conversation=conversation) - assert item.split_strategy is None - query_msgs, response_msgs = item.split_messages() - assert len(query_msgs) == 1 - assert query_msgs[0].role == "user" - - # --------------------------------------------------------------------------- # _build_testing_criteria # --------------------------------------------------------------------------- diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/README.md b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md index c423d2b600b..cf33fa28cd0 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/README.md +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md @@ -15,15 +15,24 @@ These samples demonstrate evaluating agent-framework agents using Microsoft Foun ### `evaluate_agent_sample.py` — Dataset Evaluation (Path 3) -The dev inner loop. Two patterns from simplest to most control: - -1. **`evaluate_agent()`** — One call: runs agent → converts → evaluates -2. **`FoundryEvals.evaluate()`** — Run agent yourself, convert with `AgentEvalConverter`, inspect/modify, then evaluate +The dev inner loop. Pass existing responses or let `evaluate_agent()` run the +agent against test queries before submitting provider-neutral `EvalItem` data +to Foundry. ```bash uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py ``` +### `evaluate_tool_calls_sample.py` — Explicit Eval Items + +For more control, run the agent yourself, construct public `EvalItem` instances +from the conversation and typed tools, inspect or modify them, and pass them to +`FoundryEvals.evaluate()`. + +```bash +uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py +``` + ### `evaluate_traces_sample.py` — Trace & Response Evaluation (Path 1) Evaluate what already happened — zero changes to agent code: @@ -71,5 +80,5 @@ Create a `.env` file with configuration as in the `.env.example` file in this fo - **"I want to test my agent during development"** → `evaluate_agent_sample.py`, Pattern 1 - **"I want to evaluate past agent runs"** → `evaluate_traces_sample.py` -- **"I want to inspect/modify eval data before submitting"** → `evaluate_agent_sample.py`, Pattern 2 +- **"I want to inspect/modify eval data before submitting"** → `evaluate_tool_calls_sample.py` - **"I want to score against a custom rubric I created in Foundry"** → `evaluate_with_rubric_sample.py` diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py index d9fcf051ce5..bf21b15ba67 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py @@ -13,7 +13,7 @@ import asyncio import os -from agent_framework import Agent, AgentEvalConverter +from agent_framework import Agent, EvalItem, FunctionTool, Message from agent_framework.foundry import FoundryChatClient, FoundryEvals from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -43,6 +43,19 @@ async def main() -> None: credential=AzureCliCredential(), ) + tools = [ + FunctionTool( + name="get_weather", + description="Get the current weather for a location.", + func=get_weather, + ), + FunctionTool( + name="get_flight_price", + description="Get the price of a flight between two cities.", + func=get_flight_price, + ), + ] + # Create an agent with tools agent = Agent( client=chat_client, @@ -50,7 +63,7 @@ async def main() -> None: instructions=( "You are a helpful travel assistant. Use your tools to answer questions about weather and flights." ), - tools=[get_weather, get_flight_price], + tools=tools, ) # Run the agent and convert responses to eval items @@ -65,7 +78,10 @@ async def main() -> None: print(f"Query: {q}") print(f"Response: {response.text[:100]}...") - item = AgentEvalConverter.to_eval_item(query=q, response=response, agent=agent) + item = EvalItem( + conversation=[Message("user", [q]), *response.messages], + tools=tools, + ) items.append(item) print(f" Has tools: {item.tools is not None}") From 64de3c82b505b12e6c48fb90831f12f04f7908df Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 3 Sep 2026 12:10:21 +0200 Subject: [PATCH 2/5] Python: mark eval item helper experimental Preserve the EVALS feature-stage warning and metadata on the private EvalItem construction helper after removing AgentEvalConverter. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/packages/core/agent_framework/_evaluation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 15c249b9049..553ef20808e 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -726,6 +726,7 @@ async def evaluate( # endregion +@experimental(feature_id=ExperimentalFeature.EVALS) def _to_eval_item( *, query: str | Sequence[Message], From cbf67b72ebecc35aedda8ddbf39d0d0cb7591628 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 3 Sep 2026 12:11:44 +0200 Subject: [PATCH 3/5] Python: normalize tools for eval items Apply the standard tool normalization path when constructing EvalItems so callable tools become FunctionTool instances. Simplify the Foundry tool-call sample to use @tool-decorated functions directly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../core/agent_framework/_evaluation.py | 8 ++++---- .../packages/core/tests/core/test_evaluation.py | 15 ++++++++------- .../foundry_evals/evaluate_tool_calls_sample.py | 17 ++++------------- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 553ef20808e..47383906c63 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -53,7 +53,7 @@ ) from ._feature_stage import ExperimentalFeature, experimental -from ._tools import FunctionTool +from ._tools import FunctionTool, normalize_tools from ._types import AgentResponse, Message if TYPE_CHECKING: @@ -732,7 +732,7 @@ def _to_eval_item( query: str | Sequence[Message], response: AgentResponse[Any], agent: Any | None = None, - tools: Sequence[FunctionTool] | None = None, + tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, context: str | None = None, ) -> EvalItem: """Build a provider-neutral ``EvalItem`` from an agent interaction.""" @@ -741,10 +741,10 @@ def _to_eval_item( typed_tools: list[FunctionTool] = [] if tools: - typed_tools = list(tools) + typed_tools = [tool for tool in normalize_tools(tools) if isinstance(tool, FunctionTool)] elif agent: raw_tools = getattr(agent, "default_options", {}).get("tools", []) - typed_tools = [tool for tool in raw_tools if isinstance(tool, FunctionTool)] + typed_tools = [tool for tool in normalize_tools(raw_tools) if isinstance(tool, FunctionTool)] seen = {tool.name for tool in typed_tools} for mcp in getattr(agent, "mcp_tools", []): for tool in getattr(mcp, "functions", []): diff --git a/python/packages/core/tests/core/test_evaluation.py b/python/packages/core/tests/core/test_evaluation.py index eda406c04eb..c02ae55d4fe 100644 --- a/python/packages/core/tests/core/test_evaluation.py +++ b/python/packages/core/tests/core/test_evaluation.py @@ -45,16 +45,17 @@ def test_with_context(self) -> None: assert item.context == "Some reference document." def test_with_explicit_tools(self) -> None: - tool = FunctionTool( - name="search", - description="Search the web", - func=lambda query: f"Results for {query}", - ) + def search(query: str) -> str: + """Search the web.""" + return f"Results for {query}" + response = AgentResponse(messages=[Message("assistant", ["Found it."])]) - item = _to_eval_item(query="Find info", response=response, tools=[tool]) + item = _to_eval_item(query="Find info", response=response, tools=[search]) - assert item.tools == [tool] + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "search" def test_with_agent_and_mcp_tools(self) -> None: agent_tool = FunctionTool(name="calculate", description="Calculate", func=lambda value: str(value)) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py index bf21b15ba67..1d297e9adbf 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py @@ -13,7 +13,7 @@ import asyncio import os -from agent_framework import Agent, EvalItem, FunctionTool, Message +from agent_framework import Agent, EvalItem, Message, tool from agent_framework.foundry import FoundryChatClient, FoundryEvals from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -21,6 +21,7 @@ load_dotenv() +@tool def get_weather(location: str) -> str: """Get the current weather for a location.""" weather_data = { @@ -31,6 +32,7 @@ def get_weather(location: str) -> str: return weather_data.get(location.lower(), f"Weather data not available for {location}") +@tool def get_flight_price(origin: str, destination: str) -> str: """Get the price of a flight between two cities.""" return f"Flights from {origin} to {destination}: $450 round-trip" @@ -43,18 +45,7 @@ async def main() -> None: credential=AzureCliCredential(), ) - tools = [ - FunctionTool( - name="get_weather", - description="Get the current weather for a location.", - func=get_weather, - ), - FunctionTool( - name="get_flight_price", - description="Get the price of a flight between two cities.", - func=get_flight_price, - ), - ] + tools = [get_weather, get_flight_price] # Create an agent with tools agent = Agent( From 977ebfcbe38695ec7e4b3de641f5681a532647da Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 3 Sep 2026 14:37:04 +0200 Subject: [PATCH 4/5] Python: address Foundry eval review feedback Remove the stale accepted-ADR link from the Foundry Evals module and keep the tool-call sample's decorated tools directly in the agent and eval item definitions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../foundry/agent_framework_foundry/_foundry_evals.py | 3 +-- .../evaluation/foundry_evals/evaluate_tool_calls_sample.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index ba2e291933c..37dfb4a7a4a 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -3,8 +3,7 @@ """Microsoft Foundry Evals integration for Microsoft Agent Framework. Provides ``FoundryEvals``, an ``Evaluator`` implementation backed by Azure AI -Foundry's built-in evaluators. See docs/decisions/0023-foundry-evals-integration.md -for the design rationale. +Foundry's built-in evaluators. Example: diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py index 1d297e9adbf..019c0709d79 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py @@ -45,8 +45,6 @@ async def main() -> None: credential=AzureCliCredential(), ) - tools = [get_weather, get_flight_price] - # Create an agent with tools agent = Agent( client=chat_client, @@ -54,7 +52,7 @@ async def main() -> None: instructions=( "You are a helpful travel assistant. Use your tools to answer questions about weather and flights." ), - tools=tools, + tools=[get_weather, get_flight_price], ) # Run the agent and convert responses to eval items @@ -71,7 +69,7 @@ async def main() -> None: item = EvalItem( conversation=[Message("user", [q]), *response.messages], - tools=tools, + tools=[get_weather, get_flight_price], ) items.append(item) From 5cb3206370ebf1743e11f53660dae2c9e373b00f Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 4 Sep 2026 09:16:09 +0200 Subject: [PATCH 5/5] Python: deprecate AgentEvalConverter compatibly Keep the experimental converter import and static methods available so released Foundry packages remain compatible with their declared core 1.x range. Warn on legacy method use while modern Foundry continues to own active wire serialization. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/packages/core/AGENTS.md | 4 +- .../packages/core/agent_framework/__init__.py | 2 + .../core/agent_framework/__init__.pyi | 2 + .../core/agent_framework/_evaluation.py | 132 ++++++++++++++++++ .../core/tests/core/test_evaluation.py | 46 +++++- 5 files changed, 184 insertions(+), 2 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 64b1d4eccd8..288e07a2a6e 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -222,7 +222,9 @@ agent_framework/ - Core owns provider-neutral evaluation types, local evaluators and checks, `EvalItem` construction, and the `evaluate_agent` / `evaluate_workflow` orchestration functions. - Provider packages own their service-specific evaluator implementations and wire serialization. Core evaluation - code must not emit a provider's request schema. + code must not emit a provider's request schema. The deprecated `AgentEvalConverter` remains as a temporary + compatibility shim for released Foundry packages whose declared core range still imports it; new code must not use + the shim. - Core orchestration builds `EvalItem` instances through private helpers; callers needing manual control construct the public `EvalItem` directly. diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index dde69e1075a..77c56fa7559 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -85,6 +85,7 @@ "included_token_count", ), "._evaluation": ( + "AgentEvalConverter", "CheckResult", "ConversationSplit", "ConversationSplitter", @@ -388,6 +389,7 @@ "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", "AgentContext", + "AgentEvalConverter", "AgentExecutor", "AgentExecutorRequest", "AgentExecutorResponse", diff --git a/python/packages/core/agent_framework/__init__.pyi b/python/packages/core/agent_framework/__init__.pyi index 2f64cdd9ada..fa8f6a75ae6 100644 --- a/python/packages/core/agent_framework/__init__.pyi +++ b/python/packages/core/agent_framework/__init__.pyi @@ -48,6 +48,7 @@ from ._compaction import ( included_token_count, ) from ._evaluation import ( + AgentEvalConverter, CheckResult, ConversationSplit, ConversationSplitter, @@ -352,6 +353,7 @@ __all__ = [ "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", "AgentContext", + "AgentEvalConverter", "AgentExecutor", "AgentExecutorRequest", "AgentExecutorResponse", diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 47383906c63..327258b846b 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -35,9 +35,11 @@ from __future__ import annotations import asyncio +import contextlib import inspect import json import logging +import warnings from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from enum import Enum @@ -726,6 +728,136 @@ async def evaluate( # endregion +def _warn_agent_eval_converter_deprecated() -> None: + """Warn when the legacy evaluation converter compatibility surface is used.""" + warnings.warn( + "`AgentEvalConverter` is deprecated and will be removed in a future version. " + "Construct `EvalItem` directly or use `evaluate_agent()` / `evaluate_workflow()`; " + "Foundry wire conversion is internal to `agent-framework-foundry`.", + DeprecationWarning, + stacklevel=3, + ) + + +def _convert_legacy_foundry_message(message: Message) -> list[dict[str, Any]]: + """Preserve the legacy Foundry wire conversion for package compatibility.""" + content_items: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + + for content in message.contents or []: + if content.type == "text" and content.text: + content_items.append({"type": "text", "text": content.text}) + elif content.type in ("data", "uri") and content.uri: + image: dict[str, Any] = { + "type": "input_image", + "image_url": content.uri, + } + if content.media_type: + image["detail"] = "auto" + content_items.append(image) + elif content.type == "function_call": + arguments = content.arguments + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except (json.JSONDecodeError, TypeError): + arguments = {"_raw_arguments": "[unparseable]"} + content_items.append({ + "type": "tool_call", + "tool_call_id": content.call_id or "", + "name": content.name or "", + "arguments": arguments if arguments is not None else {}, + }) + elif content.type == "function_result": + result = content.result + if isinstance(result, str): + with contextlib.suppress(json.JSONDecodeError, TypeError): + result = json.loads(result) + tool_results.append({ + "call_id": content.call_id or "", + "result": result, + }) + + if tool_results: + return [ + { + "role": "tool", + "tool_call_id": tool_result["call_id"], + "content": [{"type": "tool_result", "tool_result": tool_result["result"]}], + } + for tool_result in tool_results + ] + if content_items: + return [{"role": message.role, "content": content_items}] + return [ + { + "role": message.role, + "content": [{"type": "text", "text": ""}], + } + ] + + +@experimental(feature_id=ExperimentalFeature.EVALS) +class AgentEvalConverter: + """Deprecated compatibility surface for earlier Agent Framework releases. + + New code should construct :class:`EvalItem` directly or use + :func:`evaluate_agent` / :func:`evaluate_workflow`. Foundry-specific wire + serialization is owned by ``agent-framework-foundry``. + """ + + @staticmethod + def convert_message(message: Message) -> list[dict[str, Any]]: + """Convert one message using the legacy Foundry evaluator wire format.""" + _warn_agent_eval_converter_deprecated() + return _convert_legacy_foundry_message(message) + + @staticmethod + def convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert messages using the legacy Foundry evaluator wire format.""" + _warn_agent_eval_converter_deprecated() + return [converted for message in messages for converted in _convert_legacy_foundry_message(message)] + + @staticmethod + def extract_tools(agent: Any) -> list[dict[str, Any]]: + """Extract legacy evaluator tool-definition dictionaries from an agent.""" + _warn_agent_eval_converter_deprecated() + tools: list[dict[str, Any]] = [] + seen: set[str] = set() + raw_tools = getattr(agent, "default_options", {}).get("tools", []) + for tool in raw_tools: + if isinstance(tool, FunctionTool) and tool.name not in seen: + tools.append({ + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters(), + }) + seen.add(tool.name) + for mcp in getattr(agent, "mcp_tools", []): + for tool in getattr(mcp, "functions", []): + if isinstance(tool, FunctionTool) and tool.name not in seen: + tools.append({ + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters(), + }) + seen.add(tool.name) + return tools + + @staticmethod + def to_eval_item( + *, + query: str | Sequence[Message], + response: AgentResponse[Any], + agent: Any | None = None, + tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, + context: str | None = None, + ) -> EvalItem: + """Build an ``EvalItem`` through the provider-neutral compatibility path.""" + _warn_agent_eval_converter_deprecated() + return _to_eval_item(query=query, response=response, agent=agent, tools=tools, context=context) + + @experimental(feature_id=ExperimentalFeature.EVALS) def _to_eval_item( *, diff --git a/python/packages/core/tests/core/test_evaluation.py b/python/packages/core/tests/core/test_evaluation.py index c02ae55d4fe..f32e216bd9b 100644 --- a/python/packages/core/tests/core/test_evaluation.py +++ b/python/packages/core/tests/core/test_evaluation.py @@ -7,11 +7,55 @@ from typing import Any, cast from unittest.mock import MagicMock -from agent_framework._evaluation import ConversationSplit, EvalItem, _to_eval_item +import pytest + +from agent_framework import AgentEvalConverter as ExportedAgentEvalConverter +from agent_framework._evaluation import AgentEvalConverter, ConversationSplit, EvalItem, _to_eval_item from agent_framework._tools import FunctionTool from agent_framework._types import AgentResponse, Content, Message +class TestAgentEvalConverterCompatibility: + def test_root_export_is_legacy_converter(self) -> None: + assert ExportedAgentEvalConverter is AgentEvalConverter + + def test_convert_messages_preserves_legacy_foundry_wire_format(self) -> None: + messages = [ + Message("user", ["What's the weather?"]), + Message( + "assistant", + [Content.from_function_call(call_id="call_1", name="get_weather", arguments={"city": "Seattle"})], + ), + ] + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + converted = AgentEvalConverter.convert_messages(messages) + + assert converted == [ + {"role": "user", "content": [{"type": "text", "text": "What's the weather?"}]}, + { + "role": "assistant", + "content": [ + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "get_weather", + "arguments": {"city": "Seattle"}, + } + ], + }, + ] + + def test_to_eval_item_delegates_to_provider_neutral_builder(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["Sunny."])]) + + with pytest.warns(DeprecationWarning, match="AgentEvalConverter"): + item = AgentEvalConverter.to_eval_item(query="Weather?", response=response) + + assert item.query == "Weather?" + assert item.response == "Sunny." + + class TestToEvalItem: def test_string_query(self) -> None: response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])])