From f79a4097aa2dee973bae550945fee1a165fb1f89 Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sun, 23 Aug 2026 01:17:55 +0530 Subject: [PATCH 1/7] fix(orchestrations): preserve multimodal content during agent handoff --- .../_orchestrator_helpers.py | 32 +++++++++++++++---- .../orchestrations/tests/test_handoff.py | 19 +++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py index 1ad4d01e826..9d5cbf5674f 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py @@ -30,20 +30,40 @@ def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message] Returns: Cleaned conversation history with only text content, suitable for handoff routing """ + ALLOWED_CONTENT_TYPES = { + "text", + "text_reasoning", + "data", + "uri", + "hosted_file", + "hosted_vector_store", + } + cleaned: list[Message] = [] for msg in conversation: - # Keep only plain text history for handoff routing. Tool-control content + # Keep non-tool history for handoff routing. Tool-control content # (function_call/function_result/approval payloads) is runtime-only and # must not be replayed in future model turns. - text_parts = [content.text for content in msg.contents if content.type == "text" and content.text] - # TODO(@taochen): This is a simplified check that considers any non-text content as a tool call. - # We need to enhance this logic to specifically identify tool related contents. - if not text_parts: + retained_contents = [] + for content in msg.contents: + ctype = getattr(content, "type", "text") + + # Skip disallowed types (tools, usage, errors, etc.) + if ctype not in ALLOWED_CONTENT_TYPES: + continue + + # Skip empty text parts + if ctype == "text" and not getattr(content, "text", None): + continue + + retained_contents.append(content) + + if not retained_contents: continue msg_copy = Message( role=msg.role, - contents=[" ".join(text_parts)], + contents=retained_contents, author_name=msg.author_name, additional_properties=dict(msg.additional_properties) if msg.additional_properties else None, ) diff --git a/python/packages/orchestrations/tests/test_handoff.py b/python/packages/orchestrations/tests/test_handoff.py index 97b2a42e4fe..e6ba4f516ee 100644 --- a/python/packages/orchestrations/tests/test_handoff.py +++ b/python/packages/orchestrations/tests/test_handoff.py @@ -985,8 +985,8 @@ async def observe_properties(context: AgentContext, call_next): assert cloned_additional_properties is not coordinator.additional_properties -def test_clean_conversation_for_handoff_keeps_text_only_history() -> None: - """Tool-control messages must be excluded from persisted handoff history.""" +def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: + """Tool-control messages must be excluded, but text and multimodal data must be preserved.""" function_call = Content.from_function_call( call_id="handoff-call-1", name="handoff_to_refund_agent", @@ -997,9 +997,12 @@ def test_clean_conversation_for_handoff_keeps_text_only_history() -> None: id="approval-1", function_call=function_call, ) + + # Simulate a user attaching an image to their message + multimodal_content = Content(type="uri", uri="https://example.com/image.png", media_type="image/png") conversation = [ - Message(role="user", contents=["My order arrived damaged."]), + Message(role="user", contents=["My order arrived damaged.", multimodal_content]), Message( role="assistant", contents=[ @@ -1017,10 +1020,20 @@ def test_clean_conversation_for_handoff_keeps_text_only_history() -> None: cleaned = clean_conversation_for_handoff(conversation) assert [message.role for message in cleaned] == ["user", "assistant"] + + # Assert Text is preserved assert [message.text for message in cleaned] == [ "My order arrived damaged.", "Triage Agent: Routing you to Refund.", ] + + # Assert Multimodal URI is preserved in the first user message + user_contents = [c.type for c in cleaned[0].contents] + assert "uri" in user_contents + + # Assert Tool call is stripped from the assistant message + assistant_contents = [c.type for c in cleaned[1].contents] + assert "function_call" not in assistant_contents async def test_autonomous_mode_yields_output_without_user_request(): From a3a23c6d2c38134828b461be37f42b7013344465 Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sun, 23 Aug 2026 01:39:11 +0530 Subject: [PATCH 2/7] address PR feedback: fix docstring and strengthen test assertion --- .../_orchestrator_helpers.py | 2 +- python/packages/orchestrations/tests/test_handoff.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py index 9d5cbf5674f..dc81c6ef43e 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py @@ -28,7 +28,7 @@ def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message] Args: conversation: Full conversation history, including tool-control content Returns: - Cleaned conversation history with only text content, suitable for handoff routing + Cleaned conversation history with semantic multimodal content preserved, suitable for handoff routing """ ALLOWED_CONTENT_TYPES = { "text", diff --git a/python/packages/orchestrations/tests/test_handoff.py b/python/packages/orchestrations/tests/test_handoff.py index e6ba4f516ee..5fe66d0cb83 100644 --- a/python/packages/orchestrations/tests/test_handoff.py +++ b/python/packages/orchestrations/tests/test_handoff.py @@ -1028,8 +1028,11 @@ def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: ] # Assert Multimodal URI is preserved in the first user message - user_contents = [c.type for c in cleaned[0].contents] - assert "uri" in user_contents + user_contents = cleaned[0].contents + assert len(user_contents) == 2 + assert user_contents[0].type == "text" + assert user_contents[1].type == "uri" + assert user_contents[1].uri == "https://example.com/broken_product.jpg" # Assert Tool call is stripped from the assistant message assistant_contents = [c.type for c in cleaned[1].contents] From 8b7eb2298acba6e7aaef8025b9b730579c743584 Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sun, 23 Aug 2026 01:45:56 +0530 Subject: [PATCH 3/7] test: add SequentialBuilder regression test for multimodal content preservation --- .../orchestrations/tests/test_sequential.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/packages/orchestrations/tests/test_sequential.py b/python/packages/orchestrations/tests/test_sequential.py index 92dc7d409da..efdf3c3da53 100644 --- a/python/packages/orchestrations/tests/test_sequential.py +++ b/python/packages/orchestrations/tests/test_sequential.py @@ -246,6 +246,35 @@ async def test_sequential_checkpoint_resume_round_trip() -> None: assert baseline_text == resumed_text +async def test_sequential_builder_preserves_multimodal_content() -> None: + """Ensure that multimodal content (like URI) is preserved and passed down to subsequent agents in the sequence.""" + class _InspectorAgent(BaseAgent): + def run( + self, messages: AgentRunInputs | None = None, **kwargs: Any + ) -> Awaitable[AgentResponse[Any]] | ResponseStream[AgentResponseUpdate, AgentResponse[Any]]: + async def _run() -> AgentResponse: + uri_contents = [ + c.uri for m in (messages or []) for c in getattr(m, "contents", []) if getattr(c, "type", "") == "uri" + ] + return AgentResponse(messages=[Message("assistant", [f"Found URIs: {uri_contents}"])]) + return _run() + + a1 = _EchoAgent(id="agent1", name="A1") + inspector = _InspectorAgent(id="inspector", name="Inspector") + + wf = SequentialBuilder(participants=[a1, inspector]).build() + + multimodal_msg = Message("user", [Content(type="text", text="Look at this"), Content(type="uri", uri="https://example.com/image.png", media_type="image/png")]) + output_events = [ev for ev in await wf.run([multimodal_msg]) if ev.type == "output"] + + assert len(output_events) == 1 + response = output_events[0].data + assert isinstance(response, AgentResponse) + + combined = " ".join(m.text for m in response.messages) + assert "https://example.com/image.png" in combined + + async def test_sequential_checkpoint_runtime_only() -> None: """Test checkpointing configured ONLY at runtime, not at build time.""" storage = InMemoryCheckpointStorage() From ea2703dc325ad380213c1465be188c44716c754d Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sun, 23 Aug 2026 02:02:18 +0530 Subject: [PATCH 4/7] fix(orchestrations): remove text_reasoning from allowlist and fix URI assertion --- .../agent_framework_orchestrations/_orchestrator_helpers.py | 1 - python/packages/orchestrations/tests/test_handoff.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py index dc81c6ef43e..ad33cdaa57c 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py @@ -32,7 +32,6 @@ def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message] """ ALLOWED_CONTENT_TYPES = { "text", - "text_reasoning", "data", "uri", "hosted_file", diff --git a/python/packages/orchestrations/tests/test_handoff.py b/python/packages/orchestrations/tests/test_handoff.py index 5fe66d0cb83..34a5715c87b 100644 --- a/python/packages/orchestrations/tests/test_handoff.py +++ b/python/packages/orchestrations/tests/test_handoff.py @@ -1032,7 +1032,8 @@ def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: assert len(user_contents) == 2 assert user_contents[0].type == "text" assert user_contents[1].type == "uri" - assert user_contents[1].uri == "https://example.com/broken_product.jpg" + assert user_contents[1].uri == "https://example.com/image.png" + assert getattr(user_contents[1], "media_type", None) == "image/png" # Assert Tool call is stripped from the assistant message assistant_contents = [c.type for c in cleaned[1].contents] From 7b049be54a8d4aaab30c1b91f9038af35ca73ee5 Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sat, 5 Sep 2026 06:31:45 +0530 Subject: [PATCH 5/7] fix(orchestrations): retain multimodal content only for user messages in handoff --- .../_orchestrator_helpers.py | 37 +++++++++++-------- .../orchestrations/tests/test_handoff.py | 23 +++++++----- .../orchestrations/tests/test_sequential.py | 21 ++++++++--- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py index ad33cdaa57c..dab4d0049e8 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py @@ -14,23 +14,28 @@ def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message]: - """Keep only plain text chat history for handoff routing. + """Clean conversation history for handoff routing. Handoff executors must not replay prior tool-control artifacts (function calls, tool outputs, approval payloads) into future model turns, or providers may reject the next request due to unmatched tool-call state. - This helper builds a text-only copy of the conversation: - - Drops all non-text content from every message. - - Drops messages with no remaining text content. - - Preserves original roles and author names for retained text messages. + This helper preserves semantic content: + - For `user` messages, preserves text and multimodal content (data, uri, hosted_file, hosted_vector_store). + - For non-user messages (assistant, system, etc.), preserves only text content to avoid serializing input-only + multimodal parts into assistant roles on model providers. + - Drops tool-control payloads (function_call, function_result, approval payloads, etc.). + - Drops messages with no remaining content. + - Preserves original roles, author names, and additional properties for retained messages. Args: conversation: Full conversation history, including tool-control content + Returns: - Cleaned conversation history with semantic multimodal content preserved, suitable for handoff routing + Cleaned conversation history with semantic multimodal content preserved for user messages, + suitable for handoff routing. """ - ALLOWED_CONTENT_TYPES = { + USER_ALLOWED_CONTENT_TYPES = { "text", "data", "uri", @@ -40,23 +45,23 @@ def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message] cleaned: list[Message] = [] for msg in conversation: - # Keep non-tool history for handoff routing. Tool-control content - # (function_call/function_result/approval payloads) is runtime-only and - # must not be replayed in future model turns. + is_user = msg.role == "user" or str(msg.role).lower() == "user" + allowed_types = USER_ALLOWED_CONTENT_TYPES if is_user else {"text"} + retained_contents = [] for content in msg.contents: ctype = getattr(content, "type", "text") - - # Skip disallowed types (tools, usage, errors, etc.) - if ctype not in ALLOWED_CONTENT_TYPES: + + # Skip disallowed types + if ctype not in allowed_types: continue - + # Skip empty text parts if ctype == "text" and not getattr(content, "text", None): continue - + retained_contents.append(content) - + if not retained_contents: continue diff --git a/python/packages/orchestrations/tests/test_handoff.py b/python/packages/orchestrations/tests/test_handoff.py index 34a5715c87b..fd5deb0709e 100644 --- a/python/packages/orchestrations/tests/test_handoff.py +++ b/python/packages/orchestrations/tests/test_handoff.py @@ -986,7 +986,7 @@ async def observe_properties(context: AgentContext, call_next): def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: - """Tool-control messages must be excluded, but text and multimodal data must be preserved.""" + """Tool-control messages must be excluded, multimodal data preserved for user, and text-only for assistant.""" function_call = Content.from_function_call( call_id="handoff-call-1", name="handoff_to_refund_agent", @@ -997,17 +997,20 @@ def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: id="approval-1", function_call=function_call, ) - + # Simulate a user attaching an image to their message - multimodal_content = Content(type="uri", uri="https://example.com/image.png", media_type="image/png") + user_multimodal_content = Content(type="uri", uri="https://example.com/image.png", media_type="image/png") + # Simulate an assistant containing uri/data that should not be replayed as input-only items + assistant_multimodal_content = Content(type="uri", uri="https://example.com/output.png", media_type="image/png") conversation = [ - Message(role="user", contents=["My order arrived damaged.", multimodal_content]), + Message(role="user", contents=["My order arrived damaged.", user_multimodal_content]), Message( role="assistant", contents=[ function_call, Content.from_text(text="Triage Agent: Routing you to Refund."), + assistant_multimodal_content, ], ), Message(role="tool", contents=[Content.from_function_result(call_id="handoff-call-1", result="ok")]), @@ -1020,24 +1023,24 @@ def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: cleaned = clean_conversation_for_handoff(conversation) assert [message.role for message in cleaned] == ["user", "assistant"] - + # Assert Text is preserved assert [message.text for message in cleaned] == [ "My order arrived damaged.", "Triage Agent: Routing you to Refund.", ] - - # Assert Multimodal URI is preserved in the first user message + + # Assert Multimodal URI is preserved in the user message user_contents = cleaned[0].contents assert len(user_contents) == 2 assert user_contents[0].type == "text" assert user_contents[1].type == "uri" assert user_contents[1].uri == "https://example.com/image.png" assert getattr(user_contents[1], "media_type", None) == "image/png" - - # Assert Tool call is stripped from the assistant message + + # Assert Tool call and assistant multimodal contents are stripped from the assistant message assistant_contents = [c.type for c in cleaned[1].contents] - assert "function_call" not in assistant_contents + assert assistant_contents == ["text"] async def test_autonomous_mode_yields_output_without_user_request(): diff --git a/python/packages/orchestrations/tests/test_sequential.py b/python/packages/orchestrations/tests/test_sequential.py index efdf3c3da53..b4944f20d38 100644 --- a/python/packages/orchestrations/tests/test_sequential.py +++ b/python/packages/orchestrations/tests/test_sequential.py @@ -248,29 +248,40 @@ async def test_sequential_checkpoint_resume_round_trip() -> None: async def test_sequential_builder_preserves_multimodal_content() -> None: """Ensure that multimodal content (like URI) is preserved and passed down to subsequent agents in the sequence.""" + class _InspectorAgent(BaseAgent): def run( self, messages: AgentRunInputs | None = None, **kwargs: Any ) -> Awaitable[AgentResponse[Any]] | ResponseStream[AgentResponseUpdate, AgentResponse[Any]]: async def _run() -> AgentResponse: uri_contents = [ - c.uri for m in (messages or []) for c in getattr(m, "contents", []) if getattr(c, "type", "") == "uri" + c.uri + for m in (messages or []) + for c in getattr(m, "contents", []) + if getattr(c, "type", "") == "uri" ] return AgentResponse(messages=[Message("assistant", [f"Found URIs: {uri_contents}"])]) + return _run() a1 = _EchoAgent(id="agent1", name="A1") inspector = _InspectorAgent(id="inspector", name="Inspector") wf = SequentialBuilder(participants=[a1, inspector]).build() - - multimodal_msg = Message("user", [Content(type="text", text="Look at this"), Content(type="uri", uri="https://example.com/image.png", media_type="image/png")]) + + multimodal_msg = Message( + "user", + [ + Content(type="text", text="Look at this"), + Content(type="uri", uri="https://example.com/image.png", media_type="image/png"), + ], + ) output_events = [ev for ev in await wf.run([multimodal_msg]) if ev.type == "output"] - + assert len(output_events) == 1 response = output_events[0].data assert isinstance(response, AgentResponse) - + combined = " ".join(m.text for m in response.messages) assert "https://example.com/image.png" in combined From 784f201976e9bbd319326612fb00ca2141f9f7a6 Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sat, 5 Sep 2026 06:36:39 +0530 Subject: [PATCH 6/7] test: expand test_handoff to cover data, uri, file, and vector store multimodal types --- .../orchestrations/tests/test_handoff.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/python/packages/orchestrations/tests/test_handoff.py b/python/packages/orchestrations/tests/test_handoff.py index fd5deb0709e..cde1bbfa8e7 100644 --- a/python/packages/orchestrations/tests/test_handoff.py +++ b/python/packages/orchestrations/tests/test_handoff.py @@ -998,13 +998,26 @@ def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: function_call=function_call, ) - # Simulate a user attaching an image to their message - user_multimodal_content = Content(type="uri", uri="https://example.com/image.png", media_type="image/png") + # Simulate a user attaching multiple multimodal types to their message + uri_content = Content(type="uri", uri="https://example.com/image.png", media_type="image/png") + data_content = Content.from_data(data=b"fake-bytes", media_type="image/jpeg") + file_content = Content(type="hosted_file", file_id="file-123") + vector_content = Content(type="hosted_vector_store", vector_store_id="vs-456") + # Simulate an assistant containing uri/data that should not be replayed as input-only items assistant_multimodal_content = Content(type="uri", uri="https://example.com/output.png", media_type="image/png") conversation = [ - Message(role="user", contents=["My order arrived damaged.", user_multimodal_content]), + Message( + role="user", + contents=[ + "My order arrived damaged.", + uri_content, + data_content, + file_content, + vector_content, + ], + ), Message( role="assistant", contents=[ @@ -1030,13 +1043,14 @@ def test_clean_conversation_for_handoff_keeps_allowlist_history() -> None: "Triage Agent: Routing you to Refund.", ] - # Assert Multimodal URI is preserved in the user message + # Assert all Multimodal contents are preserved in the user message user_contents = cleaned[0].contents - assert len(user_contents) == 2 - assert user_contents[0].type == "text" - assert user_contents[1].type == "uri" + assert [c.type for c in user_contents] == ["text", "uri", "data", "hosted_file", "hosted_vector_store"] assert user_contents[1].uri == "https://example.com/image.png" - assert getattr(user_contents[1], "media_type", None) == "image/png" + assert user_contents[2].type == "data" + assert user_contents[2].uri is not None and user_contents[2].uri.startswith("data:image/jpeg;base64,") + assert getattr(user_contents[3], "file_id", None) == "file-123" + assert getattr(user_contents[4], "vector_store_id", None) == "vs-456" # Assert Tool call and assistant multimodal contents are stripped from the assistant message assistant_contents = [c.type for c in cleaned[1].contents] From 9df09bc2bc56823ee41658ef923ce20c2a1bc24d Mon Sep 17 00:00:00 2001 From: Manideep Malyala Date: Sat, 5 Sep 2026 06:57:27 +0530 Subject: [PATCH 7/7] fix(orchestrations): narrow type in sequential test and add type annotation in orchestrator helpers --- .../agent_framework_orchestrations/_orchestrator_helpers.py | 4 ++-- python/packages/orchestrations/tests/test_sequential.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py index dab4d0049e8..b560c8b298d 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py @@ -8,7 +8,7 @@ import logging -from agent_framework._types import Message +from agent_framework._types import Content, Message logger = logging.getLogger(__name__) @@ -48,7 +48,7 @@ def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message] is_user = msg.role == "user" or str(msg.role).lower() == "user" allowed_types = USER_ALLOWED_CONTENT_TYPES if is_user else {"text"} - retained_contents = [] + retained_contents: list[Content] = [] for content in msg.contents: ctype = getattr(content, "type", "text") diff --git a/python/packages/orchestrations/tests/test_sequential.py b/python/packages/orchestrations/tests/test_sequential.py index b4944f20d38..3c2475713e9 100644 --- a/python/packages/orchestrations/tests/test_sequential.py +++ b/python/packages/orchestrations/tests/test_sequential.py @@ -254,11 +254,9 @@ def run( self, messages: AgentRunInputs | None = None, **kwargs: Any ) -> Awaitable[AgentResponse[Any]] | ResponseStream[AgentResponseUpdate, AgentResponse[Any]]: async def _run() -> AgentResponse: + assert isinstance(messages, Sequence) and not isinstance(messages, str) uri_contents = [ - c.uri - for m in (messages or []) - for c in getattr(m, "contents", []) - if getattr(c, "type", "") == "uri" + c.uri for m in messages for c in getattr(m, "contents", []) if getattr(c, "type", "") == "uri" ] return AgentResponse(messages=[Message("assistant", [f"Found URIs: {uri_contents}"])])