Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions src/agent_env/a2a_agent/trajectory_follower.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
_MAX_BACKOFF_SECONDS = 60
# Once stopped, the task has ended and its log is sealed, so the next read is normally the last.
_STOPPED_INTERVAL_SECONDS = 0.2
# After the task has ended, a failing read gets a short, bounded drain: enough to ride out a blip
# without losing the trajectory's tail, short enough that a gone agent frees the step.
_STOPPED_RETRY_SECONDS = 1.0
_STOPPED_MAX_FAILURES = 3
_ERROR_AFTER_FAILURES = 10


@dataclass(frozen=True)
Expand Down Expand Up @@ -59,6 +64,16 @@ def live_trajectory_endpoint(card: dict, a2a_url: str) -> str | None:
return a2a_url + path


def _log_read_failure(task_id: str, failures: int, exc: Exception) -> None:
logger.log(
logging.ERROR if failures >= _ERROR_AFTER_FAILURES else logging.WARNING,
"Live trajectory read failed (consec=%d, will retry): %r",
failures,
exc,
extra={"event": "live_trajectory_read_failed", "consec": failures, "task_id": task_id},
)


async def follow_trajectory(
endpoint: str,
task_id: str,
Expand All @@ -72,8 +87,8 @@ async def follow_trajectory(
events and the last one, until the task has ended and every event is read.

Between reads it waits ``poll_interval_seconds``, or less once ``stop`` is set. Transport errors
and 5xx answers are retried with backoff until ``stop`` is set, and end following after it; a
task the agent no longer knows (404) ends following."""
and 5xx answers are retried with backoff. Once ``stop`` is set they are retried a few more times,
then end following; a task the agent no longer knows (404) ends following."""
after = 0
failures = 0
async with nullcontext(client) if client is not None else httpx.AsyncClient() as http:
Expand All @@ -90,10 +105,10 @@ async def follow_trajectory(
if exc.response.status_code < 500:
raise
failures += 1
logger.warning("Live trajectory read failed (consec=%d, will retry): %s", failures, exc)
_log_read_failure(task_id, failures, exc)
except httpx.TransportError as exc:
failures += 1
logger.warning("Live trajectory read failed (consec=%d, will retry): %r", failures, exc)
_log_read_failure(task_id, failures, exc)
else:
failures = 0
page = parse_response(TrajectoryEventsResponse, response.json(), operation="trajectory get")
Expand All @@ -115,12 +130,14 @@ async def follow_trajectory(
if page.has_more:
continue
if stop.is_set():
if failures:
# The turn has ended, so an agent that stops answering now is gone, not busy: retrying
# would poll a dead address until the step's drain gives up. The step marks the turn's end.
logger.warning("Agent stopped answering after task %s ended; stopped following its trajectory", task_id)
if failures >= _STOPPED_MAX_FAILURES:
logger.warning(
"Agent stopped answering after task %s ended; stopped following its trajectory",
task_id,
extra={"event": "live_trajectory_follow_abandoned", "consec": failures, "task_id": task_id},
)
return
await asyncio.sleep(_STOPPED_INTERVAL_SECONDS)
await asyncio.sleep(_STOPPED_RETRY_SECONDS * failures or _STOPPED_INTERVAL_SECONDS)
continue
# Not `wait_for`: on 3.11 it drops a cancel that lands as `stop` is set, the step's own
# teardown, and the follower then outlives its step.
Expand Down
59 changes: 57 additions & 2 deletions tst/unit/a2a_agent/test_trajectory_follower.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
import contextlib
import json
import logging
import time

import httpx
Expand Down Expand Up @@ -167,7 +168,8 @@ async def test_a_follower_cancelled_as_it_is_stopped_ends():


@pytest.mark.asyncio
async def test_an_agent_that_fails_after_the_turn_ends_ends_following():
async def test_an_agent_that_fails_after_the_turn_ends_is_retried_a_bounded_number_of_times(monkeypatch):
monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._STOPPED_RETRY_SECONDS", 0.01)
agent = _Agent(httpx.Response(200, json=_page(0, [], "running")), *[httpx.Response(503)] * 100)
stop = asyncio.Event()
async with _following(agent, stop, poll_interval_seconds=30) as follower:
Expand All @@ -176,7 +178,60 @@ async def test_an_agent_that_fails_after_the_turn_ends_ends_following():
async with asyncio.timeout(2):
await follower

assert agent.afters == [0, 0]
assert agent.afters == [0] * 4


@pytest.mark.asyncio
async def test_an_agent_that_recovers_after_the_turn_ends_still_gets_its_tail_read(monkeypatch):
monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._STOPPED_RETRY_SECONDS", 0.01)
agent = _Agent(
httpx.Response(200, json=_page(0, [], "running")),
httpx.Response(503),
httpx.Response(200, json=_page(0, [{"i": 0}], "completed")),
)
batches: list[TrajectoryBatch] = []

async def sink(batch: TrajectoryBatch) -> None:
batches.append(batch)

stop = asyncio.Event()
async with agent.client() as client:
follower = asyncio.create_task(
follow_trajectory(_ENDPOINT, "task-1", sink, poll_interval_seconds=30, stop=stop, client=client)
)
try:
async with asyncio.timeout(5):
while not agent.afters:
await asyncio.sleep(0.01)
stop.set()
async with asyncio.timeout(2):
await follower
finally:
follower.cancel()
await asyncio.wait({follower})

assert batches[-1].events == [{"i": 0}]


@pytest.mark.asyncio
async def test_a_failing_read_is_logged_with_its_count_and_escalates_to_error(monkeypatch, caplog):
monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._MAX_BACKOFF_SECONDS", 0)
monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._ERROR_AFTER_FAILURES", 3)
agent = _Agent(
*[httpx.Response(503)] * 4,
httpx.Response(200, json=_page(0, [], "completed")),
)

with caplog.at_level(logging.WARNING, logger="agent_env.a2a_agent.trajectory_follower"):
await _follow(agent)

failed = [r for r in caplog.records if getattr(r, "event", None) == "live_trajectory_read_failed"]
assert [(r.consec, r.levelno, r.task_id) for r in failed] == [
(1, logging.WARNING, "task-1"),
(2, logging.WARNING, "task-1"),
(3, logging.ERROR, "task-1"),
(4, logging.ERROR, "task-1"),
]


@pytest.mark.asyncio
Expand Down
Loading