Skip to content

Commit 7f44e73

Browse files
fix(triggers): wait for a slow executor up to its own timeout, not 30 s (FD-5333) (#544)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 35ff9e5 commit 7f44e73

2 files changed

Lines changed: 92 additions & 2 deletions

File tree

‎src/agent_env/env/gateway/triggers.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@
5959
_DROP = object() # an unresolved `${x?}` used as a whole value: drop the key / element
6060
# Real seconds between re-reads of the clock while waiting out virtual time, so a rate change, re-arm or clear is followed.
6161
_VIRTUAL_RECHECK_S = 5.0
62+
# Seconds an executor call waits to connect, and for a task poll's answer.
63+
_EXECUTOR_HTTP_TIMEOUT_S = 30.0
6264
# An allowlist, not a size bound: identifier/metadata keys only, since bodies and addresses are PII.
6365
_ECHO_KEYS = frozenset({
6466
"action", "createdTime", "fileName", "kind", "mimeType", "modifiedTime", "name", "role",
@@ -1576,8 +1578,10 @@ async def _call_executor(self, executor: dict | None, instruction: str, context_
15761578
return None
15771579
url = f"{executor['a2a_url']}/a2a"
15781580
deadline = asyncio.get_running_loop().time() + executor["timeout_seconds"]
1579-
async with httpx.AsyncClient(timeout=30) as client:
1580-
send = await client.post(url, json={
1581+
async with httpx.AsyncClient(timeout=_EXECUTOR_HTTP_TIMEOUT_S) as client:
1582+
# an executor holds message/send open until its task ends, so the read waits as long as the executor may take
1583+
read_s = max(_EXECUTOR_HTTP_TIMEOUT_S, executor["timeout_seconds"])
1584+
send = await client.post(url, timeout=httpx.Timeout(_EXECUTOR_HTTP_TIMEOUT_S, read=read_s), json={
15811585
"jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "message/send",
15821586
"params": {"message": {"role": "user", "messageId": uuid.uuid4().hex,
15831587
"contextId": context_id,

‎tst/unit/env/gateway/triggers_test.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
from __future__ import annotations
33

44
import asyncio
5+
import contextlib
56
import json
67
import random
8+
import re
79
from datetime import datetime, timedelta, timezone
810
from zoneinfo import ZoneInfo
911

@@ -1133,6 +1135,90 @@ def do_POST(self):
11331135
server.shutdown()
11341136

11351137

1138+
@contextlib.asynccontextmanager
1139+
async def _executor_answering_after(seconds: float):
1140+
"""A loopback executor whose message/send, like a deployed one's, answers only once its task ends `seconds` later;
1141+
its task polls answer at once. Yields its URL."""
1142+
from pytest_socket import enable_socket
1143+
1144+
enable_socket() # re-disabled by the autouse fixture's next setup
1145+
handlers: set[asyncio.Task] = set()
1146+
1147+
async def serve(reader, writer):
1148+
handlers.add(asyncio.current_task())
1149+
try:
1150+
while True: # one connection carries the send and then the polls
1151+
head = await reader.readuntil(b"\r\n\r\n")
1152+
req = json.loads(await reader.readexactly(int(re.search(rb"(?i)content-length: *(\d+)", head)[1])))
1153+
if req["method"] == "message/send":
1154+
await asyncio.sleep(seconds)
1155+
body = json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": {"id": "task-1", "status": {
1156+
"state": "completed", "message": {"parts": [{"kind": "text", "text": "replied"}]}}}}).encode()
1157+
writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s"
1158+
% (len(body), body))
1159+
await writer.drain()
1160+
except (asyncio.IncompleteReadError, ConnectionError):
1161+
pass # the caller hung up
1162+
finally:
1163+
writer.close()
1164+
1165+
server = await asyncio.start_server(serve, "127.0.0.1", 0)
1166+
try:
1167+
yield f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}"
1168+
finally:
1169+
server.close()
1170+
for task in handlers:
1171+
task.cancel()
1172+
await asyncio.gather(*handlers, return_exceptions=True)
1173+
1174+
1175+
def _reply_then_post(url: str, timeout_seconds: float) -> dict:
1176+
return {"executor": {"a2a_url": url, "role": "executor", "timeout_seconds": timeout_seconds},
1177+
"triggers": [{"id": "x", "when": {"type": "action", "tool": "t"},
1178+
"actions": [{"type": "nl", "instruction": "reply to the client"},
1179+
{"type": "tool", "tool": "slack_send_message",
1180+
"args": {"channel": "#c", "text": "the client replied"}}]}]}
1181+
1182+
1183+
async def _settled(engine, tid: str, within: float = 10) -> None:
1184+
for _ in range(int(within / 0.05)):
1185+
if engine._triggers[tid]["status"] not in ("firing", "queued"):
1186+
return
1187+
await asyncio.sleep(0.05)
1188+
raise AssertionError(f"trigger {tid!r} still {engine._triggers[tid]['status']} after {within}s")
1189+
1190+
1191+
@pytest.mark.asyncio
1192+
async def test_an_executor_slower_than_the_http_timeout_is_waited_out_and_the_later_actions_run(engine, monkeypatch):
1193+
"""FD-5333: a contact slower than 30 s used to fail its fire on a read timeout, though its reply still landed."""
1194+
monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) # stands in for the 30 s
1195+
async with _executor_answering_after(1.0) as url:
1196+
engine.register(_reply_then_post(url, timeout_seconds=10))
1197+
engine.on_tool_call("default", "t", {}, _result())
1198+
await asyncio.sleep(0.4)
1199+
assert engine._triggers["x"]["status"] == "firing" # past the HTTP timeout, still in flight
1200+
await _settled(engine, "x")
1201+
1202+
assert engine._triggers["x"]["status"] == "fired", engine.state()["events"][-3:]
1203+
assert [e["action_index"] for e in engine._events if e["kind"] == "action_ok"] == [0, 1]
1204+
assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})]
1205+
1206+
1207+
@pytest.mark.asyncio
1208+
async def test_an_executor_slower_than_its_own_timeout_still_fails_the_fire(engine, monkeypatch):
1209+
monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2)
1210+
async with _executor_answering_after(3.0) as url:
1211+
engine.register(_reply_then_post(url, timeout_seconds=1.0))
1212+
engine.on_tool_call("default", "t", {}, _result())
1213+
await asyncio.sleep(0.4)
1214+
assert engine._triggers["x"]["status"] == "firing"
1215+
await _settled(engine, "x", within=2.5)
1216+
1217+
[failure] = [e for e in engine._events if e["kind"] == "action_failed"]
1218+
assert (failure["action_index"], failure["detail"]) == (0, "ReadTimeout")
1219+
assert engine._triggers["x"]["status"] == "armed" and engine._test_gw.calls == []
1220+
1221+
11361222
@pytest.mark.asyncio
11371223
async def test_broken_check_is_fail_open(engine):
11381224
engine.register({"triggers": [

0 commit comments

Comments
 (0)