Skip to content

Commit bdec057

Browse files
fix(website): traffic collection makes every store write on a worker thread, so teardown's time limit holds (FD-4683)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 942a678 commit bdec057

2 files changed

Lines changed: 112 additions & 24 deletions

File tree

‎src/agent_env/task_step/task_steps/traffic_capture.py‎

Lines changed: 49 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
isn't lost with its VM, and saves what it recorded to the run's stored instance."""
77
from __future__ import annotations
88

9+
import asyncio
10+
import copy
911
import functools
1012
import logging
1113
import os
@@ -39,6 +41,7 @@
3941
#: ``context.metadata[TRAFFIC_KEY][env_id]``: each env whose captures a run collected, or tried to.
4042
TRAFFIC_KEY = "traffic_captures"
4143
_ARTIFACT_TYPE = "traffic_captures"
44+
_SAVING: set[asyncio.Future] = set()
4245
_ERROR_CHARS = 500
4346

4447

@@ -100,11 +103,31 @@ async def collect_uncollected_captures(context: TaskStepContext, envs: Sequence[
100103
if captures_traffic(deployed) and (entries.get(key) or {}).get("state") != "collected":
101104
tried = True
102105
await collect_captures(context, deployed, key=key)
103-
finally:
104-
if tried and context.instance_id and TRAFFIC_KEY in context.metadata:
105-
from agent_env.task.store import record_context_metadata
106+
except BaseException:
107+
if tried:
108+
_save_in_background(context)
109+
raise
110+
if tried:
111+
await _save(context)
112+
113+
114+
async def _save(context: TaskStepContext) -> None:
115+
"""Save the run's ``traffic_captures`` entries to its stored instance on a worker thread, which finishes the
116+
write even when teardown's time runs out while it waits."""
117+
if context.instance_id and TRAFFIC_KEY in context.metadata:
118+
from agent_env.task.store import record_context_metadata
106119

107-
record_context_metadata(context.instance_id, TRAFFIC_KEY, context.metadata[TRAFFIC_KEY])
120+
await finish_on_thread(functools.partial(record_context_metadata, context.instance_id, TRAFFIC_KEY,
121+
copy.deepcopy(context.metadata[TRAFFIC_KEY])),
122+
f"Saving the traffic captures of task instance {context.instance_id}")
123+
124+
125+
def _save_in_background(context: TaskStepContext) -> None:
126+
"""Start ``_save`` without waiting for it, as a collection teardown's time ran out on does, so the entries
127+
collected by then are saved while teardown goes on to terminate the sandboxes."""
128+
saving = asyncio.ensure_future(_save(context))
129+
_SAVING.add(saving)
130+
saving.add_done_callback(_SAVING.discard)
108131

109132

110133
def rolled_back_deployments(context: TaskStepContext) -> list[tuple[str, DeployedEnv]]:
@@ -122,9 +145,9 @@ async def collect_captures(context: TaskStepContext, deployed: DeployedEnv, *, k
122145
"""Store each capture ``deployed`` serves as a pcap FileArtifact, register them as one FileArtifactUniverse, and
123146
record it in ``context.metadata[TRAFFIC_KEY][key]``, by default its env_id. Each file is named for its container
124147
and its place among that container's files, oldest first, so they read in the order their packets were captured.
125-
Never raises: the entry, which it also returns, says what happened. A collection that stops partway, failed or
126-
cancelled, still registers the files it stored by then and records them as ``partial``, which teardown collects
127-
again."""
148+
Never raises: the entry, which it also returns, says what happened. Each file stored is registered in the universe
149+
as it lands, every store write on a worker thread, so a collection that stops partway, failed or cancelled, keeps
150+
the files it stored by then recorded as ``partial``, which teardown collects again."""
128151
key = key or deployed.env_id
129152
entries = context.metadata.setdefault(TRAFFIC_KEY, {})
130153
try:
@@ -147,7 +170,7 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
147170
get_config().check_local_run_write(universe_id)
148171
store = get_artifact_store()
149172
prefix = store.attempt_prefix(_ARTIFACT_TYPE, universe_id)
150-
files, total, seen = {}, 0, {}
173+
files, total, seen, artifact = {}, 0, {}, None
151174
try:
152175
for capture in captures:
153176
container = capture.container
@@ -164,26 +187,31 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
164187
url = await finish_on_thread(functools.partial(_upload, universe_id, prefix + filename, path),
165188
f"Uploading traffic capture {filename} of env {deployed.env_id!r}",
166189
if_never_run=functools.partial(_remove, path))
167-
files[filename] = store.put_document(FileArtifact(
190+
document = FileArtifact(
168191
id=derive_id(universe_id, filename), filename=filename, content_type=PCAP_MEDIA_TYPE, object_url=url,
169-
description=f"Traffic capture of {container} in env {deployed.env_id!r}, task instance {context.instance_id}"))
192+
description=f"Traffic capture of {container} in env {deployed.env_id!r}, task instance {context.instance_id}")
193+
files[filename] = await finish_on_thread(functools.partial(store.put_document, document),
194+
f"Storing traffic capture {filename} of env {deployed.env_id!r}")
195+
artifact = await finish_on_thread(functools.partial(_register, universe_id, dict(files)),
196+
f"Registering the traffic captures of env {deployed.env_id!r}")
197+
context.metadata[TRAFFIC_KEY][key] = _entry("partial", artifact, files, total, seen)
170198
except BaseException as e:
171-
if files:
172-
context.metadata[TRAFFIC_KEY][key] = {**_entry(universe_id, files, total, seen), "state": "partial",
199+
if artifact is not None:
200+
context.metadata[TRAFFIC_KEY][key] = {**_entry("partial", artifact, files, total, seen),
173201
"error": _failed(e)["error"]}
174202
raise
175203
logger.info("collect_traffic_capture %s: %d capture files, %d bytes", deployed.env_id, len(files), total)
176-
return _entry(universe_id, files, total, seen)
204+
return _entry("collected", artifact, files, total, seen)
205+
206+
207+
def _register(universe_id: str, files: dict) -> dict:
208+
"""Register ``files`` as the next version of the FileArtifactUniverse ``universe_id``."""
209+
universe = FileArtifactUniverse.put(id=universe_id, file_artifacts=files)
210+
return {"id": universe.id, "version": universe.version}
177211

178212

179-
def _entry(universe_id: str, files: dict, total: int, seen: dict) -> dict:
180-
"""The ``collected`` entry of ``files``, registered as one FileArtifactUniverse when there are any."""
181-
artifact = None
182-
if files:
183-
universe = FileArtifactUniverse.put(id=universe_id, file_artifacts=files)
184-
artifact = {"id": universe.id, "version": universe.version}
185-
return {"state": "collected", "artifact": artifact, "files": sorted(files), "bytes": total,
186-
"containers": sorted(seen)}
213+
def _entry(state: str, artifact: Optional[dict], files: dict, total: int, seen: dict) -> dict:
214+
return {"state": state, "artifact": artifact, "files": sorted(files), "bytes": total, "containers": sorted(seen)}
187215

188216

189217
def _upload(universe_id: str, object_url: str, path: str) -> str:

‎tst/unit/task_step/traffic_capture_test.py‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88
import asyncio
99
import os
1010
import threading
11+
import time
1112
from unittest.mock import AsyncMock, patch
1213

1314
import httpx
1415
import pytest
1516

16-
from agent_env.artifact import Artifact
17+
from agent_env.artifact import Artifact, FileArtifact
1718
from agent_env.artifact.store import get_artifact_store
1819
from agent_env.env.env import DeployedGatewayEnv
1920
from agent_env.env.gateway import gateway as gw_mod
@@ -217,6 +218,16 @@ async def test_what_teardown_collects_is_saved_to_the_runs_stored_instance(gatew
217218
assert stored == context.metadata[TRAFFIC_KEY] and stored["notes-env"]["state"] == "collected"
218219

219220

221+
async def _saved(instance_id: str, timeout_s: float = 10.0) -> dict:
222+
"""The run's saved ``traffic_captures`` entries, once a save teardown left running has landed."""
223+
deadline = asyncio.get_running_loop().time() + timeout_s
224+
while True:
225+
stored = find_task_instance(instance_id).context.get("metadata", {}).get(TRAFFIC_KEY)
226+
if stored is not None or asyncio.get_running_loop().time() > deadline:
227+
return stored
228+
await asyncio.sleep(0.05)
229+
230+
220231
@pytest.mark.asyncio
221232
async def test_what_teardown_collected_before_its_time_ran_out_is_still_saved_to_the_run(gateway, monkeypatch):
222233
instance = get_task_instance_store().create_instance("task-1", 1, total_steps=1)
@@ -234,7 +245,7 @@ async def collect(context, deployed, *, key=None):
234245
async with asyncio.timeout(1):
235246
await collect_uncollected_captures(context, context.deployed_envs)
236247

237-
stored = find_task_instance(instance.instance_id).context["metadata"][TRAFFIC_KEY]
248+
stored = await _saved(instance.instance_id)
238249
assert stored["notes-env"]["state"] == "collected" and "slow-env" not in stored
239250

240251

@@ -260,13 +271,62 @@ async def fetch(*args, **kwargs):
260271
assert entry["state"] == "partial" and entry["files"] == ["notes-website-backend-00.pcap"]
261272
universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"])
262273
assert sorted(universe.get_file_artifacts()) == ["notes-website-backend-00.pcap"]
263-
assert find_task_instance(instance.instance_id).context["metadata"][TRAFFIC_KEY]["notes-env"] == entry
274+
assert (await _saved(instance.instance_id))["notes-env"] == entry
264275

265276
monkeypatch.setattr(traffic_capture_module, "fetch_traffic_capture", real)
266277
await collect_uncollected_captures(context, context.deployed_envs)
267278
assert context.metadata[TRAFFIC_KEY]["notes-env"]["state"] == "collected"
268279

269280

281+
@pytest.mark.asyncio
282+
async def test_a_slow_document_store_doesnt_hold_teardowns_time_and_what_was_stored_still_lands(gateway, monkeypatch):
283+
instance = get_task_instance_store().create_instance("task-1", 1, total_steps=1)
284+
context = _context(_deployed())
285+
context.instance_id = instance.instance_id
286+
store = get_artifact_store()
287+
real, stored = store.put_document, []
288+
289+
def slow_put(document):
290+
if isinstance(document, FileArtifact):
291+
if stored:
292+
time.sleep(2)
293+
stored.append(document.id)
294+
return real(document)
295+
296+
monkeypatch.setattr(store, "put_document", slow_put)
297+
started = time.monotonic()
298+
with pytest.raises(TimeoutError):
299+
async with asyncio.timeout(0.5):
300+
await collect_uncollected_captures(context, context.deployed_envs)
301+
302+
assert time.monotonic() - started < 1.5
303+
entry = (await _saved(instance.instance_id))["notes-env"]
304+
assert entry["state"] == "partial" and entry["files"] == ["notes-website-backend-00.pcap"]
305+
306+
307+
@pytest.mark.asyncio
308+
async def test_a_slow_run_save_doesnt_hold_teardowns_time_and_still_lands(gateway, monkeypatch):
309+
from agent_env.task import store as task_store
310+
311+
instance = get_task_instance_store().create_instance("task-1", 1, total_steps=1)
312+
context = _context(_deployed())
313+
context.instance_id = instance.instance_id
314+
real = task_store.record_context_metadata
315+
316+
def slow_record(*args):
317+
time.sleep(2)
318+
real(*args)
319+
320+
monkeypatch.setattr(task_store, "record_context_metadata", slow_record)
321+
started = time.monotonic()
322+
with pytest.raises(TimeoutError):
323+
async with asyncio.timeout(1):
324+
await collect_uncollected_captures(context, context.deployed_envs)
325+
326+
assert time.monotonic() - started < 2
327+
assert (await _saved(instance.instance_id))["notes-env"]["state"] == "collected"
328+
329+
270330
@pytest.mark.asyncio
271331
async def test_tear_down_collects_a_retrys_rolled_back_deployment_under_its_own_key_before_its_vm_goes(gateway):
272332
rolled_back, live = _deployed(sandbox_id="vm-1"), _deployed(sandbox_id="vm-2")

0 commit comments

Comments
 (0)