Skip to content

Commit 73241f9

Browse files
fix(website): serve each capture as a whole copy, upload it under a prefix of its own, and capture loopback too (FD-4683)
- The gateway's traffic/v1 get copies a capture before sending any of it, copying again when tcpdump truncates the file to reuse it, and answers 503 when every copy was rewritten. - collect_traffic_capture writes each collection under its own attempt prefix, and the upload owns its temp file (finish_on_thread), so a cancelled collection never removes a file an upload still reads. - tcpdump captures every interface of the container, its loopback among them. - The read size and error limits are named. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent cdff317 commit 73241f9

6 files changed

Lines changed: 182 additions & 21 deletions

File tree

‎src/agent_env/env/gateway/gateway.py‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import re
1313
import socket
1414
import subprocess
15+
import tempfile
1516
from contextlib import asynccontextmanager
1617
from datetime import datetime, timezone
1718
from pathlib import Path
@@ -134,6 +135,8 @@ class Gateway:
134135
REST_PROXY_TIMEOUT_S = float(DATA_PLANE_LOAD_TIMEOUT_MAX_S) # data-plane loads
135136
REST_PROXY_CONNECT_TIMEOUT_S = 10.0 # a service that never answers the TCP connect fails in seconds, not an hour
136137
REST_PROXY_TIMEOUT = httpx.Timeout(REST_PROXY_TIMEOUT_S, connect=REST_PROXY_CONNECT_TIMEOUT_S)
138+
CAPTURE_READ_BYTES = 64 * 1024
139+
CAPTURE_SNAPSHOT_ATTEMPTS = 3
137140

138141
def __init__(
139142
self,
@@ -432,7 +435,8 @@ async def traffic_captures(request: Request) -> Response:
432435

433436
@self._mcp.custom_route("/traffic/capture", methods=["GET"])
434437
async def traffic_capture(request: Request) -> Response:
435-
return self._refuse_uncontrolled(request) or self._serve_capture(request.query_params.get("name", ""))
438+
return self._refuse_uncontrolled(request) or await asyncio.to_thread(
439+
self._serve_capture, request.query_params.get("name", ""))
436440

437441
@self._mcp.custom_route("/changes", methods=["GET"])
438442
async def env_changes(request: Request) -> Response:
@@ -1630,8 +1634,10 @@ def _serve_captures(self) -> Response:
16301634
return self._json_response({"format": "pcap", "captures": captures}, 200)
16311635

16321636
def _serve_capture(self, name: str) -> Response:
1633-
"""traffic/v1's get: the capture file ``name``, one ``list`` names, streamed as it stands; tcpdump writes each
1634-
packet as it captures it, so a file still being written ends on a whole packet, or at worst one torn."""
1637+
"""traffic/v1's get: the capture file ``name``, one ``list`` names, as a copy taken whole before any of it is
1638+
sent (``_snapshot_capture``), so a file tcpdump's rotation rewrites while it downloads never arrives mixed or
1639+
cut short; 503 when every copy was rewritten under it. A file still being written ends on a whole packet, or at
1640+
worst one torn."""
16351641
if not _CAPTURE_FILE.fullmatch(name):
16361642
return self._json_response({"ok": False, "error": {"code": "invalid_params",
16371643
"message": f"{name!r} is not a capture's name, as list gives it"}}, 400)
@@ -1643,14 +1649,42 @@ def _serve_capture(self, name: str) -> Response:
16431649
except OSError:
16441650
return self._json_response({"ok": False, "error": {"code": "not_found",
16451651
"message": f"no capture named {name!r}"}}, 404)
1652+
with handle:
1653+
snapshot = self._snapshot_capture(handle)
1654+
if snapshot is None:
1655+
return self._json_response({"ok": False, "error": {"code": "unavailable",
1656+
"message": f"capture {name!r} kept being rewritten while it was copied; "
1657+
"ask again"}}, 503)
1658+
size = os.fstat(snapshot.fileno()).st_size
16461659

16471660
def chunks():
1648-
with handle:
1649-
while chunk := handle.read(65536):
1661+
with snapshot:
1662+
while chunk := snapshot.read(self.CAPTURE_READ_BYTES):
16501663
yield chunk
16511664

16521665
return StreamingResponse(chunks(), media_type=PCAP_MEDIA_TYPE,
1653-
headers={"Content-Disposition": f'attachment; filename="{name}"'})
1666+
headers={"Content-Disposition": f'attachment; filename="{name}"',
1667+
"Content-Length": str(size)})
1668+
1669+
def _snapshot_capture(self, handle: Any) -> Any | None:
1670+
"""A copy of the open capture ``handle`` in an unlinked temp file, read back from its start, or None when each
1671+
of ``CAPTURE_SNAPSHOT_ATTEMPTS`` copies found the file shorter than what it had copied: tcpdump truncated it to
1672+
reuse it, so the copy would mix two rounds of packets. The copy is gone once it's closed."""
1673+
for _ in range(self.CAPTURE_SNAPSHOT_ATTEMPTS):
1674+
handle.seek(0)
1675+
copy = tempfile.TemporaryFile()
1676+
copied, rewritten = 0, False
1677+
while chunk := handle.read(self.CAPTURE_READ_BYTES):
1678+
copy.write(chunk)
1679+
copied += len(chunk)
1680+
if os.fstat(handle.fileno()).st_size < copied:
1681+
rewritten = True
1682+
break
1683+
if not rewritten and os.fstat(handle.fileno()).st_size >= copied:
1684+
copy.seek(0)
1685+
return copy
1686+
copy.close()
1687+
return None
16541688

16551689
def _serve_trajectory(self) -> Response:
16561690
"""Stream the whole env trajectory as JSONL; only a mid-write tail line can arrive torn."""

‎src/agent_env/providers/env_providers/env_gateway_provider.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
# How much of a service's output a failed deploy's error carries: enough for the end of a traceback.
8484
_LOG_TAIL_LINES = 30
8585
_LOG_LINE_CHARS = 300
86+
_CAPTURE_CHECK_ERROR_CHARS = 500
8687
# Gathering it may stall with the connection; past this, the deploy raises the error it had.
8788
_DOWN_SERVICES_TIMEOUT_S = 60.0
8889

@@ -1271,7 +1272,7 @@ async def _refuse_stopped_captures(self, sandbox: VmSandbox, capture_services: l
12711272
return
12721273
if exit_code != 0:
12731274
logger.warning(f"Can't check the traffic captures are running: docker compose ps exited {exit_code}: "
1274-
f"{stderr.strip()[-500:]}")
1275+
f"{stderr.strip()[-_CAPTURE_CHECK_ERROR_CHARS:]}")
12751276
return
12761277
rows = {row.get("Service"): row for row in _compose_ps_rows(stdout)}
12771278
stopped = []
@@ -1658,14 +1659,14 @@ def _traffic_mount(path: str, *, read_only: bool = False) -> str:
16581659

16591660

16601661
def _traffic_capture_service(gateway_image: str, container: str) -> list[str]:
1661-
"""The compose lines of the service capturing ``container``'s traffic: tcpdump, from the gateway's image, on the
1662-
container's own interface, every packet whole and written as it's captured, rotating through
1663-
``TRAFFIC_CAPTURE_FILES`` files of ``TRAFFIC_CAPTURE_FILE_MB`` million bytes named for the container, its oldest
1664-
overwritten once they're full, so a long run holds at most their product per container on the VM's disk. It joins
1662+
"""The compose lines of the service capturing ``container``'s traffic: tcpdump, from the gateway's image, on every
1663+
interface of the container's, its loopback among them, every packet whole and written as it's captured, rotating
1664+
through ``TRAFFIC_CAPTURE_FILES`` files of ``TRAFFIC_CAPTURE_FILE_MB`` million bytes named for the container, its
1665+
oldest overwritten once they're full, so a long run holds at most their product per container on the VM's disk. It joins
16651666
the container's network namespace, so it sees what the container sends and receives and nothing else, and is the
16661667
only service given the capabilities a capture needs. It writes to ``TRAFFIC_VOLUME``, which the gateway mounts
16671668
read-only and no website container mounts at all."""
1668-
argv = ["-i", "eth0", "-n", "-s", "0", "-U", "-Z", "root", "-C", str(TRAFFIC_CAPTURE_FILE_MB),
1669+
argv = ["-i", "any", "-n", "-s", "0", "-U", "-Z", "root", "-C", str(TRAFFIC_CAPTURE_FILE_MB),
16691670
"-W", str(TRAFFIC_CAPTURE_FILES), "-w", f"{TRAFFIC_SIDECAR_DIR}/{container}.pcap"]
16701671
return [
16711672
f" image: {gateway_image}",

‎src/agent_env/task_step/task_steps/traffic_capture.py‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,10 @@
55
didn't, before the env closes (``collect_uncollected_captures``), so a capture isn't lost with its VM."""
66
from __future__ import annotations
77

8-
import asyncio
8+
import functools
99
import logging
1010
import os
1111
import tempfile
12-
import time
1312
from collections.abc import Sequence
1413
from typing import ClassVar, Optional
1514

@@ -32,12 +31,14 @@
3231
from agent_env.store.ids import derive_id, is_local_id, validate_local_id
3332
from agent_env.task_step.context import TaskStepContext
3433
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
34+
from agent_env.task_step.thread_work import finish_on_thread
3535

3636
logger = logging.getLogger(__name__)
3737

3838
#: ``context.metadata[TRAFFIC_KEY][env_id]``: each env whose captures a run collected, or tried to.
3939
TRAFFIC_KEY = "traffic_captures"
4040
_ARTIFACT_TYPE = "traffic_captures"
41+
_ERROR_CHARS = 500
4142

4243

4344
class CollectTrafficCaptureTaskStep(TaskStep):
@@ -116,7 +117,8 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
116117
if is_local_id(universe_id):
117118
validate_local_id(universe_id)
118119
get_config().check_local_run_write(universe_id)
119-
store, version = get_artifact_store(), int(time.time())
120+
store = get_artifact_store()
121+
prefix = store.attempt_prefix(_ARTIFACT_TYPE, universe_id)
120122
files, total, seen = {}, 0, {}
121123
for capture in captures:
122124
container = capture.container
@@ -127,10 +129,12 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
127129
try:
128130
total += await fetch_traffic_capture(deployed.environment_url, deployed.environment_card, capture.name,
129131
path, headers=deployed.control_headers)
130-
url = await asyncio.to_thread(store.put_object_file, _ARTIFACT_TYPE, universe_id, version, filename, path,
131-
PCAP_MEDIA_TYPE)
132-
finally:
133-
os.unlink(path)
132+
except BaseException:
133+
_remove(path)
134+
raise
135+
url = await finish_on_thread(functools.partial(_upload, universe_id, prefix + filename, path),
136+
f"Uploading traffic capture {filename} of env {deployed.env_id!r}",
137+
if_never_run=functools.partial(_remove, path))
134138
files[filename] = store.put_document(FileArtifact(
135139
id=derive_id(universe_id, filename), filename=filename, content_type=PCAP_MEDIA_TYPE, object_url=url,
136140
description=f"Traffic capture of {container} in env {deployed.env_id!r}, task instance {context.instance_id}"))
@@ -143,5 +147,22 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
143147
"containers": sorted(seen)}
144148

145149

150+
def _upload(universe_id: str, object_url: str, path: str) -> str:
151+
"""Upload the capture at ``path`` to ``object_url``, write-once, and remove it: the upload owns the file, so a
152+
cancelled collection never removes it from under the upload."""
153+
try:
154+
return get_config().get_object_store_to_write(object_url, universe_id).put_file_at(object_url, path,
155+
PCAP_MEDIA_TYPE)
156+
finally:
157+
_remove(path)
158+
159+
160+
def _remove(path: str) -> None:
161+
try:
162+
os.unlink(path)
163+
except FileNotFoundError:
164+
pass
165+
166+
146167
def _failed(error: BaseException) -> dict:
147-
return {"state": "failed", "error": f"{type(error).__name__}: {error}"[:500]}
168+
return {"state": "failed", "error": f"{type(error).__name__}: {error}"[:_ERROR_CHARS]}

‎tst/unit/env/gateway/traffic_test.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,68 @@ async def test_list_names_each_capture_by_its_container_oldest_first_and_get_str
7171
assert (got.status_code, got.headers["content-type"], got.content) == (200, PCAP_MEDIA_TYPE, _PCAP_HEADER + b"first")
7272

7373

74+
class _Rewritten:
75+
"""An open capture file tcpdump reuses ``times`` times, each time just after a read of it: it truncates the file
76+
and writes its next round's first packets, shorter than what was read."""
77+
78+
def __init__(self, path, times: int):
79+
self.path, self.times, self.handle = path, times, open(path, "rb")
80+
self.round = path.read_bytes()
81+
82+
def read(self, size: int) -> bytes:
83+
chunk = self.handle.read(size)
84+
if chunk and self.times:
85+
self.times -= 1
86+
self.path.write_bytes(_PCAP_HEADER + b"next")
87+
return chunk
88+
89+
def seek(self, offset: int) -> None:
90+
if self.times:
91+
self.path.write_bytes(self.round)
92+
self.handle.seek(offset)
93+
94+
def fileno(self) -> int:
95+
return self.handle.fileno()
96+
97+
98+
def test_a_capture_rewritten_while_it_is_copied_is_copied_again_whole(traffic_dir):
99+
path = traffic_dir / "notes-website-backend.pcap0"
100+
path.write_bytes(_PCAP_HEADER + b"the round tcpdump overwrites")
101+
gw = _gateway(traffic_dir)
102+
103+
with gw._snapshot_capture(_Rewritten(path, times=1)) as copy:
104+
assert copy.read() == _PCAP_HEADER + b"next"
105+
path.write_bytes(_PCAP_HEADER + b"the round tcpdump overwrites")
106+
assert gw._snapshot_capture(_Rewritten(path, times=gw.CAPTURE_SNAPSHOT_ATTEMPTS)) is None
107+
108+
109+
@pytest.mark.asyncio
110+
async def test_get_answers_503_when_every_copy_of_a_capture_was_rewritten(traffic_dir, monkeypatch):
111+
_write(traffic_dir, "notes-website-backend.pcap0", _PCAP_HEADER + b"x", 1000)
112+
monkeypatch.setattr(Gateway, "_snapshot_capture", lambda self, handle: None)
113+
response = await _get(_gateway(traffic_dir), "/traffic/capture", headers=_bearer(),
114+
params={"name": "notes-website-backend.pcap0"})
115+
assert response.status_code == 503 and response.json()["error"]["code"] == "unavailable"
116+
117+
118+
@pytest.mark.asyncio
119+
async def test_get_sends_the_copy_it_took_whatever_the_file_becomes_after(traffic_dir, monkeypatch):
120+
path = traffic_dir / "notes-website-backend.pcap0"
121+
path.write_bytes(_PCAP_HEADER + b"whole")
122+
real = Gateway._snapshot_capture
123+
124+
def snapshot_then_rewrite(self, handle):
125+
copy = real(self, handle)
126+
path.write_bytes(_PCAP_HEADER + b"the next round, longer")
127+
return copy
128+
129+
monkeypatch.setattr(Gateway, "_snapshot_capture", snapshot_then_rewrite)
130+
response = await _get(_gateway(traffic_dir), "/traffic/capture", headers=_bearer(),
131+
params={"name": "notes-website-backend.pcap0"})
132+
assert (response.content, response.headers["content-length"]) == (_PCAP_HEADER + b"whole",
133+
str(len(_PCAP_HEADER) + 5))
134+
135+
74136
@pytest.mark.asyncio
75137
@pytest.mark.parametrize("name, status", [
76138
("../trajectory.jsonl", 400), ("notes-website-backend.pcap0/../../x", 400), ("stray.txt", 400),

‎tst/unit/providers/env_providers/traffic_capture_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def test_each_website_container_gets_a_capture_in_its_own_network_namespace():
6565
assert capture["depends_on"] == [container]
6666
assert capture["cap_add"] == ["NET_RAW", "NET_ADMIN"]
6767
assert capture["entrypoint"] == ["tcpdump"]
68-
assert capture["command"] == ["-i", "eth0", "-n", "-s", "0", "-U", "-Z", "root", "-C", str(TRAFFIC_CAPTURE_FILE_MB),
68+
assert capture["command"] == ["-i", "any", "-n", "-s", "0", "-U", "-Z", "root", "-C", str(TRAFFIC_CAPTURE_FILE_MB),
6969
"-W", str(TRAFFIC_CAPTURE_FILES), "-w", f"/captures/{container}.pcap"]
7070
assert capture["volumes"] == ["traffic-captures:/captures"]
7171
assert TRAFFIC_CAPTURE_FILE_MB * TRAFFIC_CAPTURE_FILES <= 500 # a long run can't fill the VM's disk

‎tst/unit/task_step/traffic_capture_test.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55

66
from __future__ import annotations
77

8+
import asyncio
89
import os
10+
import threading
911
from unittest.mock import AsyncMock, patch
1012

1113
import httpx
@@ -20,6 +22,7 @@
2022
from agent_env.task.teardown import recorded_sandboxes, tear_down
2123
from agent_env.task_step.context import TaskStepContext
2224
from agent_env.task_step.registry import get_task_step_registry
25+
from agent_env.task_step.task_steps import traffic_capture as traffic_capture_module
2326
from agent_env.task_step.task_steps.traffic_capture import (
2427
TRAFFIC_KEY,
2528
CollectTrafficCaptureTaskStep,
@@ -95,6 +98,46 @@ async def test_each_capture_is_stored_as_a_pcap_in_one_universe_oldest_first(gat
9598
assert get_artifact_store().get_object(files["notes-website-backend-01.pcap"].object_url) == _PCAP + b"newer"
9699

97100

101+
@pytest.mark.asyncio
102+
async def test_two_collections_of_one_env_in_the_same_second_are_each_stored_under_a_prefix_of_their_own(gateway):
103+
context = _context(_deployed())
104+
first = dict(await traffic_capture_module.collect_captures(context, _deployed()))
105+
second = await traffic_capture_module.collect_captures(context, _deployed())
106+
107+
assert (first["state"], second["state"]) == ("collected", "collected")
108+
urls = [{name: artifact.object_url for name, artifact in
109+
Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]).get_file_artifacts().items()}
110+
for entry in (first, second)]
111+
assert sorted(urls[0]) == sorted(urls[1]) and not set(urls[0].values()) & set(urls[1].values())
112+
113+
114+
@pytest.mark.asyncio
115+
async def test_a_collection_cancelled_mid_upload_leaves_the_upload_its_file_to_finish(gateway, monkeypatch):
116+
started, release, uploaded = threading.Event(), threading.Event(), []
117+
real = traffic_capture_module._upload
118+
119+
def held(universe_id, object_url, path):
120+
started.set()
121+
release.wait(10)
122+
uploaded.append((path, os.path.exists(path)))
123+
return real(universe_id, object_url, path)
124+
125+
monkeypatch.setattr(traffic_capture_module, "_upload", held)
126+
collecting = asyncio.ensure_future(traffic_capture_module.collect_captures(_context(_deployed()), _deployed()))
127+
assert await asyncio.to_thread(started.wait, 10)
128+
collecting.cancel()
129+
with pytest.raises(asyncio.CancelledError):
130+
await collecting
131+
release.set()
132+
for _ in range(200):
133+
if uploaded and not os.path.exists(uploaded[0][0]):
134+
break
135+
await asyncio.sleep(0.01)
136+
137+
[(path, existed)] = uploaded
138+
assert existed and not os.path.exists(path)
139+
140+
98141
@pytest.mark.asyncio
99142
async def test_an_env_that_doesnt_capture_is_recorded_as_failed_and_fails_the_task_only_when_asked(gateway):
100143
context = await _step().execute(_context(_deployed(capturing=False)))

0 commit comments

Comments
 (0)