66isn't lost with its VM, and saves what it recorded to the run's stored instance."""
77from __future__ import annotations
88
9+ import asyncio
10+ import copy
911import functools
1012import logging
1113import os
3941#: ``context.metadata[TRAFFIC_KEY][env_id]``: each env whose captures a run collected, or tried to.
4042TRAFFIC_KEY = "traffic_captures"
4143_ARTIFACT_TYPE = "traffic_captures"
44+ _SAVING : set [asyncio .Future ] = set ()
4245_ERROR_CHARS = 500
4346
4447
@@ -100,11 +103,31 @@ async def collect_uncollected_captures(context: TaskStepContext, envs: Sequence[
100103 if captures_traffic (deployed ) and (entries .get (key ) or {}).get ("state" ) != "collected" :
101104 tried = True
102105 await collect_captures (context , deployed , key = key )
103- finally :
104- if tried and context .instance_id and TRAFFIC_KEY in context .metadata :
105- from agent_env .task .store import record_context_metadata
106+ except BaseException :
107+ if tried :
108+ _save_in_background (context )
109+ raise
110+ if tried :
111+ await _save (context )
112+
113+
114+ async def _save (context : TaskStepContext ) -> None :
115+ """Save the run's ``traffic_captures`` entries to its stored instance on a worker thread, which finishes the
116+ write even when teardown's time runs out while it waits."""
117+ if context .instance_id and TRAFFIC_KEY in context .metadata :
118+ from agent_env .task .store import record_context_metadata
106119
107- record_context_metadata (context .instance_id , TRAFFIC_KEY , context .metadata [TRAFFIC_KEY ])
120+ await finish_on_thread (functools .partial (record_context_metadata , context .instance_id , TRAFFIC_KEY ,
121+ copy .deepcopy (context .metadata [TRAFFIC_KEY ])),
122+ f"Saving the traffic captures of task instance { context .instance_id } " )
123+
124+
125+ def _save_in_background (context : TaskStepContext ) -> None :
126+ """Start ``_save`` without waiting for it, as a collection teardown's time ran out on does, so the entries
127+ collected by then are saved while teardown goes on to terminate the sandboxes."""
128+ saving = asyncio .ensure_future (_save (context ))
129+ _SAVING .add (saving )
130+ saving .add_done_callback (_SAVING .discard )
108131
109132
110133def rolled_back_deployments (context : TaskStepContext ) -> list [tuple [str , DeployedEnv ]]:
@@ -122,9 +145,9 @@ async def collect_captures(context: TaskStepContext, deployed: DeployedEnv, *, k
122145 """Store each capture ``deployed`` serves as a pcap FileArtifact, register them as one FileArtifactUniverse, and
123146 record it in ``context.metadata[TRAFFIC_KEY][key]``, by default its env_id. Each file is named for its container
124147 and its place among that container's files, oldest first, so they read in the order their packets were captured.
125- Never raises: the entry, which it also returns, says what happened. A collection that stops partway, failed or
126- cancelled, still registers the files it stored by then and records them as ``partial``, which teardown collects
127- again."""
148+ Never raises: the entry, which it also returns, says what happened. Each file stored is registered in the universe
149+ as it lands, every store write on a worker thread, so a collection that stops partway, failed or cancelled, keeps
150+ the files it stored by then recorded as ``partial``, which teardown collects again."""
128151 key = key or deployed .env_id
129152 entries = context .metadata .setdefault (TRAFFIC_KEY , {})
130153 try :
@@ -147,7 +170,7 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
147170 get_config ().check_local_run_write (universe_id )
148171 store = get_artifact_store ()
149172 prefix = store .attempt_prefix (_ARTIFACT_TYPE , universe_id )
150- files , total , seen = {}, 0 , {}
173+ files , total , seen , artifact = {}, 0 , {}, None
151174 try :
152175 for capture in captures :
153176 container = capture .container
@@ -164,26 +187,31 @@ async def _store(context: TaskStepContext, deployed: DeployedEnv, captures: list
164187 url = await finish_on_thread (functools .partial (_upload , universe_id , prefix + filename , path ),
165188 f"Uploading traffic capture { filename } of env { deployed .env_id !r} " ,
166189 if_never_run = functools .partial (_remove , path ))
167- files [ filename ] = store . put_document ( FileArtifact (
190+ document = FileArtifact (
168191 id = derive_id (universe_id , filename ), filename = filename , content_type = PCAP_MEDIA_TYPE , object_url = url ,
169- description = f"Traffic capture of { container } in env { deployed .env_id !r} , task instance { context .instance_id } " ))
192+ description = f"Traffic capture of { container } in env { deployed .env_id !r} , task instance { context .instance_id } " )
193+ files [filename ] = await finish_on_thread (functools .partial (store .put_document , document ),
194+ f"Storing traffic capture { filename } of env { deployed .env_id !r} " )
195+ artifact = await finish_on_thread (functools .partial (_register , universe_id , dict (files )),
196+ f"Registering the traffic captures of env { deployed .env_id !r} " )
197+ context .metadata [TRAFFIC_KEY ][key ] = _entry ("partial" , artifact , files , total , seen )
170198 except BaseException as e :
171- if files :
172- context .metadata [TRAFFIC_KEY ][key ] = {** _entry (universe_id , files , total , seen ), "state" : "partial" ,
199+ if artifact is not None :
200+ context .metadata [TRAFFIC_KEY ][key ] = {** _entry ("partial" , artifact , files , total , seen ),
173201 "error" : _failed (e )["error" ]}
174202 raise
175203 logger .info ("collect_traffic_capture %s: %d capture files, %d bytes" , deployed .env_id , len (files ), total )
176- return _entry (universe_id , files , total , seen )
204+ return _entry ("collected" , artifact , files , total , seen )
205+
206+
207+ def _register (universe_id : str , files : dict ) -> dict :
208+ """Register ``files`` as the next version of the FileArtifactUniverse ``universe_id``."""
209+ universe = FileArtifactUniverse .put (id = universe_id , file_artifacts = files )
210+ return {"id" : universe .id , "version" : universe .version }
177211
178212
179- def _entry (universe_id : str , files : dict , total : int , seen : dict ) -> dict :
180- """The ``collected`` entry of ``files``, registered as one FileArtifactUniverse when there are any."""
181- artifact = None
182- if files :
183- universe = FileArtifactUniverse .put (id = universe_id , file_artifacts = files )
184- artifact = {"id" : universe .id , "version" : universe .version }
185- return {"state" : "collected" , "artifact" : artifact , "files" : sorted (files ), "bytes" : total ,
186- "containers" : sorted (seen )}
213+ def _entry (state : str , artifact : Optional [dict ], files : dict , total : int , seen : dict ) -> dict :
214+ return {"state" : state , "artifact" : artifact , "files" : sorted (files ), "bytes" : total , "containers" : sorted (seen )}
187215
188216
189217def _upload (universe_id : str , object_url : str , path : str ) -> str :
0 commit comments