Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@
#: How long a frame's capture may go unanswered before a second capture is sent: Chrome answers a capture of a tab
#: behind the active one only once another capture of it arrives, and then answers both.
NUDGE_S = 0.5
#: How long a frame may take to capture, longer than a screenshot an agent waits on, since a frame is taken in the
#: background: a page too busy to paint, or between documents mid-navigation, gets this long to show, a capture the
#: browser refused for being between documents asked again every ``FRAME_RETRY_S``.
FRAME_CAPTURE_S = 10.0
FRAME_RETRY_S = 0.25
#: What the browser says of a capture refused for the moment, as while a navigation swaps the tab's document.
BETWEEN_DOCUMENTS = ("Not attached to an active page", "Unable to capture screenshot")
#: How often a wait for a tab to load looks again.
LOAD_POLL_S = 0.1
#: Where an agent's files to upload are, and where downloads land, under the session user's home.
Expand Down Expand Up @@ -626,10 +633,10 @@ async def capture_page(self) -> dict:

async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, str, Optional[str], Optional[str]]:
"""A frame of the tab ``target``, else, for none or one that's gone, of the active tab: once its page has loaded,
waiting up to ``load_wait_s``, a JPEG of it as it shows, or, with a dialog keeping it from painting, the frame it
last took, else its last capture, a PNG; a tab that closes while its page loads is the active tab's. The image
base64, its format and the tab and its URL; DeviceUnavailable with no tab open, or for one that paints nothing
in time."""
waiting up to ``load_wait_s``, a JPEG of it as it shows, within ``FRAME_CAPTURE_S``, or, with a dialog keeping
it from painting, the frame it last took, else its last capture, a PNG; a tab that closes while its page loads
is the active tab's. The image base64, its format and the tab and its URL; DeviceUnavailable with no tab open,
or for one that paints nothing in time."""
session = next((s for s, t in self._sessions.items() if t == target), None) if target else None
if session is None:
session = await self.active_session()
Expand All @@ -640,26 +647,42 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s
if session not in self._sessions:
session = await self.active_session()
target = self._sessions[session]
if session in self._open_dialogs:
if (earlier := self._frames.get(target)) is None and (shot := self._shots.get(target)) is not None:
earlier = shot, "png"
if earlier is None:
raise DeviceUnavailable("a dialog keeps the tab from painting and it had no capture before it")
return *earlier, target, self.url_of(target)
try:
shot = await self._nudged_capture(session)
except asyncio.TimeoutError:
raise DeviceUnavailable(f"the tab painted nothing within {CAPTURE_TIMEOUT_S:g} s") from None
except (ConnectionError, DevToolsError) as e:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
loop = asyncio.get_running_loop()
deadline = loop.time() + FRAME_CAPTURE_S
while True:
if session in self._open_dialogs:
return self._held_frame(target)
try:
shot = await self._nudged_capture(session, deadline - loop.time())
break
except asyncio.TimeoutError:
raise DeviceUnavailable(f"the tab painted nothing within {FRAME_CAPTURE_S:g} s") from None
except DevToolsError as e:
if not any(why in str(e) for why in BETWEEN_DOCUMENTS) or loop.time() + FRAME_RETRY_S >= deadline:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
except ConnectionError as e:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
await asyncio.sleep(FRAME_RETRY_S)
if session not in self._sessions:
session = await self.active_session()
target = self._sessions[session]
Comment thread
earakely-scale marked this conversation as resolved.
Resolved
self._frames[target] = shot["data"], "jpg"
return shot["data"], "jpg", target, self.url_of(target)

async def _nudged_capture(self, session: str) -> dict:
def _held_frame(self, target: str) -> tuple[str, str, Optional[str], Optional[str]]:
"""The frame of a tab a dialog keeps from painting: the one it last took, else its last capture, a PNG;
DeviceUnavailable for a tab with neither."""
if (earlier := self._frames.get(target)) is None and (shot := self._shots.get(target)) is not None:
earlier = shot, "png"
if earlier is None:
raise DeviceUnavailable("a dialog keeps the tab from painting and it had no capture before it")
return *earlier, target, self.url_of(target)

async def _nudged_capture(self, session: str, timeout_s: float) -> dict:
"""The tab's JPEG from the first of its captures to answer: a second follows one unanswered after ``NUDGE_S``,
since a tab behind the active one answers only then, all within ``CAPTURE_TIMEOUT_S``."""
since a tab behind the active one answers only then, all within ``timeout_s``."""
loop = asyncio.get_running_loop()
deadline = loop.time() + CAPTURE_TIMEOUT_S
deadline = loop.time() + timeout_s

def capture() -> asyncio.Future:
return asyncio.ensure_future(self.devtools.call("Page.captureScreenshot", {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
from pydantic import BaseModel, ValidationError

from ..computer.errors import InvalidArgument
from ..transfers import NamespaceUploader, WriteNamespaceGrant
from ..transfers import TRANSFER_STALL_BUDGET_SECONDS, NamespaceUploader, WriteNamespaceGrant
from .control import FRAME_CAPTURE_S
from .log import ERROR_CHARS, BrowserLog

if TYPE_CHECKING:
Expand Down Expand Up @@ -45,6 +46,13 @@
#: looks again.
LOG_WAIT_S = 10.0
LOG_POLL_S = 0.2
#: The longest a start takes, its first frame taken and uploaded, and an end, a frame still being taken, its last two,
#: the wait for its log and the log's and the manifest's uploads, with each upload as long as the transfer helpers
#: allow one that stalls and a reply's margin: what a client of the extension waits on each.
_FRAME_S = LOAD_WAIT_S + FRAME_CAPTURE_S + TRANSFER_STALL_BUDGET_SECONDS
_REPLY_S = 30.0
START_S = _FRAME_S + _REPLY_S
END_S = 3 * _FRAME_S + LOG_WAIT_S + 2 * TRANSFER_STALL_BUDGET_SECONDS + _REPLY_S


class RecordedFrame(BaseModel):
Expand Down
84 changes: 84 additions & 0 deletions packages/agentenv-protocol/tests/test_browser_recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
from agentenv_protocol.computer.errors import DeviceUnavailable, InvalidArgument
from agentenv_protocol.transfers import HttpPostPolicyGrant, WriteNamespaceGrant
from agentenv_protocol.website_browser import recording as browser_recording
from agentenv_protocol.website_browser import control as browser_control
from agentenv_protocol.website_browser.control import NUDGE_S, BrowserControl
from agentenv_protocol.website_browser.devtools import DevToolsError
from agentenv_protocol.website_browser.log import BrowserLog
from agentenv_protocol.website_browser.recording import BrowserRecordingManifest, BrowserRecordings

Expand Down Expand Up @@ -260,6 +262,88 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None):
return await asyncio.wait_for(answer, timeout_s)


class RefusingPipe(Pipe):
"""A pipe whose browser refuses its first ``refusals`` captures with ``why``, as it does a tab between documents
while a navigation swaps them, and answers the rest."""

def __init__(self, refusals: int, why: str = "Page.captureScreenshot: Not attached to an active page") -> None:
super().__init__()
self.refusals, self.why = refusals, why

async def call(self, method, params=None, *, session_id=None, timeout_s=None):
if method == "Page.captureScreenshot" and self.refusals:
self.refusals -= 1
self.captured.append((session_id, params, asyncio.get_running_loop().time()))
raise DevToolsError(self.why)
return await super().call(method, params, session_id=session_id, timeout_s=timeout_s)


class BusyPipe(Pipe):
"""A pipe whose page is too busy to paint for ``busy_s``, every capture answering only after it."""

def __init__(self, busy_s: float) -> None:
super().__init__()
self.busy_s = busy_s

async def call(self, method, params=None, *, session_id=None, timeout_s=None):
if method == "Page.captureScreenshot":
await asyncio.wait_for(asyncio.sleep(self.busy_s), timeout_s)
return await super().call(method, params, session_id=session_id, timeout_s=timeout_s)


def _one_tab(pipe: Pipe) -> BrowserControl:
control = BrowserControl(pipe, "agent", (1280, 800))
pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": "T1", "type": "page", "url": "about:blank"}})
pipe.emit("Target.attachedToTarget", {"sessionId": "s-T1", "targetInfo": {"targetId": "T1", "type": "page"}})
return control


@pytest.mark.asyncio
async def test_a_frame_refused_while_its_tab_is_between_documents_is_asked_again_until_it_shows():
pipe = RefusingPipe(refusals=2)

data, _, tab, _ = await _one_tab(pipe).frame("T1", load_wait_s=0)

assert (base64.b64decode(data), tab, len(pipe.captured)) == (b"jpeg", "T1", 3)


@pytest.mark.asyncio
async def test_a_tab_that_closes_while_its_frame_is_asked_again_is_framed_as_the_active_tab_a_dialog_holds():
pipe = RefusingPipe(refusals=0)
control = BrowserControl(pipe, "agent", (1280, 800))
for tab in ("T1", "T2"):
pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": tab, "type": "page", "url": "about:blank"}})
pipe.emit("Target.attachedToTarget", {"sessionId": f"s-{tab}", "targetInfo": {"targetId": tab, "type": "page"}})
held = await control.frame("T1", load_wait_s=0)
pipe.emit("Page.javascriptDialogOpening", {"type": "confirm", "message": "Leave?", "url": "https://a.test/"}, "s-T1")
pipe.refusals = 1
asyncio.get_running_loop().call_later(0.05, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T2"})

framed = await control.frame("T2", load_wait_s=0)

assert framed[:3] == held[:3] and held[2] == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T2"]


@pytest.mark.asyncio
async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once():
pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error")

with pytest.raises(DeviceUnavailable, match="Internal error"):
await _one_tab(pipe).frame("T1", load_wait_s=0)
assert len(pipe.captured) == 1


@pytest.mark.asyncio
async def test_a_page_too_busy_to_paint_has_the_frame_budget_to_show_and_no_longer(monkeypatch):
monkeypatch.setattr(browser_control, "FRAME_CAPTURE_S", 0.6)
monkeypatch.setattr(browser_control, "NUDGE_S", 0.1)

data, _, _, _ = await _one_tab(BusyPipe(busy_s=0.3)).frame("T1", load_wait_s=0)
assert base64.b64decode(data) == b"jpeg"
with pytest.raises(DeviceUnavailable, match="painted nothing within 0.6 s"):
await _one_tab(BusyPipe(busy_s=1.0)).frame("T1", load_wait_s=0)


@pytest.mark.asyncio
async def test_a_tab_behind_the_active_one_is_framed_once_a_second_capture_nudges_it():
pipe = BehindPipe()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from agentenv_protocol.computer.app import COMPUTER_APP
from agentenv_protocol.computer.app_runtime import HOLD_BOOT_LAUNCH_ENV
from agentenv_protocol.website_browser import BROWSER, BROWSER_RECORDING, LOADED, start_tab_why
from agentenv_protocol.website_browser.recording import END_S as BROWSER_RECORDING_END_S

from agent_env.artifact.artifacts.computer import ComputerArtifact
from agent_env.attribution import Attribution
Expand Down Expand Up @@ -350,7 +351,7 @@ async def end_recordings(record: DeployedSandboxEnv, reason: str) -> None:
continue
try:
await record.invoke(BROWSER_RECORDING, "end_all", {"reason": reason}, environment_name=child.get("name"),
timeout=_END_RECORDINGS_TIMEOUT_S)
timeout=max(_END_RECORDINGS_TIMEOUT_S, BROWSER_RECORDING_END_S))
except Exception as e:
logger.warning(f"'{record.env_id}': {child.get('name')}'s browser recordings weren't ended before its "
f"computer went: {e}")
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/task_step/task_steps/browser/recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

from agentenv_protocol.client import find_extension
from agentenv_protocol.website_browser import BROWSER_RECORDING, BrowserRecordingManifest, RecordedFrame
from agentenv_protocol.website_browser.recording import LOG, MANIFEST, MAX_FRAMES, frame_of
from agentenv_protocol.website_browser.recording import END_S, LOG, MANIFEST, MAX_FRAMES, START_S, frame_of

from agent_env.artifact import FileArtifact
from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse
Expand Down Expand Up @@ -83,7 +83,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
namespace = recording_namespace(request_id, _KEY_PREFIX)
started = await record.invoke(BROWSER_RECORDING, "start", {
"request_id": request_id, "frames": self.frames, "upload": _grant(record, card, namespace)},
environment_name=browser)
environment_name=browser, timeout=START_S)
except Exception as e:
recordings[self.name] = {"env_id": self.env_id, "browser": self.browser, "frames": self.frames,
"state": "failed", "error": why(e)}
Expand Down Expand Up @@ -166,7 +166,7 @@ async def end_browser_recording(context: TaskStepContext, name: str, reason: str
record, browser, card = find_browser(deployed_env(context, entry["env_id"]), entry["browser"])
manifest = BrowserRecordingManifest.model_validate(await record.invoke(BROWSER_RECORDING, "end", {
"recording_id": entry["recording_id"], "upload": _grant(record, card, entry["namespace"], "end"),
"reason": reason}, environment_name=browser))
"reason": reason}, environment_name=browser, timeout=END_S))
except Exception as e:
logger.warning("end_browser_recording %s: %s; registering what its namespace holds", name, e)
entry["error"] = why(e)
Expand Down
6 changes: 5 additions & 1 deletion tst/unit/task_step/browser_recording_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import pytest
from agentenv_protocol.computer import COMPUTER_RECORDING
from agentenv_protocol.website_browser import BROWSER, BROWSER_RECORDING
from agentenv_protocol.website_browser.recording import END_S, START_S

from agent_env.artifact import Artifact
from agent_env.config import set_object_store
Expand Down Expand Up @@ -43,10 +44,12 @@ def __init__(self, store, misses: bool = False, forgets: bool = False, recording
self.environment_card = {"name": "browser", "children_environments": children}
self.store, self.misses, self.forgets, self.gaps = store, misses, forgets, gaps
self.calls: list[tuple[str, dict, str]] = []
self.timeouts: list[float] = []

async def invoke(self, uri, method, params=None, environment_name=None, **_):
async def invoke(self, uri, method, params=None, environment_name=None, timeout=None, **_):
assert uri == BROWSER_RECORDING
self.calls.append((method, params, environment_name))
self.timeouts.append(timeout)
if method == "start":
self.root = params["upload"]["root_path"]
self.store.put(f"{self.root}/rec-1/frames/0000.jpg", b"jpeg-0", content_type="image/jpeg")
Expand Down Expand Up @@ -99,6 +102,7 @@ async def test_a_browser_recording_is_started_through_its_browsers_card_and_regi
(_, start, browser), (_, end, _) = computer.calls
assert (start["request_id"], start["frames"], browser) == ("run-1:start-run", True, "chromium")
assert start["upload"]["max_objects"] == 102 and "/agentenv-browser-recordings/" in started["namespace"]
assert computer.timeouts == [START_S, END_S]
assert end["upload"]["root_path"] == start["upload"]["root_path"] and end["reason"] == "end"
assert (entry["state"], entry["frames_taken"], entry["missing_frames"], entry["log_uploaded"]) == \
("ended", 2, 0, True)
Expand Down
Loading