diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index f8371b997..7e7bbd59b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -49,6 +49,13 @@ #: How long a frame's capture may go unanswered before a second capture is sent: Chrome answers a capture of a tab #: behind the active one only once another capture of it arrives, and then answers both. NUDGE_S = 0.5 +#: How long a frame may take to capture, longer than a screenshot an agent waits on, since a frame is taken in the +#: background: a page too busy to paint, or between documents mid-navigation, gets this long to show, a capture the +#: browser refused for being between documents asked again every ``FRAME_RETRY_S``. +FRAME_CAPTURE_S = 10.0 +FRAME_RETRY_S = 0.25 +#: What the browser says of a capture refused for the moment, as while a navigation swaps the tab's document. +BETWEEN_DOCUMENTS = ("Not attached to an active page", "Unable to capture screenshot") #: How often a wait for a tab to load looks again. LOAD_POLL_S = 0.1 #: Where an agent's files to upload are, and where downloads land, under the session user's home. @@ -626,10 +633,10 @@ async def capture_page(self) -> dict: async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, str, Optional[str], Optional[str]]: """A frame of the tab ``target``, else, for none or one that's gone, of the active tab: once its page has loaded, - waiting up to ``load_wait_s``, a JPEG of it as it shows, or, with a dialog keeping it from painting, the frame it - last took, else its last capture, a PNG; a tab that closes while its page loads is the active tab's. The image - base64, its format and the tab and its URL; DeviceUnavailable with no tab open, or for one that paints nothing - in time.""" + waiting up to ``load_wait_s``, a JPEG of it as it shows, within ``FRAME_CAPTURE_S``, or, with a dialog keeping + it from painting, the frame it last took, else its last capture, a PNG; a tab that closes while its page loads + is the active tab's. The image base64, its format and the tab and its URL; DeviceUnavailable with no tab open, + or for one that paints nothing in time.""" session = next((s for s, t in self._sessions.items() if t == target), None) if target else None if session is None: session = await self.active_session() @@ -640,26 +647,42 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s if session not in self._sessions: session = await self.active_session() target = self._sessions[session] - if session in self._open_dialogs: - if (earlier := self._frames.get(target)) is None and (shot := self._shots.get(target)) is not None: - earlier = shot, "png" - if earlier is None: - raise DeviceUnavailable("a dialog keeps the tab from painting and it had no capture before it") - return *earlier, target, self.url_of(target) - try: - shot = await self._nudged_capture(session) - except asyncio.TimeoutError: - raise DeviceUnavailable(f"the tab painted nothing within {CAPTURE_TIMEOUT_S:g} s") from None - except (ConnectionError, DevToolsError) as e: - raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None + loop = asyncio.get_running_loop() + deadline = loop.time() + FRAME_CAPTURE_S + while True: + if session in self._open_dialogs: + return self._held_frame(target) + try: + shot = await self._nudged_capture(session, deadline - loop.time()) + break + except asyncio.TimeoutError: + raise DeviceUnavailable(f"the tab painted nothing within {FRAME_CAPTURE_S:g} s") from None + except DevToolsError as e: + if not any(why in str(e) for why in BETWEEN_DOCUMENTS) or loop.time() + FRAME_RETRY_S >= deadline: + raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None + except ConnectionError as e: + raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None + await asyncio.sleep(FRAME_RETRY_S) + if session not in self._sessions: + session = await self.active_session() + target = self._sessions[session] self._frames[target] = shot["data"], "jpg" return shot["data"], "jpg", target, self.url_of(target) - async def _nudged_capture(self, session: str) -> dict: + def _held_frame(self, target: str) -> tuple[str, str, Optional[str], Optional[str]]: + """The frame of a tab a dialog keeps from painting: the one it last took, else its last capture, a PNG; + DeviceUnavailable for a tab with neither.""" + if (earlier := self._frames.get(target)) is None and (shot := self._shots.get(target)) is not None: + earlier = shot, "png" + if earlier is None: + raise DeviceUnavailable("a dialog keeps the tab from painting and it had no capture before it") + return *earlier, target, self.url_of(target) + + async def _nudged_capture(self, session: str, timeout_s: float) -> dict: """The tab's JPEG from the first of its captures to answer: a second follows one unanswered after ``NUDGE_S``, - since a tab behind the active one answers only then, all within ``CAPTURE_TIMEOUT_S``.""" + since a tab behind the active one answers only then, all within ``timeout_s``.""" loop = asyncio.get_running_loop() - deadline = loop.time() + CAPTURE_TIMEOUT_S + deadline = loop.time() + timeout_s def capture() -> asyncio.Future: return asyncio.ensure_future(self.devtools.call("Page.captureScreenshot", { diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py index 1894927c3..e2f1a50ac 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py @@ -16,7 +16,8 @@ from pydantic import BaseModel, ValidationError from ..computer.errors import InvalidArgument -from ..transfers import NamespaceUploader, WriteNamespaceGrant +from ..transfers import TRANSFER_STALL_BUDGET_SECONDS, NamespaceUploader, WriteNamespaceGrant +from .control import FRAME_CAPTURE_S from .log import ERROR_CHARS, BrowserLog if TYPE_CHECKING: @@ -45,6 +46,13 @@ #: looks again. LOG_WAIT_S = 10.0 LOG_POLL_S = 0.2 +#: The longest a start takes, its first frame taken and uploaded, and an end, a frame still being taken, its last two, +#: the wait for its log and the log's and the manifest's uploads, with each upload as long as the transfer helpers +#: allow one that stalls and a reply's margin: what a client of the extension waits on each. +_FRAME_S = LOAD_WAIT_S + FRAME_CAPTURE_S + TRANSFER_STALL_BUDGET_SECONDS +_REPLY_S = 30.0 +START_S = _FRAME_S + _REPLY_S +END_S = 3 * _FRAME_S + LOG_WAIT_S + 2 * TRANSFER_STALL_BUDGET_SECONDS + _REPLY_S class RecordedFrame(BaseModel): diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 4b35d1899..497dd0100 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -15,7 +15,9 @@ from agentenv_protocol.computer.errors import DeviceUnavailable, InvalidArgument from agentenv_protocol.transfers import HttpPostPolicyGrant, WriteNamespaceGrant from agentenv_protocol.website_browser import recording as browser_recording +from agentenv_protocol.website_browser import control as browser_control from agentenv_protocol.website_browser.control import NUDGE_S, BrowserControl +from agentenv_protocol.website_browser.devtools import DevToolsError from agentenv_protocol.website_browser.log import BrowserLog from agentenv_protocol.website_browser.recording import BrowserRecordingManifest, BrowserRecordings @@ -260,6 +262,88 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None): return await asyncio.wait_for(answer, timeout_s) +class RefusingPipe(Pipe): + """A pipe whose browser refuses its first ``refusals`` captures with ``why``, as it does a tab between documents + while a navigation swaps them, and answers the rest.""" + + def __init__(self, refusals: int, why: str = "Page.captureScreenshot: Not attached to an active page") -> None: + super().__init__() + self.refusals, self.why = refusals, why + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Page.captureScreenshot" and self.refusals: + self.refusals -= 1 + self.captured.append((session_id, params, asyncio.get_running_loop().time())) + raise DevToolsError(self.why) + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + +class BusyPipe(Pipe): + """A pipe whose page is too busy to paint for ``busy_s``, every capture answering only after it.""" + + def __init__(self, busy_s: float) -> None: + super().__init__() + self.busy_s = busy_s + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Page.captureScreenshot": + await asyncio.wait_for(asyncio.sleep(self.busy_s), timeout_s) + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + +def _one_tab(pipe: Pipe) -> BrowserControl: + control = BrowserControl(pipe, "agent", (1280, 800)) + pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": "T1", "type": "page", "url": "about:blank"}}) + pipe.emit("Target.attachedToTarget", {"sessionId": "s-T1", "targetInfo": {"targetId": "T1", "type": "page"}}) + return control + + +@pytest.mark.asyncio +async def test_a_frame_refused_while_its_tab_is_between_documents_is_asked_again_until_it_shows(): + pipe = RefusingPipe(refusals=2) + + data, _, tab, _ = await _one_tab(pipe).frame("T1", load_wait_s=0) + + assert (base64.b64decode(data), tab, len(pipe.captured)) == (b"jpeg", "T1", 3) + + +@pytest.mark.asyncio +async def test_a_tab_that_closes_while_its_frame_is_asked_again_is_framed_as_the_active_tab_a_dialog_holds(): + pipe = RefusingPipe(refusals=0) + control = BrowserControl(pipe, "agent", (1280, 800)) + for tab in ("T1", "T2"): + pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": tab, "type": "page", "url": "about:blank"}}) + pipe.emit("Target.attachedToTarget", {"sessionId": f"s-{tab}", "targetInfo": {"targetId": tab, "type": "page"}}) + held = await control.frame("T1", load_wait_s=0) + pipe.emit("Page.javascriptDialogOpening", {"type": "confirm", "message": "Leave?", "url": "https://a.test/"}, "s-T1") + pipe.refusals = 1 + asyncio.get_running_loop().call_later(0.05, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T2"}) + + framed = await control.frame("T2", load_wait_s=0) + + assert framed[:3] == held[:3] and held[2] == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T2"] + + +@pytest.mark.asyncio +async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once(): + pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error") + + with pytest.raises(DeviceUnavailable, match="Internal error"): + await _one_tab(pipe).frame("T1", load_wait_s=0) + assert len(pipe.captured) == 1 + + +@pytest.mark.asyncio +async def test_a_page_too_busy_to_paint_has_the_frame_budget_to_show_and_no_longer(monkeypatch): + monkeypatch.setattr(browser_control, "FRAME_CAPTURE_S", 0.6) + monkeypatch.setattr(browser_control, "NUDGE_S", 0.1) + + data, _, _, _ = await _one_tab(BusyPipe(busy_s=0.3)).frame("T1", load_wait_s=0) + assert base64.b64decode(data) == b"jpeg" + with pytest.raises(DeviceUnavailable, match="painted nothing within 0.6 s"): + await _one_tab(BusyPipe(busy_s=1.0)).frame("T1", load_wait_s=0) + + @pytest.mark.asyncio async def test_a_tab_behind_the_active_one_is_framed_once_a_second_capture_nudges_it(): pipe = BehindPipe() diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 4615656ac..851733000 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -15,6 +15,7 @@ from agentenv_protocol.computer.app import COMPUTER_APP from agentenv_protocol.computer.app_runtime import HOLD_BOOT_LAUNCH_ENV from agentenv_protocol.website_browser import BROWSER, BROWSER_RECORDING, LOADED, start_tab_why +from agentenv_protocol.website_browser.recording import END_S as BROWSER_RECORDING_END_S from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.attribution import Attribution @@ -350,7 +351,7 @@ async def end_recordings(record: DeployedSandboxEnv, reason: str) -> None: continue try: await record.invoke(BROWSER_RECORDING, "end_all", {"reason": reason}, environment_name=child.get("name"), - timeout=_END_RECORDINGS_TIMEOUT_S) + timeout=max(_END_RECORDINGS_TIMEOUT_S, BROWSER_RECORDING_END_S)) except Exception as e: logger.warning(f"'{record.env_id}': {child.get('name')}'s browser recordings weren't ended before its " f"computer went: {e}") diff --git a/src/agent_env/task_step/task_steps/browser/recording.py b/src/agent_env/task_step/task_steps/browser/recording.py index c0eca067e..fbbc57e97 100644 --- a/src/agent_env/task_step/task_steps/browser/recording.py +++ b/src/agent_env/task_step/task_steps/browser/recording.py @@ -13,7 +13,7 @@ from agentenv_protocol.client import find_extension from agentenv_protocol.website_browser import BROWSER_RECORDING, BrowserRecordingManifest, RecordedFrame -from agentenv_protocol.website_browser.recording import LOG, MANIFEST, MAX_FRAMES, frame_of +from agentenv_protocol.website_browser.recording import END_S, LOG, MANIFEST, MAX_FRAMES, START_S, frame_of from agent_env.artifact import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse @@ -83,7 +83,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: namespace = recording_namespace(request_id, _KEY_PREFIX) started = await record.invoke(BROWSER_RECORDING, "start", { "request_id": request_id, "frames": self.frames, "upload": _grant(record, card, namespace)}, - environment_name=browser) + environment_name=browser, timeout=START_S) except Exception as e: recordings[self.name] = {"env_id": self.env_id, "browser": self.browser, "frames": self.frames, "state": "failed", "error": why(e)} @@ -166,7 +166,7 @@ async def end_browser_recording(context: TaskStepContext, name: str, reason: str record, browser, card = find_browser(deployed_env(context, entry["env_id"]), entry["browser"]) manifest = BrowserRecordingManifest.model_validate(await record.invoke(BROWSER_RECORDING, "end", { "recording_id": entry["recording_id"], "upload": _grant(record, card, entry["namespace"], "end"), - "reason": reason}, environment_name=browser)) + "reason": reason}, environment_name=browser, timeout=END_S)) except Exception as e: logger.warning("end_browser_recording %s: %s; registering what its namespace holds", name, e) entry["error"] = why(e) diff --git a/tst/unit/task_step/browser_recording_test.py b/tst/unit/task_step/browser_recording_test.py index 229f1bf07..1d9866d71 100644 --- a/tst/unit/task_step/browser_recording_test.py +++ b/tst/unit/task_step/browser_recording_test.py @@ -10,6 +10,7 @@ import pytest from agentenv_protocol.computer import COMPUTER_RECORDING from agentenv_protocol.website_browser import BROWSER, BROWSER_RECORDING +from agentenv_protocol.website_browser.recording import END_S, START_S from agent_env.artifact import Artifact from agent_env.config import set_object_store @@ -43,10 +44,12 @@ def __init__(self, store, misses: bool = False, forgets: bool = False, recording self.environment_card = {"name": "browser", "children_environments": children} self.store, self.misses, self.forgets, self.gaps = store, misses, forgets, gaps self.calls: list[tuple[str, dict, str]] = [] + self.timeouts: list[float] = [] - async def invoke(self, uri, method, params=None, environment_name=None, **_): + async def invoke(self, uri, method, params=None, environment_name=None, timeout=None, **_): assert uri == BROWSER_RECORDING self.calls.append((method, params, environment_name)) + self.timeouts.append(timeout) if method == "start": self.root = params["upload"]["root_path"] self.store.put(f"{self.root}/rec-1/frames/0000.jpg", b"jpeg-0", content_type="image/jpeg") @@ -99,6 +102,7 @@ async def test_a_browser_recording_is_started_through_its_browsers_card_and_regi (_, start, browser), (_, end, _) = computer.calls assert (start["request_id"], start["frames"], browser) == ("run-1:start-run", True, "chromium") assert start["upload"]["max_objects"] == 102 and "/agentenv-browser-recordings/" in started["namespace"] + assert computer.timeouts == [START_S, END_S] assert end["upload"]["root_path"] == start["upload"]["root_path"] and end["reason"] == "end" assert (entry["state"], entry["frames_taken"], entry["missing_frames"], entry["log_uploaded"]) == \ ("ended", 2, 0, True)