diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index 7e7bbd59b..e2f2f8d1b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -51,11 +51,12 @@ NUDGE_S = 0.5 #: How long a frame may take to capture, longer than a screenshot an agent waits on, since a frame is taken in the #: background: a page too busy to paint, or between documents mid-navigation, gets this long to show, a capture the -#: browser refused for being between documents asked again every ``FRAME_RETRY_S``. +#: browser refused for the moment asked again every ``FRAME_RETRY_S``. FRAME_CAPTURE_S = 10.0 FRAME_RETRY_S = 0.25 -#: What the browser says of a capture refused for the moment, as while a navigation swaps the tab's document. -BETWEEN_DOCUMENTS = ("Not attached to an active page", "Unable to capture screenshot") +#: What the browser says of a capture refused for the moment: while a navigation swaps the tab's document, or once the +#: session it was asked on is gone, as when the tab closes or the browser attaches it anew. +FOR_THE_MOMENT = ("Not attached to an active page", "Unable to capture screenshot", "Session with given id not found") #: How often a wait for a tab to load looks again. LOAD_POLL_S = 0.1 #: Where an agent's files to upload are, and where downloads land, under the session user's home. @@ -658,14 +659,20 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s except asyncio.TimeoutError: raise DeviceUnavailable(f"the tab painted nothing within {FRAME_CAPTURE_S:g} s") from None except DevToolsError as e: - if not any(why in str(e) for why in BETWEEN_DOCUMENTS) or loop.time() + FRAME_RETRY_S >= deadline: + if not any(why in str(e) for why in FOR_THE_MOMENT) or loop.time() + FRAME_RETRY_S >= deadline: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None except ConnectionError as e: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None await asyncio.sleep(FRAME_RETRY_S) - if session not in self._sessions: - session = await self.active_session() - target = self._sessions[session] + while session not in self._sessions: + try: + session = next((s for s, t in self._sessions.items() if t == target), None) or ( + await self.active_session()) + except DeviceUnavailable: + if loop.time() + FRAME_RETRY_S >= deadline: + raise + await asyncio.sleep(FRAME_RETRY_S) + target = self._sessions[session] self._frames[target] = shot["data"], "jpg" return shot["data"], "jpg", target, self.url_of(target) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py index 1a25df719..e06bdb77c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py @@ -400,24 +400,27 @@ def note(self, kind: str, body: dict) -> int: self.write(seq, {kind: body}) return seq - def read(self, after: int = 0, limit: int = READ_LIMIT) -> dict: + def read(self, after: int = 0, limit: int = READ_LIMIT, settle_through: Optional[int] = None) -> dict: """The lines after sequence number ``after``, in order, up to ``limit`` of them and to the last whose every earlier line is written or stalled: ``lines``, ``last``, the sequence number of that last one, and ``more`` - when the limit left some.""" - complete, picked, more = self._picked(after, limit) + when the limit left some. An action begun at or before ``settle_through`` and still going counts as stalled + now, as a recording that ends while an action is in flight needs.""" + complete, picked, more = self._picked(after, limit, settle_through) return self._loaded(after, complete, picked, more) - async def read_async(self, after: int = 0, limit: int = READ_LIMIT) -> dict: + async def read_async(self, after: int = 0, limit: int = READ_LIMIT, settle_through: Optional[int] = None) -> dict: """``read``, its file read in a thread.""" - complete, picked, more = self._picked(after, limit) + complete, picked, more = self._picked(after, limit, settle_through) return await asyncio.to_thread(self._loaded, after, complete, picked, more) - def _picked(self, after: int, limit: int) -> tuple[int, list[tuple[int, int, int]], bool]: + def _picked(self, after: int, limit: int, settle_through: Optional[int] = None + ) -> tuple[int, list[tuple[int, int, int]], bool]: """Where the lines a read returns are: the last sequence number it may return, and each line's number, offset and length, in order.""" now = time.monotonic() for seq, began in list(self._pending.items()): - if now - began >= STALLED_S and (begun := self._begun.get(seq)) is not None: + settled = settle_through is not None and seq <= settle_through + if (settled or now - began >= STALLED_S) and (begun := self._begun.get(seq)) is not None: self._stood_in.add(seq) self.write(seq, {"action": {**begun["action"], "unfinished": True}}, at=begun["at"]) complete = min(self._pending) - 1 if self._pending else self._seq diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py index e2f1a50ac..9e4b0eed6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py @@ -244,15 +244,18 @@ async def end(self, grant: Optional[WriteNamespaceGrant], reason: str) -> dict: return wire async def _log_lines(self) -> list[dict]: - """The log's lines from the recording's start to its end, once every one of them is written, or what's - written of them after ``LOG_WAIT_S``.""" + """The log's lines from the recording's start to its end, once every one of them is written; after + ``LOG_WAIT_S``, an action still going is written as unfinished, as it was at the end, and what's written of + them is what there is.""" manifest, log = self.manifest, self.recordings.log deadline = asyncio.get_running_loop().time() + LOG_WAIT_S wanted = manifest.log_last - manifest.log_after while True: - lines = [line for line in (await log.read_async(manifest.log_after, limit=wanted))["lines"] - if line.get("seq", 0) <= manifest.log_last] - if len(lines) >= wanted or asyncio.get_running_loop().time() >= deadline: + late = asyncio.get_running_loop().time() >= deadline + read = await log.read_async(manifest.log_after, limit=wanted, + settle_through=manifest.log_last if late else None) + lines = [line for line in read["lines"] if line.get("seq", 0) <= manifest.log_last] + if len(lines) >= wanted or late: return lines await asyncio.sleep(LOG_POLL_S) diff --git a/packages/agentenv-protocol/tests/test_browser_log.py b/packages/agentenv-protocol/tests/test_browser_log.py index 79808ff00..32a2f61ed 100644 --- a/packages/agentenv-protocol/tests/test_browser_log.py +++ b/packages/agentenv-protocol/tests/test_browser_log.py @@ -178,6 +178,19 @@ def test_an_action_still_going_after_the_limit_is_written_unfinished_and_its_out assert [line["seq"] for line in log.read()["lines"]] == [1, 2, 3] +def test_a_read_settling_through_an_entry_writes_the_actions_still_going_up_to_it_unfinished_now(tmp_path): + log = BrowserLog(tmp_path / "browser-chromium.jsonl") + early = log.begin({"via": "mcp", "tool": "browser_navigate", "acts": True, "tab": "T1"}) + late = log.begin({"via": "cdp", "method": "Runtime.evaluate", "acts": True, "tab": "T1"}) + + read = log.read(settle_through=early["seq"]) + + assert [(line["seq"], line["action"]["unfinished"]) for line in read["lines"]] == [(early["seq"], True)] + assert read["last"] == early["seq"] and log.read(after=early["seq"])["lines"] == [] + log.finish(late, ok=True) + assert [line["seq"] for line in log.read(after=early["seq"])["lines"]] == [late["seq"]] + + def test_a_restarted_server_carries_on_the_sequence_on_a_new_line_after_a_partial_one(tmp_path): path = tmp_path / "browser-chromium.jsonl" path.write_text('{"seq": 1, "at": "t", "reset": {}}\n{"seq": 2, "at": "t", "tab": {}}\n{"seq": 3, "at"') diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 497dd0100..3a23d94f8 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -112,7 +112,8 @@ async def test_a_burst_still_settling_at_the_end_gets_its_frame_before_the_last( @pytest.mark.asyncio -async def test_an_end_waits_for_entries_in_flight_and_lists_any_still_unwritten_as_missing(recorded, monkeypatch): +async def test_an_end_waits_for_entries_in_flight_writes_an_action_still_going_as_unfinished_and_lists_the_rest( + recorded, monkeypatch): log, control, uploads, recordings = recorded monkeypatch.setattr(browser_recording, "LOG_WAIT_S", 0.3) started = await recordings.start("run-1:record", _grant(), frames=False) @@ -120,11 +121,16 @@ async def test_an_end_waits_for_entries_in_flight_and_lists_any_still_unwritten_ asyncio.get_running_loop().call_later(0.1, lambda: log.finish(finishing, ok=True)) complete = await recordings.end(started["recording_id"]) started = await recordings.start("run-2:record", _grant(), frames=False) - stuck = log.begin({"via": "cdp", "who": "ada", "method": "Page.navigate", "acts": True, "tab": "T1"}) - incomplete = await recordings.end(started["recording_id"]) + going = log.begin({"via": "mcp", "who": "default", "tool": "browser_navigate", "acts": True, "tab": "T1"}) + after = _acted(log, "cdp", "Input.insertText", "T1") + held = log.reserve() + ended = await recordings.end(started["recording_id"]) + lines = [json.loads(line) for line in uploads.by_path[ + f"agentenv-browser-recordings/r1/{ended['recording_id']}/log.jsonl"].decode().splitlines()] assert complete["log_missing"] == [] and complete["log_last"] == finishing["seq"] - assert incomplete["log_missing"] == [stuck["seq"]] + assert ended["log_missing"] == [held] and [line["seq"] for line in lines] == [going["seq"], after] + assert lines[0]["action"]["unfinished"] is True @pytest.mark.asyncio @@ -324,6 +330,35 @@ async def test_a_tab_that_closes_while_its_frame_is_asked_again_is_framed_as_the assert framed[:3] == held[:3] and held[2] == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T2"] +@pytest.mark.asyncio +async def test_a_frame_whose_session_is_gone_is_taken_on_the_session_its_tab_was_attached_on_anew(): + pipe = RefusingPipe(refusals=1, why="Page.captureScreenshot: Session with given id not found") + control = _one_tab(pipe) + + def reattach() -> None: + pipe.emit("Target.detachedFromTarget", {"sessionId": "s-T1"}) + pipe.emit("Target.attachedToTarget", {"sessionId": "s-T1b", "targetInfo": {"targetId": "T1", "type": "page"}}) + asyncio.get_running_loop().call_later(0.05, reattach) + + _, _, tab, _ = await control.frame("T1", load_wait_s=0) + + assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] + + +@pytest.mark.asyncio +async def test_a_frame_whose_only_tab_is_attached_anew_only_after_a_retry_waits_for_it_within_its_budget(): + pipe = RefusingPipe(refusals=1, why="Page.captureScreenshot: Session with given id not found") + control = _one_tab(pipe) + loop = asyncio.get_running_loop() + loop.call_later(0.05, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T1"}) + loop.call_later(0.6, pipe.emit, "Target.attachedToTarget", + {"sessionId": "s-T1b", "targetInfo": {"targetId": "T1", "type": "page"}}) + + _, _, tab, _ = await control.frame("T1", load_wait_s=0) + + assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] + + @pytest.mark.asyncio async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once(): pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error")