From f1091394f47d114e77cbe77c8d0802094dc0a2ba Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 13:29:40 -0800 Subject: [PATCH 1/2] fix(browser-recording): a recording ends complete when an action is still going, or a tab's session went away Online-Mind2Web's agreement runs lost two runs to the recording's end: - An action still going when a run ended held back every later log line. The log stands one in as unfinished only after STALLED_S (60 s), and an end waits LOG_WAIT_S (10 s), so that run's log was missing 357 entries. Now, once its wait is up, an end writes the actions still going up to its last entry as unfinished (BrowserLog.read(settle_through=...)). The log then holds every line to the end, an action in flight marked as such. - A frame refused with "Session with given id not found", its tab detached or attached anew, was missing at once. It's now asked again within the frame budget, on the session the tab has then, else the active tab's. Co-Authored-By: Claude Opus 5.5 --- .../website_browser/control.py | 11 +++---- .../agentenv_protocol/website_browser/log.py | 17 ++++++----- .../website_browser/recording.py | 13 +++++---- .../tests/test_browser_log.py | 13 +++++++++ .../tests/test_browser_recording.py | 29 ++++++++++++++++--- 5 files changed, 62 insertions(+), 21 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index 7e7bbd59b..aef4ffdfb 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -51,11 +51,12 @@ NUDGE_S = 0.5 #: How long a frame may take to capture, longer than a screenshot an agent waits on, since a frame is taken in the #: background: a page too busy to paint, or between documents mid-navigation, gets this long to show, a capture the -#: browser refused for being between documents asked again every ``FRAME_RETRY_S``. +#: browser refused for the moment asked again every ``FRAME_RETRY_S``. FRAME_CAPTURE_S = 10.0 FRAME_RETRY_S = 0.25 -#: What the browser says of a capture refused for the moment, as while a navigation swaps the tab's document. -BETWEEN_DOCUMENTS = ("Not attached to an active page", "Unable to capture screenshot") +#: What the browser says of a capture refused for the moment: while a navigation swaps the tab's document, or once the +#: session it was asked on is gone, as when the tab closes or the browser attaches it anew. +FOR_THE_MOMENT = ("Not attached to an active page", "Unable to capture screenshot", "Session with given id not found") #: How often a wait for a tab to load looks again. LOAD_POLL_S = 0.1 #: Where an agent's files to upload are, and where downloads land, under the session user's home. @@ -658,13 +659,13 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s except asyncio.TimeoutError: raise DeviceUnavailable(f"the tab painted nothing within {FRAME_CAPTURE_S:g} s") from None except DevToolsError as e: - if not any(why in str(e) for why in BETWEEN_DOCUMENTS) or loop.time() + FRAME_RETRY_S >= deadline: + if not any(why in str(e) for why in FOR_THE_MOMENT) or loop.time() + FRAME_RETRY_S >= deadline: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None except ConnectionError as e: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None await asyncio.sleep(FRAME_RETRY_S) if session not in self._sessions: - session = await self.active_session() + session = next((s for s, t in self._sessions.items() if t == target), None) or await self.active_session() target = self._sessions[session] self._frames[target] = shot["data"], "jpg" return shot["data"], "jpg", target, self.url_of(target) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py index 1a25df719..e06bdb77c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py @@ -400,24 +400,27 @@ def note(self, kind: str, body: dict) -> int: self.write(seq, {kind: body}) return seq - def read(self, after: int = 0, limit: int = READ_LIMIT) -> dict: + def read(self, after: int = 0, limit: int = READ_LIMIT, settle_through: Optional[int] = None) -> dict: """The lines after sequence number ``after``, in order, up to ``limit`` of them and to the last whose every earlier line is written or stalled: ``lines``, ``last``, the sequence number of that last one, and ``more`` - when the limit left some.""" - complete, picked, more = self._picked(after, limit) + when the limit left some. An action begun at or before ``settle_through`` and still going counts as stalled + now, as a recording that ends while an action is in flight needs.""" + complete, picked, more = self._picked(after, limit, settle_through) return self._loaded(after, complete, picked, more) - async def read_async(self, after: int = 0, limit: int = READ_LIMIT) -> dict: + async def read_async(self, after: int = 0, limit: int = READ_LIMIT, settle_through: Optional[int] = None) -> dict: """``read``, its file read in a thread.""" - complete, picked, more = self._picked(after, limit) + complete, picked, more = self._picked(after, limit, settle_through) return await asyncio.to_thread(self._loaded, after, complete, picked, more) - def _picked(self, after: int, limit: int) -> tuple[int, list[tuple[int, int, int]], bool]: + def _picked(self, after: int, limit: int, settle_through: Optional[int] = None + ) -> tuple[int, list[tuple[int, int, int]], bool]: """Where the lines a read returns are: the last sequence number it may return, and each line's number, offset and length, in order.""" now = time.monotonic() for seq, began in list(self._pending.items()): - if now - began >= STALLED_S and (begun := self._begun.get(seq)) is not None: + settled = settle_through is not None and seq <= settle_through + if (settled or now - began >= STALLED_S) and (begun := self._begun.get(seq)) is not None: self._stood_in.add(seq) self.write(seq, {"action": {**begun["action"], "unfinished": True}}, at=begun["at"]) complete = min(self._pending) - 1 if self._pending else self._seq diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py index e2f1a50ac..9e4b0eed6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py @@ -244,15 +244,18 @@ async def end(self, grant: Optional[WriteNamespaceGrant], reason: str) -> dict: return wire async def _log_lines(self) -> list[dict]: - """The log's lines from the recording's start to its end, once every one of them is written, or what's - written of them after ``LOG_WAIT_S``.""" + """The log's lines from the recording's start to its end, once every one of them is written; after + ``LOG_WAIT_S``, an action still going is written as unfinished, as it was at the end, and what's written of + them is what there is.""" manifest, log = self.manifest, self.recordings.log deadline = asyncio.get_running_loop().time() + LOG_WAIT_S wanted = manifest.log_last - manifest.log_after while True: - lines = [line for line in (await log.read_async(manifest.log_after, limit=wanted))["lines"] - if line.get("seq", 0) <= manifest.log_last] - if len(lines) >= wanted or asyncio.get_running_loop().time() >= deadline: + late = asyncio.get_running_loop().time() >= deadline + read = await log.read_async(manifest.log_after, limit=wanted, + settle_through=manifest.log_last if late else None) + lines = [line for line in read["lines"] if line.get("seq", 0) <= manifest.log_last] + if len(lines) >= wanted or late: return lines await asyncio.sleep(LOG_POLL_S) diff --git a/packages/agentenv-protocol/tests/test_browser_log.py b/packages/agentenv-protocol/tests/test_browser_log.py index 79808ff00..32a2f61ed 100644 --- a/packages/agentenv-protocol/tests/test_browser_log.py +++ b/packages/agentenv-protocol/tests/test_browser_log.py @@ -178,6 +178,19 @@ def test_an_action_still_going_after_the_limit_is_written_unfinished_and_its_out assert [line["seq"] for line in log.read()["lines"]] == [1, 2, 3] +def test_a_read_settling_through_an_entry_writes_the_actions_still_going_up_to_it_unfinished_now(tmp_path): + log = BrowserLog(tmp_path / "browser-chromium.jsonl") + early = log.begin({"via": "mcp", "tool": "browser_navigate", "acts": True, "tab": "T1"}) + late = log.begin({"via": "cdp", "method": "Runtime.evaluate", "acts": True, "tab": "T1"}) + + read = log.read(settle_through=early["seq"]) + + assert [(line["seq"], line["action"]["unfinished"]) for line in read["lines"]] == [(early["seq"], True)] + assert read["last"] == early["seq"] and log.read(after=early["seq"])["lines"] == [] + log.finish(late, ok=True) + assert [line["seq"] for line in log.read(after=early["seq"])["lines"]] == [late["seq"]] + + def test_a_restarted_server_carries_on_the_sequence_on_a_new_line_after_a_partial_one(tmp_path): path = tmp_path / "browser-chromium.jsonl" path.write_text('{"seq": 1, "at": "t", "reset": {}}\n{"seq": 2, "at": "t", "tab": {}}\n{"seq": 3, "at"') diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 497dd0100..456ed6e6e 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -112,7 +112,8 @@ async def test_a_burst_still_settling_at_the_end_gets_its_frame_before_the_last( @pytest.mark.asyncio -async def test_an_end_waits_for_entries_in_flight_and_lists_any_still_unwritten_as_missing(recorded, monkeypatch): +async def test_an_end_waits_for_entries_in_flight_writes_an_action_still_going_as_unfinished_and_lists_the_rest( + recorded, monkeypatch): log, control, uploads, recordings = recorded monkeypatch.setattr(browser_recording, "LOG_WAIT_S", 0.3) started = await recordings.start("run-1:record", _grant(), frames=False) @@ -120,11 +121,16 @@ async def test_an_end_waits_for_entries_in_flight_and_lists_any_still_unwritten_ asyncio.get_running_loop().call_later(0.1, lambda: log.finish(finishing, ok=True)) complete = await recordings.end(started["recording_id"]) started = await recordings.start("run-2:record", _grant(), frames=False) - stuck = log.begin({"via": "cdp", "who": "ada", "method": "Page.navigate", "acts": True, "tab": "T1"}) - incomplete = await recordings.end(started["recording_id"]) + going = log.begin({"via": "mcp", "who": "default", "tool": "browser_navigate", "acts": True, "tab": "T1"}) + after = _acted(log, "cdp", "Input.insertText", "T1") + held = log.reserve() + ended = await recordings.end(started["recording_id"]) + lines = [json.loads(line) for line in uploads.by_path[ + f"agentenv-browser-recordings/r1/{ended['recording_id']}/log.jsonl"].decode().splitlines()] assert complete["log_missing"] == [] and complete["log_last"] == finishing["seq"] - assert incomplete["log_missing"] == [stuck["seq"]] + assert ended["log_missing"] == [held] and [line["seq"] for line in lines] == [going["seq"], after] + assert lines[0]["action"]["unfinished"] is True @pytest.mark.asyncio @@ -324,6 +330,21 @@ async def test_a_tab_that_closes_while_its_frame_is_asked_again_is_framed_as_the assert framed[:3] == held[:3] and held[2] == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T2"] +@pytest.mark.asyncio +async def test_a_frame_whose_session_is_gone_is_taken_on_the_session_its_tab_was_attached_on_anew(): + pipe = RefusingPipe(refusals=1, why="Page.captureScreenshot: Session with given id not found") + control = _one_tab(pipe) + + def reattach() -> None: + pipe.emit("Target.detachedFromTarget", {"sessionId": "s-T1"}) + pipe.emit("Target.attachedToTarget", {"sessionId": "s-T1b", "targetInfo": {"targetId": "T1", "type": "page"}}) + asyncio.get_running_loop().call_later(0.05, reattach) + + _, _, tab, _ = await control.frame("T1", load_wait_s=0) + + assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] + + @pytest.mark.asyncio async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once(): pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error") From 1a5967d9e4684746f18f42cfaa06d64b79d2855d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 13:33:24 -0800 Subject: [PATCH 2/2] fix(browser-recording): a frame waits within its budget for a tab attached anew A frame retried after its session went away looked once for the tab's new session, else the active tab's. With the only tab detached and its new session not there yet, that raised "no tab open" with most of the frame's budget left. It now waits for a session within the budget. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/website_browser/control.py | 12 +++++++++--- .../tests/test_browser_recording.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index aef4ffdfb..e2f2f8d1b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -664,9 +664,15 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s except ConnectionError as e: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None await asyncio.sleep(FRAME_RETRY_S) - if session not in self._sessions: - session = next((s for s, t in self._sessions.items() if t == target), None) or await self.active_session() - target = self._sessions[session] + while session not in self._sessions: + try: + session = next((s for s, t in self._sessions.items() if t == target), None) or ( + await self.active_session()) + except DeviceUnavailable: + if loop.time() + FRAME_RETRY_S >= deadline: + raise + await asyncio.sleep(FRAME_RETRY_S) + target = self._sessions[session] self._frames[target] = shot["data"], "jpg" return shot["data"], "jpg", target, self.url_of(target) diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 456ed6e6e..3a23d94f8 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -345,6 +345,20 @@ def reattach() -> None: assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] +@pytest.mark.asyncio +async def test_a_frame_whose_only_tab_is_attached_anew_only_after_a_retry_waits_for_it_within_its_budget(): + pipe = RefusingPipe(refusals=1, why="Page.captureScreenshot: Session with given id not found") + control = _one_tab(pipe) + loop = asyncio.get_running_loop() + loop.call_later(0.05, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T1"}) + loop.call_later(0.6, pipe.emit, "Target.attachedToTarget", + {"sessionId": "s-T1b", "targetInfo": {"targetId": "T1", "type": "page"}}) + + _, _, tab, _ = await control.frame("T1", load_wait_s=0) + + assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] + + @pytest.mark.asyncio async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once(): pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error")