From f056e5cc74a9925db60f2557a67ff999717074a5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 14:24:03 -0800 Subject: [PATCH] fix(browser-recording): a frame whose tab closes mid-capture is taken of the tab left A capture asked of a tab that then closes is never answered, so its frame waited out FRAME_CAPTURE_S (10 s) and went missing, failing the run as an infrastructure failure. #650's retry catches the browser refusing a capture of a session already gone; this is the capture already in flight. While a frame's capture waits, the control now checks that the tab's session is still there. If it went away, the capture is dropped and asked again, within the same budget, of the tab the browser has then. Adds two integration tests, each reproducing one failure against a real browser computer: - a recording that ends while an agent's command is in flight, which #650 fixed: it fails on the main before #650 - a tab that closes while its frame's capture waits: it fails on main and on #650, and passes here Co-Authored-By: Claude Opus 5.5 --- .../website_browser/control.py | 15 +++- .../tests/test_browser_recording.py | 31 +++++++ .../computer/browser_computer_test.py | 82 +++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index 4f0c360b7..05a480013 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -85,6 +85,10 @@ PASSWORD_CHECK_TIMEOUT_S = 2.0 +class _TabGone(Exception): + """The tab a capture was asked of went away while the capture was waiting: the browser answers it never.""" + + class BrowserControl: """agentenv's control of a browser over its pipe: at each launch it attaches to every page with flattened sessions and keeps a record of the browser since this server started, its open tabs in opened order, those closed and who @@ -750,6 +754,10 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s except DevToolsError as e: if not any(why in str(e) for why in FOR_THE_MOMENT) or loop.time() + FRAME_RETRY_S >= deadline: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None + except _TabGone: + if loop.time() + FRAME_RETRY_S >= deadline: + raise DeviceUnavailable(f"the tab went away while its frame was taken, {FRAME_CAPTURE_S:g} s in all" + ) from None except ConnectionError as e: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None await asyncio.sleep(FRAME_RETRY_S) @@ -776,7 +784,8 @@ def _held_frame(self, target: str) -> tuple[str, str, Optional[str], Optional[st async def _nudged_capture(self, session: str, timeout_s: float) -> dict: """The tab's JPEG from the first of its captures to answer: a second follows one unanswered after ``NUDGE_S``, - since a tab behind the active one answers only then, all within ``timeout_s``.""" + since a tab behind the active one answers only then, all within ``timeout_s``; _TabGone once the tab's session + is gone with none answered, as a closed tab's captures never are.""" loop = asyncio.get_running_loop() deadline = loop.time() + timeout_s @@ -791,11 +800,13 @@ def capture() -> asyncio.Future: if not done: captures.add(capture()) while True: - done, captures = await asyncio.wait(captures, return_when=asyncio.FIRST_COMPLETED) + done, captures = await asyncio.wait(captures, timeout=FRAME_RETRY_S, return_when=asyncio.FIRST_COMPLETED) if answers := [c.result() for c in done if c.exception() is None]: return answers[0] if not captures: raise next(iter(done)).exception() + if session not in self._sessions: + raise _TabGone() finally: for pending in captures: pending.cancel() diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 3a23d94f8..e3376c88f 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -359,6 +359,37 @@ async def test_a_frame_whose_only_tab_is_attached_anew_only_after_a_retry_waits_ assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] +class HangingPipe(Pipe): + """A pipe whose captures of the sessions in ``hanging`` never answer, as a tab's that closes mid-capture.""" + + def __init__(self, *hanging: str) -> None: + super().__init__() + self.hanging = set(hanging) + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Page.captureScreenshot" and session_id in self.hanging: + self.captured.append((session_id, params, asyncio.get_running_loop().time())) + await asyncio.sleep(timeout_s) + raise asyncio.TimeoutError() + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + +@pytest.mark.asyncio +async def test_a_frame_whose_tab_closes_while_its_capture_waits_is_taken_of_the_tab_left_at_once(): + pipe = HangingPipe("s-T2") + control = BrowserControl(pipe, "agent", (1280, 800)) + for tab in ("T1", "T2"): + pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": tab, "type": "page", "url": "about:blank"}}) + pipe.emit("Target.attachedToTarget", {"sessionId": f"s-{tab}", "targetInfo": {"targetId": tab, "type": "page"}}) + loop = asyncio.get_running_loop() + loop.call_later(0.1, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T2"}) + started = loop.time() + + _, _, tab, _ = await control.frame("T2", load_wait_s=0) + + assert tab == "T1" and pipe.captured[-1][0] == "s-T1" and loop.time() - started < 2 + + @pytest.mark.asyncio async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once(): pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error") diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index bee44425c..4ad96acb0 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1375,6 +1375,88 @@ async def _cdp_agent_acts_behind(ws_url: str) -> str: return target +@pytest.mark.asyncio +async def test_a_browser_recording_that_ends_while_an_agents_command_is_in_flight_logs_it_unfinished(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(instance_id=f"run-{uuid.uuid4().hex[:8]}", deployed_envs=[deployed]) + try: + context = await StartBrowserRecordingTaskStep(id="start", version=None, env_id=browser_computer.id, + name="flight").execute(context) + ws_url = (await _cdp(deployed, "pi"))["ws_url"] + async with _cdp_calls(ws_url, reply_s=300) as waiting, _cdp_calls(ws_url) as call: + target = await _page_target(call) + held = (await waiting("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"] + hanging = asyncio.ensure_future(waiting("Runtime.evaluate", { + "expression": "new Promise((done) => setTimeout(done, 120000))", "awaitPromise": True}, held)) + await asyncio.sleep(1) + session = (await call("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"] + for n in range(3): + await call("Runtime.evaluate", {"expression": f"document.body.append('after {n}')"}, session) + context = await EndBrowserRecordingTaskStep(id="end", version=None, name="flight", + fail_task_on_error=False).execute(context) + hanging.cancel() + finally: + report = await teardown_run(context) + grant_server(None, None).close() + entry = context.metadata[BROWSER_RECORDINGS]["flight"] + lines = _recorded_log(entry) + + evaluated = [line["action"] for line in lines if (line.get("action") or {}).get("method") == "Runtime.evaluate" + and line["action"].get("who") == "pi"] + assert not report.failed and (entry["state"], entry["log_missing"]) == ("ended", 0), entry + assert [bool(action.get("unfinished")) for action in evaluated] == [True, False, False, False], evaluated + + +@pytest.mark.asyncio +async def test_a_browser_recording_frames_the_active_tab_when_the_agents_tab_closes_mid_capture(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(instance_id=f"run-{uuid.uuid4().hex[:8]}", deployed_envs=[deployed]) + try: + context = await StartBrowserRecordingTaskStep(id="start", version=None, env_id=browser_computer.id, + name="closing").execute(context) + async with _cdp_calls((await _cdp(deployed, "pi"))["ws_url"]) as call: + tab = (await call("Target.createTarget", {"url": BEHIND[0]}))["result"]["targetId"] + session = (await call("Target.attachToTarget", {"targetId": tab, "flatten": True}))["result"]["sessionId"] + await asyncio.sleep(2) + await call("Runtime.evaluate", {"expression": + "setTimeout(() => { const t = Date.now(); while (Date.now() - t < 8000) {} }, 200)"}, session) + await asyncio.sleep(1.5) + await call("Target.closeTarget", {"targetId": tab}) + await asyncio.sleep(3) + context = await EndBrowserRecordingTaskStep(id="end", version=None, name="closing", + fail_task_on_error=False).execute(context) + finally: + report = await teardown_run(context) + grant_server(None, None).close() + entry = context.metadata[BROWSER_RECORDINGS]["closing"] + manifest = _recorded_manifest(entry) + + frames = manifest["frames"] + missing = [(frame["index"], frame.get("error")) for frame in frames if not frame["uploaded"]] + assert not report.failed and (entry["state"], missing) == ("ended", []), missing + + +async def _page_target(call) -> str: + """The first page the browser has open, as a CDP agent finds it.""" + targets = (await call("Target.getTargets", {}))["result"]["targetInfos"] + return next(t["targetId"] for t in targets if t["type"] == "page") + + +def _recorded_manifest(entry: dict) -> dict: + universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) + return json.loads(get_config().get_object_store().get(Artifact.get(universe.file_artifact_ids["manifest.json"]).object_url)) + + +def _recorded_log(entry: dict) -> list[dict]: + universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) + text = get_config().get_object_store().get(Artifact.get(universe.file_artifact_ids["log.jsonl"]).object_url).decode() + return [json.loads(line) for line in text.splitlines() if line.strip()] + + async def _mcp(deployed, work): """What ``work`` makes of one MCP session on the computer, as a role-less agent opens one.""" async with streamablehttp_client(deployed.mcp_url) as (read, write, _):