From 0612eb960f4be59ecbf63f5fc8c30730694fd2f4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 11:35:57 -0800 Subject: [PATCH 1/3] fix(computers): the probe tries a drag or a chord once more when a busy desk may have missed it Gate 3's P7 drag check (reference driver) and P6 chord check (cua-driver) failed now and then on busy runners, which failed the compose and every test that needed that computer. Neither was the driver: the X11 driver moves through each step of a drag with a pause, but a Tk desk starved through all 400 ms of it sees the motion as one event ("in 3 points": the press, one motion, the release), so its waypoint is lost; and keys sent while the click that focuses a box still waits go wherever focus was. Each of the two now gets a second try before it fails, and a pass on its second try says in its detail what the first found, so a driver that only sometimes manages it shows. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 33 +++++++---- .../test_computer_conformance_attempts.py | 55 +++++++++++++++++++ 2 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 packages/agentenv-protocol/tests/test_computer_conformance_attempts.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index c55db6c60..c15a261b3 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -64,6 +64,9 @@ NORM = 1000 #: How long the probe records, and the frame rate it asks for. RECORD_S = 2.0 +#: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag +#: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. +BUSY_DESK_ATTEMPTS = 2 RECORD_FPS = 5 #: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer. READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();" @@ -165,15 +168,22 @@ async def full(self) -> None: # Recording - async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool: - try: - problem = await test() - except NotApplicable as e: - self._record(check, name, "not_applicable", str(e), extensions) - return True - except Exception as e: - problem = f"{type(e).__name__}: {e}" - self._record(check, name, "failed" if problem else "passed", problem or "", extensions) + async def check(self, check: str, name: str, test: Test, *extensions: str, attempts: int = 1) -> bool: + """Run ``test`` up to ``attempts`` times until it passes; a pass after a failed try says what that try found.""" + tried: list[str] = [] + while True: + try: + problem = await test() + except NotApplicable as e: + self._record(check, name, "not_applicable", str(e), extensions) + return True + except Exception as e: + problem = f"{type(e).__name__}: {e}" + if problem is None or len(tried) + 1 == attempts: + break + tried.append(problem) + detail = problem or (f"passed on try {len(tried) + 1} of {attempts}; before: {'; '.join(tried)}" if tried else "") + self._record(check, name, "failed" if problem else "passed", detail, extensions) return problem is None async def _guarded(self, check: str, section: Callable[[], Awaitable[None]]) -> None: @@ -837,7 +847,7 @@ async def held() -> str | None: await self.check("P6", "Unicode and mixed-case text arrives whole", unicode) await self.check("P6", "typed text arrives as key events", events) - await self.check("P6", "a chord selects all, and Backspace clears it", chord) + await self.check("P6", "a chord selects all, and Backspace clears it", chord, attempts=BUSY_DESK_ATTEMPTS) await self.check("P6", "a repeated key presses that many times", repeat) await self.check("P6", "tabs and newlines arrive as typed", whitespace) await self.check("P6", "typing leaves the clipboard as it was", clipboard) @@ -878,7 +888,8 @@ async def drag() -> str | None: await self.check("P7", "a scroll moves a list", scroll_list) await self.check("P7", "scrolls up, down, left and right each arrive that many times", directions) - await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag) + await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag, + attempts=BUSY_DESK_ATTEMPTS) async def _pointer_input(self) -> None: if not self.driver.declares(COMPUTER_POINTER): diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_attempts.py b/packages/agentenv-protocol/tests/test_computer_conformance_attempts.py new file mode 100644 index 000000000..af0bdc4c4 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance_attempts.py @@ -0,0 +1,55 @@ +"""A check whose effect a busy desk can miss is tried again before it fails, and a pass on a later try says what the +earlier one found; any other check is tried once.""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from agentenv_protocol.computer.conformance import checks + +STARVED = "the stroke was [50, 50] to [300, 250] in 3 points" + + +def _probe() -> checks.Probe: + return checks.Probe(SimpleNamespace(params=lambda uri: {}), session=None, ready_timeout_s=1) + + +def _answering(*answers: str | None): + calls: list[int] = [] + pending = iter(answers) + + async def test() -> str | None: + calls.append(len(calls) + 1) + return next(pending) + + return test, calls + + +@pytest.mark.asyncio +async def test_a_check_tried_again_passes_saying_what_its_first_try_found(): + probe, (drag, calls) = _probe(), _answering(STARVED, None) + + assert await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS) + + [result] = probe.results + assert (result.status, result.detail, calls) == ("passed", f"passed on try 2 of 2; before: {STARVED}", [1, 2]) + + +@pytest.mark.asyncio +async def test_a_check_that_fails_every_try_fails_with_what_its_last_try_found(): + probe, (drag, calls) = _probe(), _answering(STARVED, "the stroke missed its waypoint (200, 100)") + + assert not await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS) + + [result] = probe.results + assert (result.status, result.detail, calls) == ("failed", "the stroke missed its waypoint (200, 100)", [1, 2]) + + +@pytest.mark.asyncio +async def test_any_other_check_is_tried_once(): + probe, (test, calls) = _probe(), _answering("the fixture shows 'abc'", None) + + assert not await probe.check("P6", "typed text arrives", test) + + assert (probe.results[0].status, calls) == ("failed", [1]) From 643b2892c6f5a711ec59e04c240b14ea44e04183 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 11:40:52 -0800 Subject: [PATCH 2/3] fix(computers): a drag tried again is judged by its own stroke, not the one before The drag check waited for any stroke, so on its second try the first try's stroke, still in the desk's state, answered at once and was judged again (Greptile). It now notes the desk's strokes before it drags and waits for a new one. A probe test drives a desk starved through its first drag that saves its second a second after the drag: the check passes on its second try, and failed without this. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 3 +- .../tests/test_computer_conformance.py | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index c15a261b3..4ed8212f4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -875,8 +875,9 @@ async def drag() -> str | None: start, end, waypoint = (50, 50), (300, 250), (200, 100) points = [start, waypoint, end] if int(self.features.get("max_drag_points", 2)) >= 3 else [start, end] path = [list(self._at("pad", scale, x, y)) for x, y in points] + before = (await self.state())["drags"] await self.act("computer_drag", {"path": path, "duration_ms": 400, "frame_id": frame}) - ok, state = await self.until(lambda s: bool(s["drags"])) + ok, state = await self.until(lambda s: s["drags"] != before) if not ok: return "no drag reached the pad" stroke = state["drags"][-1] diff --git a/packages/agentenv-protocol/tests/test_computer_conformance.py b/packages/agentenv-protocol/tests/test_computer_conformance.py index 0adacfd7e..405e3187b 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance.py @@ -4,6 +4,7 @@ fronts that exports is asked for its export once; the report's verdicts and the CLI.""" from __future__ import annotations +import asyncio import importlib import json import subprocess @@ -89,6 +90,38 @@ async def test_a_driver_given_a_control_token_passes_with_it_and_refuses_calls_w assert not unauthorized.passed and "401 unauthorized" in _failed(unauthorized)[0] +#: How long after its drag the desk of ``StarvedOnce`` saves the stroke: after the probe first looks, before it gives up. +STROKE_SAVED_AFTER_S = 1.0 +DRAG_CHECK = "a drag starts and ends where it says, through its waypoints" + + +class StarvedOnce(FixtureDriver): + """A desk starved through its first drag, which it sees as the press, one motion and the release, and slow to save + the next, which it records ``STROKE_SAVED_AFTER_S`` after the drag.""" + + drags = 0 + + @drag + async def drag_along(self, path, button, duration_ms, modifiers): + await super(FixtureDriver, self).drag_along(path, button, duration_ms, modifiers) + self.drags += 1 + if self.drags > 1: + asyncio.get_running_loop().call_later(STROKE_SAVED_AFTER_S, self.desk.drag, path) + return + start, end = self.desk._in("pad", *path[0]), self.desk._in("pad", *path[-1]) + self.desk.state["drags"].append({"start": list(start), "end": list(end), "points": 3, + "path": [list(start), list(end), list(end)]}) + + +@pytest.mark.asyncio +async def test_a_drag_tried_again_is_judged_by_its_own_stroke_not_the_one_before(home): + report = await _probe(StarvedOnce()) + + [check] = [c for c in report.checks if c.name == DRAG_CHECK] + assert check.status == "passed" and check.detail.startswith("passed on try 2 of 2; before: "), check + assert "in 3 points" in check.detail + + class StaleFrame(FixtureDriver): """Its first ``stale`` captures after a click are from before the click, as when an app records a click before it repaints.""" From 7c4867c323cd86997f4f41f7003723f2c3a0695a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 11:51:27 -0800 Subject: [PATCH 3/3] test: the drag check's unit test has its desk record the stroke when the drag happens test_a_drag_that_doesnt_arrive_whole_fails_p7 served a desk that held its stroke before any drag, which the check, now waiting for a stroke the drag adds, read as no drag at all (CodeBuild on #671). Its desk now records the stroke as a real one does, on the drag. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_computer_conformance_checks.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_checks.py b/packages/agentenv-protocol/tests/test_computer_conformance_checks.py index f44521b13..ec3fb3cd8 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_checks.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_checks.py @@ -252,7 +252,16 @@ async def test_crops_the_driver_refuses_fail_p3_and_p4(monkeypatch): ]) async def test_a_drag_that_doesnt_arrive_whole_fails_p7(drags, problem, monkeypatch): monkeypatch.setattr(checks, "SETTLE_S", 0.01) - p = fixture({"drags": drags, "scroll_top": 0, "wheel": {}}) + state = {"drags": [], "scroll_top": 0, "wheel": {}} + p = fixture(state) + answer = p.session.call + + def call(name, args): + if name == "computer_drag": + state["drags"] = state["drags"] + drags + return answer(name, args) + + p.session.call = call await p._scroll_and_drag() assert details(p)["a drag starts and ends where it says, through its waypoints"] == ("failed", problem)