diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index c55db6c60..4ed8212f4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -64,6 +64,9 @@ NORM = 1000 #: How long the probe records, and the frame rate it asks for. RECORD_S = 2.0 +#: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag +#: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. +BUSY_DESK_ATTEMPTS = 2 RECORD_FPS = 5 #: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer. READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();" @@ -165,15 +168,22 @@ async def full(self) -> None: # Recording - async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool: - try: - problem = await test() - except NotApplicable as e: - self._record(check, name, "not_applicable", str(e), extensions) - return True - except Exception as e: - problem = f"{type(e).__name__}: {e}" - self._record(check, name, "failed" if problem else "passed", problem or "", extensions) + async def check(self, check: str, name: str, test: Test, *extensions: str, attempts: int = 1) -> bool: + """Run ``test`` up to ``attempts`` times until it passes; a pass after a failed try says what that try found.""" + tried: list[str] = [] + while True: + try: + problem = await test() + except NotApplicable as e: + self._record(check, name, "not_applicable", str(e), extensions) + return True + except Exception as e: + problem = f"{type(e).__name__}: {e}" + if problem is None or len(tried) + 1 == attempts: + break + tried.append(problem) + detail = problem or (f"passed on try {len(tried) + 1} of {attempts}; before: {'; '.join(tried)}" if tried else "") + self._record(check, name, "failed" if problem else "passed", detail, extensions) return problem is None async def _guarded(self, check: str, section: Callable[[], Awaitable[None]]) -> None: @@ -837,7 +847,7 @@ async def held() -> str | None: await self.check("P6", "Unicode and mixed-case text arrives whole", unicode) await self.check("P6", "typed text arrives as key events", events) - await self.check("P6", "a chord selects all, and Backspace clears it", chord) + await self.check("P6", "a chord selects all, and Backspace clears it", chord, attempts=BUSY_DESK_ATTEMPTS) await self.check("P6", "a repeated key presses that many times", repeat) await self.check("P6", "tabs and newlines arrive as typed", whitespace) await self.check("P6", "typing leaves the clipboard as it was", clipboard) @@ -865,8 +875,9 @@ async def drag() -> str | None: start, end, waypoint = (50, 50), (300, 250), (200, 100) points = [start, waypoint, end] if int(self.features.get("max_drag_points", 2)) >= 3 else [start, end] path = [list(self._at("pad", scale, x, y)) for x, y in points] + before = (await self.state())["drags"] await self.act("computer_drag", {"path": path, "duration_ms": 400, "frame_id": frame}) - ok, state = await self.until(lambda s: bool(s["drags"])) + ok, state = await self.until(lambda s: s["drags"] != before) if not ok: return "no drag reached the pad" stroke = state["drags"][-1] @@ -878,7 +889,8 @@ async def drag() -> str | None: await self.check("P7", "a scroll moves a list", scroll_list) await self.check("P7", "scrolls up, down, left and right each arrive that many times", directions) - await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag) + await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag, + attempts=BUSY_DESK_ATTEMPTS) async def _pointer_input(self) -> None: if not self.driver.declares(COMPUTER_POINTER): diff --git a/packages/agentenv-protocol/tests/test_computer_conformance.py b/packages/agentenv-protocol/tests/test_computer_conformance.py index 0adacfd7e..405e3187b 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance.py @@ -4,6 +4,7 @@ fronts that exports is asked for its export once; the report's verdicts and the CLI.""" from __future__ import annotations +import asyncio import importlib import json import subprocess @@ -89,6 +90,38 @@ async def test_a_driver_given_a_control_token_passes_with_it_and_refuses_calls_w assert not unauthorized.passed and "401 unauthorized" in _failed(unauthorized)[0] +#: How long after its drag the desk of ``StarvedOnce`` saves the stroke: after the probe first looks, before it gives up. +STROKE_SAVED_AFTER_S = 1.0 +DRAG_CHECK = "a drag starts and ends where it says, through its waypoints" + + +class StarvedOnce(FixtureDriver): + """A desk starved through its first drag, which it sees as the press, one motion and the release, and slow to save + the next, which it records ``STROKE_SAVED_AFTER_S`` after the drag.""" + + drags = 0 + + @drag + async def drag_along(self, path, button, duration_ms, modifiers): + await super(FixtureDriver, self).drag_along(path, button, duration_ms, modifiers) + self.drags += 1 + if self.drags > 1: + asyncio.get_running_loop().call_later(STROKE_SAVED_AFTER_S, self.desk.drag, path) + return + start, end = self.desk._in("pad", *path[0]), self.desk._in("pad", *path[-1]) + self.desk.state["drags"].append({"start": list(start), "end": list(end), "points": 3, + "path": [list(start), list(end), list(end)]}) + + +@pytest.mark.asyncio +async def test_a_drag_tried_again_is_judged_by_its_own_stroke_not_the_one_before(home): + report = await _probe(StarvedOnce()) + + [check] = [c for c in report.checks if c.name == DRAG_CHECK] + assert check.status == "passed" and check.detail.startswith("passed on try 2 of 2; before: "), check + assert "in 3 points" in check.detail + + class StaleFrame(FixtureDriver): """Its first ``stale`` captures after a click are from before the click, as when an app records a click before it repaints.""" diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_attempts.py b/packages/agentenv-protocol/tests/test_computer_conformance_attempts.py new file mode 100644 index 000000000..af0bdc4c4 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance_attempts.py @@ -0,0 +1,55 @@ +"""A check whose effect a busy desk can miss is tried again before it fails, and a pass on a later try says what the +earlier one found; any other check is tried once.""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from agentenv_protocol.computer.conformance import checks + +STARVED = "the stroke was [50, 50] to [300, 250] in 3 points" + + +def _probe() -> checks.Probe: + return checks.Probe(SimpleNamespace(params=lambda uri: {}), session=None, ready_timeout_s=1) + + +def _answering(*answers: str | None): + calls: list[int] = [] + pending = iter(answers) + + async def test() -> str | None: + calls.append(len(calls) + 1) + return next(pending) + + return test, calls + + +@pytest.mark.asyncio +async def test_a_check_tried_again_passes_saying_what_its_first_try_found(): + probe, (drag, calls) = _probe(), _answering(STARVED, None) + + assert await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS) + + [result] = probe.results + assert (result.status, result.detail, calls) == ("passed", f"passed on try 2 of 2; before: {STARVED}", [1, 2]) + + +@pytest.mark.asyncio +async def test_a_check_that_fails_every_try_fails_with_what_its_last_try_found(): + probe, (drag, calls) = _probe(), _answering(STARVED, "the stroke missed its waypoint (200, 100)") + + assert not await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS) + + [result] = probe.results + assert (result.status, result.detail, calls) == ("failed", "the stroke missed its waypoint (200, 100)", [1, 2]) + + +@pytest.mark.asyncio +async def test_any_other_check_is_tried_once(): + probe, (test, calls) = _probe(), _answering("the fixture shows 'abc'", None) + + assert not await probe.check("P6", "typed text arrives", test) + + assert (probe.results[0].status, calls) == ("failed", [1]) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_checks.py b/packages/agentenv-protocol/tests/test_computer_conformance_checks.py index f44521b13..ec3fb3cd8 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_checks.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_checks.py @@ -252,7 +252,16 @@ async def test_crops_the_driver_refuses_fail_p3_and_p4(monkeypatch): ]) async def test_a_drag_that_doesnt_arrive_whole_fails_p7(drags, problem, monkeypatch): monkeypatch.setattr(checks, "SETTLE_S", 0.01) - p = fixture({"drags": drags, "scroll_top": 0, "wheel": {}}) + state = {"drags": [], "scroll_top": 0, "wheel": {}} + p = fixture(state) + answer = p.session.call + + def call(name, args): + if name == "computer_drag": + state["drags"] = state["drags"] + drags + return answer(name, args) + + p.session.call = call await p._scroll_and_drag() assert details(p)["a drag starts and ends where it says, through its waypoints"] == ("failed", problem)