Skip to content

Commit 59bf426

Browse files
fix(computers): the probe tries a drag or a chord once more when a busy desk may have missed it (#671)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0faae73 commit 59bf426

4 files changed

Lines changed: 122 additions & 13 deletions

File tree

‎packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@
6464
NORM = 1000
6565
#: How long the probe records, and the frame rate it asks for.
6666
RECORD_S = 2.0
67+
#: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag
68+
#: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was.
69+
BUSY_DESK_ATTEMPTS = 2
6770
RECORD_FPS = 5
6871
#: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer.
6972
READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();"
@@ -165,15 +168,22 @@ async def full(self) -> None:
165168

166169
# Recording
167170

168-
async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool:
169-
try:
170-
problem = await test()
171-
except NotApplicable as e:
172-
self._record(check, name, "not_applicable", str(e), extensions)
173-
return True
174-
except Exception as e:
175-
problem = f"{type(e).__name__}: {e}"
176-
self._record(check, name, "failed" if problem else "passed", problem or "", extensions)
171+
async def check(self, check: str, name: str, test: Test, *extensions: str, attempts: int = 1) -> bool:
172+
"""Run ``test`` up to ``attempts`` times until it passes; a pass after a failed try says what that try found."""
173+
tried: list[str] = []
174+
while True:
175+
try:
176+
problem = await test()
177+
except NotApplicable as e:
178+
self._record(check, name, "not_applicable", str(e), extensions)
179+
return True
180+
except Exception as e:
181+
problem = f"{type(e).__name__}: {e}"
182+
if problem is None or len(tried) + 1 == attempts:
183+
break
184+
tried.append(problem)
185+
detail = problem or (f"passed on try {len(tried) + 1} of {attempts}; before: {'; '.join(tried)}" if tried else "")
186+
self._record(check, name, "failed" if problem else "passed", detail, extensions)
177187
return problem is None
178188

179189
async def _guarded(self, check: str, section: Callable[[], Awaitable[None]]) -> None:
@@ -837,7 +847,7 @@ async def held() -> str | None:
837847

838848
await self.check("P6", "Unicode and mixed-case text arrives whole", unicode)
839849
await self.check("P6", "typed text arrives as key events", events)
840-
await self.check("P6", "a chord selects all, and Backspace clears it", chord)
850+
await self.check("P6", "a chord selects all, and Backspace clears it", chord, attempts=BUSY_DESK_ATTEMPTS)
841851
await self.check("P6", "a repeated key presses that many times", repeat)
842852
await self.check("P6", "tabs and newlines arrive as typed", whitespace)
843853
await self.check("P6", "typing leaves the clipboard as it was", clipboard)
@@ -865,8 +875,9 @@ async def drag() -> str | None:
865875
start, end, waypoint = (50, 50), (300, 250), (200, 100)
866876
points = [start, waypoint, end] if int(self.features.get("max_drag_points", 2)) >= 3 else [start, end]
867877
path = [list(self._at("pad", scale, x, y)) for x, y in points]
878+
before = (await self.state())["drags"]
868879
await self.act("computer_drag", {"path": path, "duration_ms": 400, "frame_id": frame})
869-
ok, state = await self.until(lambda s: bool(s["drags"]))
880+
ok, state = await self.until(lambda s: s["drags"] != before)
870881
if not ok:
871882
return "no drag reached the pad"
872883
stroke = state["drags"][-1]
@@ -878,7 +889,8 @@ async def drag() -> str | None:
878889

879890
await self.check("P7", "a scroll moves a list", scroll_list)
880891
await self.check("P7", "scrolls up, down, left and right each arrive that many times", directions)
881-
await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag)
892+
await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag,
893+
attempts=BUSY_DESK_ATTEMPTS)
882894

883895
async def _pointer_input(self) -> None:
884896
if not self.driver.declares(COMPUTER_POINTER):

‎packages/agentenv-protocol/tests/test_computer_conformance.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
fronts that exports is asked for its export once; the report's verdicts and the CLI."""
55
from __future__ import annotations
66

7+
import asyncio
78
import importlib
89
import json
910
import subprocess
@@ -89,6 +90,38 @@ async def test_a_driver_given_a_control_token_passes_with_it_and_refuses_calls_w
8990
assert not unauthorized.passed and "401 unauthorized" in _failed(unauthorized)[0]
9091

9192

93+
#: How long after its drag the desk of ``StarvedOnce`` saves the stroke: after the probe first looks, before it gives up.
94+
STROKE_SAVED_AFTER_S = 1.0
95+
DRAG_CHECK = "a drag starts and ends where it says, through its waypoints"
96+
97+
98+
class StarvedOnce(FixtureDriver):
99+
"""A desk starved through its first drag, which it sees as the press, one motion and the release, and slow to save
100+
the next, which it records ``STROKE_SAVED_AFTER_S`` after the drag."""
101+
102+
drags = 0
103+
104+
@drag
105+
async def drag_along(self, path, button, duration_ms, modifiers):
106+
await super(FixtureDriver, self).drag_along(path, button, duration_ms, modifiers)
107+
self.drags += 1
108+
if self.drags > 1:
109+
asyncio.get_running_loop().call_later(STROKE_SAVED_AFTER_S, self.desk.drag, path)
110+
return
111+
start, end = self.desk._in("pad", *path[0]), self.desk._in("pad", *path[-1])
112+
self.desk.state["drags"].append({"start": list(start), "end": list(end), "points": 3,
113+
"path": [list(start), list(end), list(end)]})
114+
115+
116+
@pytest.mark.asyncio
117+
async def test_a_drag_tried_again_is_judged_by_its_own_stroke_not_the_one_before(home):
118+
report = await _probe(StarvedOnce())
119+
120+
[check] = [c for c in report.checks if c.name == DRAG_CHECK]
121+
assert check.status == "passed" and check.detail.startswith("passed on try 2 of 2; before: "), check
122+
assert "in 3 points" in check.detail
123+
124+
92125
class StaleFrame(FixtureDriver):
93126
"""Its first ``stale`` captures after a click are from before the click, as when an app records a click before it
94127
repaints."""
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
"""A check whose effect a busy desk can miss is tried again before it fails, and a pass on a later try says what the
2+
earlier one found; any other check is tried once."""
3+
from __future__ import annotations
4+
5+
from types import SimpleNamespace
6+
7+
import pytest
8+
9+
from agentenv_protocol.computer.conformance import checks
10+
11+
STARVED = "the stroke was [50, 50] to [300, 250] in 3 points"
12+
13+
14+
def _probe() -> checks.Probe:
15+
return checks.Probe(SimpleNamespace(params=lambda uri: {}), session=None, ready_timeout_s=1)
16+
17+
18+
def _answering(*answers: str | None):
19+
calls: list[int] = []
20+
pending = iter(answers)
21+
22+
async def test() -> str | None:
23+
calls.append(len(calls) + 1)
24+
return next(pending)
25+
26+
return test, calls
27+
28+
29+
@pytest.mark.asyncio
30+
async def test_a_check_tried_again_passes_saying_what_its_first_try_found():
31+
probe, (drag, calls) = _probe(), _answering(STARVED, None)
32+
33+
assert await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS)
34+
35+
[result] = probe.results
36+
assert (result.status, result.detail, calls) == ("passed", f"passed on try 2 of 2; before: {STARVED}", [1, 2])
37+
38+
39+
@pytest.mark.asyncio
40+
async def test_a_check_that_fails_every_try_fails_with_what_its_last_try_found():
41+
probe, (drag, calls) = _probe(), _answering(STARVED, "the stroke missed its waypoint (200, 100)")
42+
43+
assert not await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS)
44+
45+
[result] = probe.results
46+
assert (result.status, result.detail, calls) == ("failed", "the stroke missed its waypoint (200, 100)", [1, 2])
47+
48+
49+
@pytest.mark.asyncio
50+
async def test_any_other_check_is_tried_once():
51+
probe, (test, calls) = _probe(), _answering("the fixture shows 'abc'", None)
52+
53+
assert not await probe.check("P6", "typed text arrives", test)
54+
55+
assert (probe.results[0].status, calls) == ("failed", [1])

‎packages/agentenv-protocol/tests/test_computer_conformance_checks.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,16 @@ async def test_crops_the_driver_refuses_fail_p3_and_p4(monkeypatch):
252252
])
253253
async def test_a_drag_that_doesnt_arrive_whole_fails_p7(drags, problem, monkeypatch):
254254
monkeypatch.setattr(checks, "SETTLE_S", 0.01)
255-
p = fixture({"drags": drags, "scroll_top": 0, "wheel": {}})
255+
state = {"drags": [], "scroll_top": 0, "wheel": {}}
256+
p = fixture(state)
257+
answer = p.session.call
258+
259+
def call(name, args):
260+
if name == "computer_drag":
261+
state["drags"] = state["drags"] + drags
262+
return answer(name, args)
263+
264+
p.session.call = call
256265
await p._scroll_and_drag()
257266
assert details(p)["a drag starts and ends where it says, through its waypoints"] == ("failed", problem)
258267

0 commit comments

Comments
 (0)