Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@
NORM = 1000
#: How long the probe records, and the frame rate it asks for.
RECORD_S = 2.0
#: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag
#: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was.
BUSY_DESK_ATTEMPTS = 2
RECORD_FPS = 5
#: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer.
READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();"
Expand Down Expand Up @@ -165,15 +168,22 @@ async def full(self) -> None:

# Recording

async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool:
try:
problem = await test()
except NotApplicable as e:
self._record(check, name, "not_applicable", str(e), extensions)
return True
except Exception as e:
problem = f"{type(e).__name__}: {e}"
self._record(check, name, "failed" if problem else "passed", problem or "", extensions)
async def check(self, check: str, name: str, test: Test, *extensions: str, attempts: int = 1) -> bool:
"""Run ``test`` up to ``attempts`` times until it passes; a pass after a failed try says what that try found."""
tried: list[str] = []
while True:
try:
problem = await test()
except NotApplicable as e:
self._record(check, name, "not_applicable", str(e), extensions)
return True
except Exception as e:
problem = f"{type(e).__name__}: {e}"
if problem is None or len(tried) + 1 == attempts:
break
tried.append(problem)
detail = problem or (f"passed on try {len(tried) + 1} of {attempts}; before: {'; '.join(tried)}" if tried else "")
self._record(check, name, "failed" if problem else "passed", detail, extensions)
return problem is None

async def _guarded(self, check: str, section: Callable[[], Awaitable[None]]) -> None:
Expand Down Expand Up @@ -837,7 +847,7 @@ async def held() -> str | None:

await self.check("P6", "Unicode and mixed-case text arrives whole", unicode)
await self.check("P6", "typed text arrives as key events", events)
await self.check("P6", "a chord selects all, and Backspace clears it", chord)
await self.check("P6", "a chord selects all, and Backspace clears it", chord, attempts=BUSY_DESK_ATTEMPTS)
await self.check("P6", "a repeated key presses that many times", repeat)
await self.check("P6", "tabs and newlines arrive as typed", whitespace)
await self.check("P6", "typing leaves the clipboard as it was", clipboard)
Expand Down Expand Up @@ -865,8 +875,9 @@ async def drag() -> str | None:
start, end, waypoint = (50, 50), (300, 250), (200, 100)
points = [start, waypoint, end] if int(self.features.get("max_drag_points", 2)) >= 3 else [start, end]
path = [list(self._at("pad", scale, x, y)) for x, y in points]
before = (await self.state())["drags"]
await self.act("computer_drag", {"path": path, "duration_ms": 400, "frame_id": frame})
ok, state = await self.until(lambda s: bool(s["drags"]))
ok, state = await self.until(lambda s: s["drags"] != before)
if not ok:
return "no drag reached the pad"
stroke = state["drags"][-1]
Expand All @@ -878,7 +889,8 @@ async def drag() -> str | None:

await self.check("P7", "a scroll moves a list", scroll_list)
await self.check("P7", "scrolls up, down, left and right each arrive that many times", directions)
await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag)
await self.check("P7", "a drag starts and ends where it says, through its waypoints", drag,
attempts=BUSY_DESK_ATTEMPTS)
Comment thread
earakely-scale marked this conversation as resolved.
Resolved

async def _pointer_input(self) -> None:
if not self.driver.declares(COMPUTER_POINTER):
Expand Down
33 changes: 33 additions & 0 deletions packages/agentenv-protocol/tests/test_computer_conformance.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
fronts that exports is asked for its export once; the report's verdicts and the CLI."""
from __future__ import annotations

import asyncio
import importlib
import json
import subprocess
Expand Down Expand Up @@ -89,6 +90,38 @@ async def test_a_driver_given_a_control_token_passes_with_it_and_refuses_calls_w
assert not unauthorized.passed and "401 unauthorized" in _failed(unauthorized)[0]


#: How long after its drag the desk of ``StarvedOnce`` saves the stroke: after the probe first looks, before it gives up.
STROKE_SAVED_AFTER_S = 1.0
DRAG_CHECK = "a drag starts and ends where it says, through its waypoints"


class StarvedOnce(FixtureDriver):
"""A desk starved through its first drag, which it sees as the press, one motion and the release, and slow to save
the next, which it records ``STROKE_SAVED_AFTER_S`` after the drag."""

drags = 0

@drag
async def drag_along(self, path, button, duration_ms, modifiers):
await super(FixtureDriver, self).drag_along(path, button, duration_ms, modifiers)
self.drags += 1
if self.drags > 1:
asyncio.get_running_loop().call_later(STROKE_SAVED_AFTER_S, self.desk.drag, path)
return
start, end = self.desk._in("pad", *path[0]), self.desk._in("pad", *path[-1])
self.desk.state["drags"].append({"start": list(start), "end": list(end), "points": 3,
"path": [list(start), list(end), list(end)]})


@pytest.mark.asyncio
async def test_a_drag_tried_again_is_judged_by_its_own_stroke_not_the_one_before(home):
report = await _probe(StarvedOnce())

[check] = [c for c in report.checks if c.name == DRAG_CHECK]
assert check.status == "passed" and check.detail.startswith("passed on try 2 of 2; before: "), check
assert "in 3 points" in check.detail


class StaleFrame(FixtureDriver):
"""Its first ``stale`` captures after a click are from before the click, as when an app records a click before it
repaints."""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""A check whose effect a busy desk can miss is tried again before it fails, and a pass on a later try says what the
earlier one found; any other check is tried once."""
from __future__ import annotations

from types import SimpleNamespace

import pytest

from agentenv_protocol.computer.conformance import checks

STARVED = "the stroke was [50, 50] to [300, 250] in 3 points"


def _probe() -> checks.Probe:
return checks.Probe(SimpleNamespace(params=lambda uri: {}), session=None, ready_timeout_s=1)


def _answering(*answers: str | None):
calls: list[int] = []
pending = iter(answers)

async def test() -> str | None:
calls.append(len(calls) + 1)
return next(pending)

return test, calls


@pytest.mark.asyncio
async def test_a_check_tried_again_passes_saying_what_its_first_try_found():
probe, (drag, calls) = _probe(), _answering(STARVED, None)

assert await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS)

[result] = probe.results
assert (result.status, result.detail, calls) == ("passed", f"passed on try 2 of 2; before: {STARVED}", [1, 2])


@pytest.mark.asyncio
async def test_a_check_that_fails_every_try_fails_with_what_its_last_try_found():
probe, (drag, calls) = _probe(), _answering(STARVED, "the stroke missed its waypoint (200, 100)")

assert not await probe.check("P7", "a drag", drag, attempts=checks.BUSY_DESK_ATTEMPTS)

[result] = probe.results
assert (result.status, result.detail, calls) == ("failed", "the stroke missed its waypoint (200, 100)", [1, 2])


@pytest.mark.asyncio
async def test_any_other_check_is_tried_once():
probe, (test, calls) = _probe(), _answering("the fixture shows 'abc'", None)

assert not await probe.check("P6", "typed text arrives", test)

assert (probe.results[0].status, calls) == ("failed", [1])
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,16 @@ async def test_crops_the_driver_refuses_fail_p3_and_p4(monkeypatch):
])
async def test_a_drag_that_doesnt_arrive_whole_fails_p7(drags, problem, monkeypatch):
monkeypatch.setattr(checks, "SETTLE_S", 0.01)
p = fixture({"drags": drags, "scroll_top": 0, "wheel": {}})
state = {"drags": [], "scroll_top": 0, "wheel": {}}
p = fixture(state)
answer = p.session.call

def call(name, args):
if name == "computer_drag":
state["drags"] = state["drags"] + drags
return answer(name, args)

p.session.call = call
await p._scroll_and_drag()
assert details(p)["a drag starts and ends where it says, through its waypoints"] == ("failed", problem)

Expand Down
Loading