Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -990,7 +990,7 @@ async def on_call_tool(self, context: Any, call_next: Callable) -> Any:

def _fastmcp_tool_result(answer: Any) -> Any:
"""A hook's ``mcp.types.CallToolResult`` as the ``ToolResult`` a fastmcp middleware chain returns, the error
flag kept, under the field spellings of mcp 1 and 2."""
flag and ``_meta`` kept, under the field spellings of mcp 1 and 2."""
from fastmcp.tools import ToolResult
# mcp 2's spellings first: on mcp 1 they are plain misses, on mcp 2 the camel-case ones are deprecation shims.
structured = getattr(answer, "structured_content", None)
Expand All @@ -999,7 +999,8 @@ def _fastmcp_tool_result(answer: Any) -> Any:
is_error = getattr(answer, "is_error", None)
if is_error is None:
is_error = getattr(answer, "isError", False)
return ToolResult(content=list(answer.content), structured_content=structured, is_error=bool(is_error))
return ToolResult(content=list(answer.content), structured_content=structured, meta=getattr(answer, "meta", None),
is_error=bool(is_error))


def _mcp_tool_annotations(annotations: Any) -> Any:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ def result(self) -> FrameData:
height=self.height,
display=DisplayData(width=self.display.width, height=self.display.height, scale=self.display.scale),
parent_frame_id=self.parent_id,
captured_at=self.captured_at.isoformat().replace("+00:00", "Z"),
page=_page_data(self.page),
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@
answers is what its schema says."""
from __future__ import annotations

from datetime import datetime
from typing import Literal, Optional

from mcp.types import CallToolResult, ImageContent, TextContent
from pydantic import BaseModel

from .operations import Effect

#: The ``_meta`` key of a tool answer that carries a frame: when the driver captured it, in real time (RFC 3339, UTC).
CAPTURED_AT_META = "agentenv.io/capturedAt"


class DisplayData(BaseModel):
width: int
Expand Down Expand Up @@ -36,12 +40,14 @@ class PageData(BaseModel):


class FrameData(BaseModel):
"""A frame as an agent sees it. When it was captured isn't part of it: the driver keeps real time, which in a
computer on the universe's clock would tell the agent the real date; the tool's answer carries it in its ``_meta``
(``CAPTURED_AT_META``) for the callers that line frames up with a recording."""
frame_id: str
width: int
height: int
display: DisplayData
parent_frame_id: Optional[str] = None
captured_at: str
page: Optional[PageData] = None
Comment thread
earakely-scale marked this conversation as resolved.
Resolved


Expand Down Expand Up @@ -118,10 +124,14 @@ class PointerListResult(BaseModel):
pointers: list[PointerData]


def send(result: BaseModel, image: tuple[str, str] | None = None) -> CallToolResult:
def send(result: BaseModel, image: tuple[str, str] | None = None,
captured_at: datetime | None = None) -> CallToolResult:
"""A result as a tool answers: the summary as the text block, then the image (base64, MIME type) when there is
one, and the result as structured content with its unset fields left out."""
one, and the result as structured content with its unset fields left out; when its frame was ``captured_at``,
that real time in its ``_meta``, which a client doesn't show its model."""
content: list = [TextContent(type="text", text=result.summary)]
if image is not None:
content.append(ImageContent(type="image", data=image[0], mimeType=image[1]))
return CallToolResult(content=content, structuredContent=result.model_dump(mode="json", exclude_none=True))
meta = {CAPTURED_AT_META: captured_at.isoformat().replace("+00:00", "Z")} if captured_at is not None else None
return CallToolResult(content=content, structuredContent=result.model_dump(mode="json", exclude_none=True),
_meta=meta)
Comment thread
earakely-scale marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra
result.summary = f"{result.summary}; the screenshot after it failed ({e.code}: {e.message})"
return send(result)
result.summary, result.frame = f"{result.summary}, then captured {new.id}{new.note()}", new.result()
return send(result, new.encode())
return send(result, new.encode(), new.captured_at)

async def _hold_for(self, ms: int, keys: list[str]) -> None:
"""Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ async def computer_screenshot(
identity=identity, parent=parent, origin=(box[0], box[1]))
summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})"
result = ScreenshotResult(summary=summary, **frame.result().model_dump())
return send(result, frame.encode(format, quality))
return send(result, frame.encode(format, quality), frame.captured_at)

@tool(annotations=_INPUT)
async def computer_click(
Expand Down Expand Up @@ -242,7 +242,7 @@ async def computer_wait(
frame = await self._computer.capture(self.caller_identity(ctx))
return send(WaitResult(summary=f"{summary}, then captured {frame.id}{frame.note()}", waited_ms=ms,
frame=frame.result()),
frame.encode())
frame.encode(), frame.captured_at)

@tool(annotations=_INPUT)
async def computer_pointer_move(
Expand Down
13 changes: 6 additions & 7 deletions packages/agentenv-protocol/tests/test_computer_frames.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

from agentenv_protocol import ROLE_META_KEY # noqa: E402
from agentenv_protocol.computer.frames import KEEP # noqa: E402
from agentenv_protocol.computer.results import CAPTURED_AT_META # noqa: E402


def _image(result) -> PILImage.Image:
Expand All @@ -29,7 +30,9 @@ async def test_a_screenshot_is_a_native_frame_with_its_geometry():
data = first.structuredContent
assert data["frame_id"] == "f1" and (data["width"], data["height"]) == (1280, 800)
assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and "parent_frame_id" not in data
assert data["captured_at"].endswith("Z") and data["summary"] == "Captured f1: 1280×800 of the 1280×800 display"
assert data["summary"] == "Captured f1: 1280×800 of the 1280×800 display"
assert "captured_at" not in data # the driver's real time, which would tell an agent the real date
assert first.meta[CAPTURED_AT_META].endswith("Z") # kept for whoever lines frames up with a recording
assert first.content[0].text == data["summary"] and first.content[1].mimeType == "image/png"
assert _image(first).size == (1280, 800)
assert second.structuredContent["frame_id"] == "f2" and second.content[1].mimeType == "image/jpeg"
Expand All @@ -49,23 +52,19 @@ async def test_a_region_is_a_native_crop_whose_points_map_back_exactly():
driver = FakeDriver(FakeScreen(1280, 800))
driver.screen.paint(700, 450, (1, 2, 3))
async with mcp_session(driver) as session:
await session.call_tool("computer_screenshot", {})
first_at = (await session.call_tool("computer_screenshot", {})).meta[CAPTURED_AT_META]
crop = await session.call_tool("computer_screenshot", {"region": [600, 400, 900, 500]})
nested = await session.call_tool("computer_screenshot", {"region": [100, 50, 200, 100], "frame_id": "f2"})
clicked = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f3"})
assert crop.structuredContent["parent_frame_id"] == "f1" and (crop.structuredContent["width"],
crop.structuredContent["height"]) == (300, 100)
assert crop.structuredContent["summary"] == "Cropped f2: 300×100 of f1 at (600, 400)"
assert crop.structuredContent["captured_at"] == (await _captured_at(driver, "f1"))
assert crop.meta[CAPTURED_AT_META] == first_at # a crop was captured when its frame was
assert _image(crop).getpixel((100, 50))[:3] == (1, 2, 3)
assert nested.structuredContent["parent_frame_id"] == "f2" and _image(nested).getpixel((0, 0))[:3] == (1, 2, 3)
assert not clicked.isError and driver.screen.log[-1][1:3] == (701, 451)


async def _captured_at(driver: FakeDriver, frame_id: str) -> str:
return driver._computer.frames.get(frame_id, "default").result().captured_at


@pytest.mark.asyncio
@pytest.mark.parametrize("args, code", [
({"frame_id": "f1"}, "invalid_argument"),
Expand Down
4 changes: 3 additions & 1 deletion packages/agentenv-protocol/tests/test_computer_spaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from agentenv_protocol.computer import ComputerSettings, InvalidArgument, OutOfBounds # noqa: E402
from agentenv_protocol.computer.frames import FrameStore # noqa: E402
from agentenv_protocol.computer.operations import Display # noqa: E402
from agentenv_protocol.computer.results import CAPTURED_AT_META # noqa: E402
from agentenv_protocol.computer.spaces import check_space, frame_point # noqa: E402


Expand Down Expand Up @@ -97,12 +98,13 @@ async def test_setting_the_default_space_rewrites_the_descriptions_on_fastmcp():
card = (await http.get("/.well-known/agent-env.json")).json()
async with fastmcp.Client(app) as client:
listed = {t.name: t for t in await client.list_tools()}
await client.call_tool("computer_screenshot", {})
shot = await client.call_tool("computer_screenshot", {})
clicked = await client.call_tool("computer_click", {"x": 500, "y": 500})
card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click")
assert "norm space by default: 0 to 1000" in listed["computer_click"].description
assert card_click["description"] == listed["computer_click"].description
assert clicked.structured_content["summary"].startswith("Clicked left at (500, 500) (norm 1000) in f1")
assert shot.meta[CAPTURED_AT_META].endswith("Z") and "captured_at" not in shot.structured_content


@pytest.mark.asyncio
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from PIL import Image

from agentenv_protocol.computer.conformance.client import Driver
from agentenv_protocol.computer.results import CAPTURED_AT_META

THUMB = (64, 40)
DESK = {name: resources.files("agentenv_protocol.computer.conformance").joinpath(name).read_bytes()
Expand Down Expand Up @@ -58,7 +59,7 @@ def centre(box):
if result.isError:
return [{"id": "drove", "result": False, "description": f"{tool}: {result.content[0].text}"}]
shots.append({"step": tool, "args": args, "sent_at": sent_at,
"captured_at": result.structuredContent["frame"]["captured_at"], "thumb": _thumb(result),
"captured_at": result.meta[CAPTURED_AT_META], "thumb": _thumb(result),
"cell": list(_image(result).convert("RGB").getpixel((clicked[0] + 8, clicked[1] + 8)))})
await asyncio.sleep(0.7)
return [{"id": "drove", "result": True, "description": json.dumps({"shots": shots, "clicked_cell": clicked})}]
Loading