diff --git a/.gitignore b/.gitignore index edb9006f9..9c75a6675 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,8 @@ cython_debug/ # Generated local-store state (stores default local; see runtime._ensure_agentenv_gitignore) /.agentenv/ +# ...and what the protocol package's tests write when run from its own folder (launches, browser tab events) +/packages/agentenv-protocol/.agentenv/ # Written by the release workflow into the wheels it publishes src/agent_env/env/published_images.json diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 5f17fed1e..174561232 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -15,6 +15,7 @@ STANDARD_EXTENSIONS, TRAJECTORY_V1, TRIGGERS_V1, + CLOCK_V1, AttributionProbeResponse, BundleSkillRequest, CdpAddRequest, @@ -152,6 +153,7 @@ "STANDARD_EXTENSIONS", "TRAJECTORY_V1", "TRIGGERS_V1", + "CLOCK_V1", "A2AAgentApplication", "AgentCapabilities", "AgentConfig", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/clock.py new file mode 100644 index 000000000..d1d66aef6 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/clock.py @@ -0,0 +1,102 @@ +"""An agent's side of clock/v1. An agent image that follows the env's clock runs the computer base's follower, and +agent-env's libfaketime is preloaded for the agent's user, so every program the A2A server starts reads universe time +from the follower's page; the server itself starts with the real-time marker and stays on real time. + +The framework serves clock/v1's ``sync_time``, the method every server and computer that follows the clock serves, on +the card of an agent in such an image (``follows``), so the card is what tells core the agent follows. Unlike a +computer's, whose config core writes through the sandbox's root channel, an agent's request carries the follower's +config itself (``watch_url``, ``name``, ``read_token``), and this server, running as the agent's user, writes it: the +model's own commands could rewrite it and fool their own view of the time, though never the env's clock, and the token +only reads. It answers once the follower shows the gateway's current version, or says why it doesn't. The task's zone +is kept in the config beside what the follower reads, so a server restarted mid-run (``restore_zone``) brings the zone +back as the follower brings back the clock.""" +from __future__ import annotations + +import json +import os +import time +import uuid +from pathlib import Path +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from ..computer import clock as follower +from ..computer.errors import InvalidArgument, TargetUnavailable + + +def follows() -> bool: + """Whether this agent's image runs the clock follower.""" + return follower.follows() + + +def write_config(watch_url: str, name: str, read_token: str, timezone: str | None = None) -> int: + """The follower's config, with the task's ``timezone`` for ``restore_zone``, written beside its place and renamed + into it, so the follower, which reads it every quarter second, never reads half of one; readable by this user and + the follower's group alone. When it was + written (``st_mtime_ns``), which a status must postdate to be of this config; 0 when the config was this one + already, which the follower goes on following without writing its status anew.""" + path = Path(follower.CONFIG) + raw = json.dumps({"watch_url": watch_url, "name": name, "token": read_token, + **({"timezone": timezone} if timezone else {})}).encode() + try: + if path.read_bytes() == raw: + return 0 + except OSError: + pass + staged = path.with_name(f"{path.name}.{uuid.uuid4().hex}") + fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) + try: + with os.fdopen(fd, "wb") as f: + f.write(raw) + os.chmod(staged, 0o640) + os.replace(staged, path) + except BaseException: + staged.unlink(missing_ok=True) + raise + return path.stat().st_mtime_ns + + +def set_zone(timezone: str) -> str: + """The task's zone for every program this server starts from now on: ``TZ`` in its environment, which a started + program inherits; this server's own records keep UTC timestamps either way.""" + zone = timezone.strip() + try: + ZoneInfo(zone) + except (ZoneInfoNotFoundError, ValueError) as e: + raise InvalidArgument(f"timezone must be an IANA name such as 'America/Chicago', got {timezone!r}") from e + os.environ["TZ"] = zone + time.tzset() + return zone + + +def restore_zone() -> str | None: + """The task's zone ``sync_time`` saved, set again for what this server starts: called when it starts, so one that + restarts mid-run keeps the zone, as the follower keeps the clock. None when there's none, or it isn't a zone.""" + try: + zone = json.loads(Path(follower.CONFIG).read_text()).get("timezone") + except (OSError, ValueError, AttributeError): + return None + if not isinstance(zone, str) or not zone: + return None + try: + return set_zone(zone) + except InvalidArgument: + return None + + +async def sync_time(env_get_time_url: str, timezone: str | None = None, watch_url: str | None = None, + name: str | None = None, read_token: str | None = None) -> dict: + """``clock/v1``'s ``sync_time`` on an agent: the follower configured to follow the gateway at ``watch_url`` as + ``name`` with ``read_token``, the task's ``timezone`` set for what this server starts, and an answer once the + follower shows the gateway's version for that config, never what it showed of one before. ``env_get_time_url`` is taken as every server takes it and not used: the + follower reads the gateway's watch, not its time.""" + if not follows(): + raise TargetUnavailable("this agent's image doesn't run the clock follower") + if not (watch_url and name and read_token): + raise InvalidArgument("an agent follows the clock with its watch_url, name and read_token, which " + "sync_env_clock sends; this request names " + + (", ".join(k for k, v in (("watch_url", watch_url), ("name", name), + ("read_token", read_token)) if v) or "none of them")) + zone = set_zone(timezone) if timezone is not None else os.environ.get("TZ") or "UTC" + written = write_config(watch_url, name, read_token, zone) + shown = await follower.following("this agent's clock follower", follower.CONNECT_WAIT_S, url=watch_url, name=name, since_ns=written) + return {"synced": True, "timezone": zone, "env_get_time_url": env_get_time_url, "follower": shown} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index d48ff283b..e6200211d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -134,6 +134,18 @@ class BundleSkillRequest(ExtensionRequest): skill_bundle: SkillBundle +class ClockSyncTimeRequest(ExtensionRequest): + """clock/v1's ``sync_time`` on an agent: what every server's takes, ``env_get_time_url`` and ``timezone``, and the + config of the follower an agent runs itself, the gateway's watch URL as its sandbox reaches it, the name the + gateway knows it by and its read token.""" + + env_get_time_url: str + timezone: str | None = None + watch_url: str | None = None + name: str | None = None + read_token: str | None = None + + class SkillAddResponse(ExtensionResponse): name: str @@ -1345,6 +1357,25 @@ def _configure_trajectory(*, live: bool = False) -> ExtensionConfiguration: }, ) +CLOCK_V1 = ExtensionDefinition( + uri="urn:agentenv:clock/v1", + description=( + "Follow the env's clock: the agent's image runs the clock follower, so every program the agent runs reads " + "the env's time. Served where the image has the follower." + ), + endpoint="/ext/clock", + core_operations={ + "sync_time": _op( + "sync_time", + "POST", + "/ext/clock/sync-time", + SDK, + request_model=ClockSyncTimeRequest, + response_required=("synced", "timezone", "follower"), + ), + }, +) + INSTALL_V1 = ExtensionDefinition( uri="urn:agentenv:install/v1", description="Declare commands for installing the agent into an existing target.", @@ -1392,6 +1423,7 @@ def _configure_trajectory(*, live: bool = False) -> ExtensionConfiguration: PEER_AGENTS_V1, ATTRIBUTION_PROBE_V1, TRIGGERS_V1, + CLOCK_V1, INSTALL_V1, PROXY_V1, ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py index ce344d08a..6851ab3de 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import dataclasses import inspect import json import logging @@ -39,6 +40,8 @@ SKILL_CONFIG_V1, TRAJECTORY_V1, TRIGGERS_V1, + CLOCK_V1, + ClockSyncTimeRequest, ContextEventsTrajectoryRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, @@ -59,6 +62,8 @@ TriggerRegisterRequest, enable, ) +from ..computer.errors import ComputerError +from . import clock as agent_clock from .peer_agents import PEER_MCP_PATH, PEER_MCP_SERVER_NAME, PeerAgents from .registry import RegisteredOperation, build_registry from .staging import STAGING_ENDPOINT, StagingStore, staging_routes @@ -631,6 +636,7 @@ def handlers(self) -> dict[tuple[str, str], Callable]: (TRIGGERS_V1.uri, "register"): self.triggers_register, (TRIGGERS_V1.uri, "decide"): self.triggers_decide, (TRIGGERS_V1.uri, "state"): self.triggers_state, + (CLOCK_V1.uri, "sync_time"): self.clock_sync_time, } async def agent_config_set(self, request: AgentConfig) -> dict[str, Any]: @@ -876,6 +882,22 @@ async def triggers_decide(self, request: TriggerDecideRequest) -> dict[str, Any] async def triggers_state(self) -> dict[str, Any]: return self.trigger_engine.state() + async def clock_sync_time(self, request: ClockSyncTimeRequest) -> dict[str, Any]: + try: + return await agent_clock.sync_time(**request.model_dump()) + except ComputerError as exc: + raise HTTPException(status_code=exc.STATUS, detail=f"{exc.CODE}: {exc}") from exc + + +def _with_clock(definition: _AgentDefinition) -> _AgentDefinition: + """``definition`` serving clock/v1 where the agent's image runs the clock follower, so every agent in such an + image follows the env's clock with no code of its own, and takes back the task's zone if it restarted mid-run; + unchanged elsewhere, or where the agent declares it.""" + if not agent_clock.follows() or any(a.definition.uri == CLOCK_V1.uri for a in definition.extensions): + return definition + agent_clock.restore_zone() # a server restarted mid-run takes back the task's zone + return dataclasses.replace(definition, extensions=(*definition.extensions, ExtensionActivation(CLOCK_V1))) + class DefaultExtensionHandlers: """Stable access to default implementations of SDK-owned operations.""" @@ -968,6 +990,7 @@ def __init__(self, agent: Any, definition: _AgentDefinition | None = None) -> No "agent instance is already bound to an application; create a new " "agent instance for each application" ) + definition = _with_clock(definition) self.agent = agent self.definition = definition self.rpc_path = _rpc_path(definition.identity.url) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/clock.py index 19667b72d..d19851871 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/clock.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/clock.py @@ -47,11 +47,17 @@ def session_zone() -> str: def follower_status() -> dict: + return _status_written()[0] + + +def _status_written() -> tuple[dict, int]: + """The follower's status and when it wrote it (``st_mtime_ns``; 0 when there is none).""" try: - status = json.loads(Path(STATUS).read_text()) + path = Path(STATUS) + written, status = path.stat().st_mtime_ns, json.loads(path.read_text()) except (OSError, ValueError): - return {} - return status if isinstance(status, dict) else {} + return {}, 0 + return (status, written) if isinstance(status, dict) else ({}, 0) def gateway_version(watch_url: str) -> str: @@ -60,7 +66,10 @@ def gateway_version(watch_url: str) -> str: return json.loads(answer.read())["version"] -def _why(status: dict, current: Optional[str], unread: Optional[str]) -> str: +def _why(status: dict, current: Optional[str], unread: Optional[str], stale: bool = False) -> str: + if stale: + return (f"it still shows the config it had, the gateway at {status.get('url')} as {status.get('name')!r}, " + f"not the one just written") if status.get("error"): return f"it can't reach the gateway at {status.get('url')}: {status['error']}" if not status.get("url"): @@ -82,19 +91,35 @@ async def sync_time(env_get_time_url: str, timezone: Optional[str] = None) -> di if timezone is not None and timezone.strip() != zone: raise InvalidArgument(f"the task's clock is in {timezone.strip()!r} and this computer's session in {zone!r}; " f"deploy it with computer_settings.timezone = {timezone.strip()!r}") - deadline = time.monotonic() + CONNECT_WAIT_S + follower = await following("this computer's clock follower") + return {"synced": True, "timezone": zone, "env_get_time_url": env_get_time_url, "follower": follower} + + +async def following(whose: str, wait_s: float = CONNECT_WAIT_S, *, url: Optional[str] = None, + name: Optional[str] = None, since_ns: int = 0) -> dict: + """The follower's status once it shows the gateway's current version; TargetUnavailable saying why ``whose`` + doesn't within ``wait_s``. A computer's ``sync_time`` and an agent's (``agentenv_protocol.a2a_agent.clock``) both answer + with it. An agent's, which has just written the follower's config, names it: only a status of that ``url`` and + ``name`` the follower wrote since ``since_ns`` counts, so what it showed of a config before is never taken for + this one's.""" + deadline = time.monotonic() + wait_s + + def read() -> tuple[dict, bool]: + status, written = _status_written() + stale = bool(status) and (written < since_ns or (url is not None and status.get("url") != url) + or (name is not None and status.get("name") != name)) + return status, stale + while True: - status, current, unread = follower_status(), None, None - if status.get("connected") and status.get("url"): + (status, stale), current, unread = read(), None, None + if not stale and status.get("connected") and status.get("url"): try: current = await asyncio.to_thread(gateway_version, status["url"]) except Exception as e: # noqa: BLE001 - away or answering something else: said as why, tried again unread = f"{type(e).__name__}: {e}"[:300] - status = follower_status() - if current is not None and status.get("connected") and status.get("version") == current: - return {"synced": True, "timezone": zone, "env_get_time_url": env_get_time_url, - "follower": {key: status.get(key) for key in ("name", "url", "version", "armed")}} + status, stale = read() + if not stale and current is not None and status.get("connected") and status.get("version") == current: + return {key: status.get(key) for key in ("name", "url", "version", "armed")} if time.monotonic() >= deadline: - raise TargetUnavailable("this computer's clock follower isn't following the gateway: " - + _why(status, current, unread)) + raise TargetUnavailable(f"{whose} isn't following the gateway: " + _why(status, current, unread, stale)) await asyncio.sleep(0.1) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py new file mode 100644 index 000000000..b23e5192f --- /dev/null +++ b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py @@ -0,0 +1,181 @@ +"""An agent's side of clock/v1: in an image that runs the clock follower, every AgentEnvAgent serves sync_time with no +code of its own; called by sync_env_clock, it writes the follower's config from the request, sets the task's zone for +what the server starts, and answers once the follower shows the gateway's current version, or says why it doesn't.""" +from __future__ import annotations + +import json +import os +import stat +import threading +import time + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.a2a_agent import ( + CLOCK_V1, + AgentEnvAgent, + AgentIdentity, + TaskRequest, + TaskResult, + a2a_agent, +) +from agentenv_protocol.computer import clock as follower + +REQUEST = {"env_get_time_url": "http://gw/clock/time", "watch_url": "http://gw/clock/watch", "name": "agent-solver", + "read_token": "token"} + + +@pytest.fixture +def image(tmp_path, monkeypatch): + """An image that runs the follower: its config and status under ``tmp_path``, the gateway at version b:3.""" + (tmp_path / "clock-follower").write_text("") + monkeypatch.setattr(follower, "FOLLOWER", str(tmp_path / "clock-follower")) + monkeypatch.setattr(follower, "DISK_GUEST", str(tmp_path / "image.env")) + monkeypatch.setattr(follower, "CONFIG", str(tmp_path / "follower.json")) + monkeypatch.setattr(follower, "STATUS", str(tmp_path / "status.json")) + monkeypatch.setattr(follower, "CONNECT_WAIT_S", 0.3) + monkeypatch.setattr(follower, "gateway_version", lambda url: "b:3") + monkeypatch.setenv("TZ", "UTC") + yield tmp_path + time.tzset() + + +def _status(image, **status) -> None: + (image / "status.json").write_text(json.dumps(status)) + + +def _follows_the_next_config(image, **status) -> threading.Thread: + """A follower that, as the real one, writes its status once it reads a config other than the one it had.""" + config = image / "follower.json" + before = config.read_bytes() if config.exists() else None + + def run() -> None: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if config.exists() and config.read_bytes() != before: + _status(image, **status) + return + time.sleep(0.01) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + return thread + + +def _app(**declared): + @a2a_agent(identity=AgentIdentity(name="clocked", description="test", version="1"), **declared) + class Agent(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + return TaskResult.text("ok") + + return TestClient(Agent().create_app()) + + +def _clock(client) -> dict | None: + card = client.get("/.well-known/agent.json").json() + return next((e for e in card["capabilities"]["extensions"] if e["uri"] == CLOCK_V1.uri), None) + + +def test_every_agent_in_an_image_with_the_follower_serves_sync_time(image): + with _app() as client: + served = _clock(client) + + assert served is not None + request = served["params"]["methods"]["sync_time"]["request"] + assert request == {"required": ["env_get_time_url"], "optional": ["timezone", "watch_url", "name", "read_token"]} + + +def test_an_agent_in_an_image_without_the_follower_serves_no_clock(image, monkeypatch): + monkeypatch.setattr(follower, "FOLLOWER", str(image / "absent")) + with _app() as client: + assert _clock(client) is None + + +def test_sync_time_writes_the_followers_config_sets_the_zone_and_answers_once_it_follows(image): + _follows_the_next_config(image, connected=True, url="http://gw/clock/watch", name="agent-solver", version="b:3", + armed=True) + with _app() as client: + answer = client.post("/ext/clock/sync-time", json={**REQUEST, "timezone": "America/Chicago"}) + + assert answer.status_code == 200, answer.text + assert answer.json() == {"synced": True, "timezone": "America/Chicago", "env_get_time_url": "http://gw/clock/time", + "follower": {"name": "agent-solver", "url": "http://gw/clock/watch", "version": "b:3", + "armed": True}} + config = image / "follower.json" + assert json.loads(config.read_text()) == {"watch_url": "http://gw/clock/watch", "name": "agent-solver", + "token": "token", "timezone": "America/Chicago"} + assert stat.S_IMODE(config.stat().st_mode) == 0o640 + assert os.environ["TZ"] == "America/Chicago" and time.strftime("%Z") in {"CST", "CDT"} + assert [p.name for p in image.iterdir() if p.name.startswith("follower.json.")] == [] + + +def test_a_follower_behind_the_gateway_is_named(image): + _follows_the_next_config(image, connected=True, url="http://gw/clock/watch", name="agent-solver", version="b:2") + with _app() as client: + answer = client.post("/ext/clock/sync-time", json=REQUEST) + + assert answer.status_code == 404 + assert "this agent's clock follower isn't following the gateway: it shows the gateway's version 'b:2'" in answer.text + + +@pytest.mark.parametrize("missing", ["watch_url", "name", "read_token"]) +def test_a_request_without_the_followers_config_is_refused_and_nothing_is_written(image, missing): + with _app() as client: + answer = client.post("/ext/clock/sync-time", json={k: v for k, v in REQUEST.items() if k != missing}) + + assert answer.status_code == 400 and "watch_url, name and read_token" in answer.text + assert not (image / "follower.json").exists() + + +def test_a_zone_that_isnt_one_is_refused(image): + with _app() as client: + answer = client.post("/ext/clock/sync-time", json={**REQUEST, "timezone": "Mars/Olympus"}) + + assert answer.status_code == 400 and "IANA" in answer.text and os.environ["TZ"] == "UTC" + + +@pytest.mark.parametrize("before", [ + {"url": "http://old-gw/clock/watch", "name": "agent-solver"}, # another gateway: it may answer the same version + {"url": "http://gw/clock/watch", "name": "agent-solver"}, # this one, under the token the config replaced +]) +def test_what_the_follower_showed_of_its_config_before_never_passes_for_the_new_one(image, before): + _status(image, connected=True, version="b:3", armed=True, **before) + with _app() as client: + answer = client.post("/ext/clock/sync-time", json=REQUEST) + + assert answer.status_code == 404 + assert "still shows the config it had" in answer.text + + +def test_the_config_it_already_follows_resent_answers_from_its_status(image): + with _app() as client: + _follows_the_next_config(image, connected=True, url="http://gw/clock/watch", name="agent-solver", + version="b:3") + assert client.post("/ext/clock/sync-time", json=REQUEST).status_code == 200 + answer = client.post("/ext/clock/sync-time", json=REQUEST) # the follower writes nothing anew + + assert answer.status_code == 200, answer.text + + +def test_a_server_restarted_mid_run_takes_back_the_tasks_zone(image, monkeypatch): + _follows_the_next_config(image, connected=True, url="http://gw/clock/watch", name="agent-solver", version="b:3") + with _app() as client: + assert client.post("/ext/clock/sync-time", json={**REQUEST, "timezone": "America/Chicago"}).status_code == 200 + monkeypatch.setenv("TZ", "UTC") # the restarted server's environment: the image's, without the task's zone + time.tzset() + + with _app() as client: # starting it again, as supervisord does after the server dies + assert client.get("/.well-known/agent.json").status_code == 200 + + assert os.environ["TZ"] == "America/Chicago" and time.strftime("%Z") in {"CST", "CDT"} + + +@pytest.mark.parametrize("config", ['{"watch_url": "u", "name": "n", "token": "t"}', '{"timezone": "Mars/Olympus"}', + "not json", None]) +def test_a_server_starting_without_a_saved_zone_keeps_its_own(image, monkeypatch, config): + if config is not None: + (image / "follower.json").write_text(config) + with _app() as client: + assert client.get("/.well-known/agent.json").status_code == 200 + assert os.environ["TZ"] == "UTC" diff --git a/src/agent_env/a2a_agent/clock.py b/src/agent_env/a2a_agent/clock.py new file mode 100644 index 000000000..87156d198 --- /dev/null +++ b/src/agent_env/a2a_agent/clock.py @@ -0,0 +1,206 @@ +"""Agents on the env's clock (FD-5916): an agent whose image runs the clock follower serves clock/v1's ``sync_time`` +(``agentenv_protocol.a2a_agent.clock``), and follows the gateway's clock the way a computer does, so every program it +runs reads the env's time. ``follow`` declares it at the gateway as a follower with a read token of its own, then calls +its ``sync_time`` with the follower's config: the gateway's watch URL as the agent's sandbox reaches it, the name the +gateway knows it by and that token. From then on a clock write that waits for followers (``wait_followers_s``) waits +for the agent too. ``sync_env_clock`` has every agent registered with the env follow once it has armed the clock, and +``deploy_agent`` an agent it deploys while the clock is armed. An agent has one follower, so it follows one env's +clock: the first env it's registered with (``clock_env``), whatever order steps arm clocks in, even at once; another +env's is reported and skipped (``follows_another_env``). When ``teardown_sandboxes`` takes the agent's sandbox down, +``retire_followers`` has the gateway it follows stop waiting for it. Which +env each agent follows is kept under ``metadata[FOLLOWING_KEY]``, one key per agent, so steps that run at once and +each save the context never overwrite another's.""" +from __future__ import annotations + +import logging +import secrets +from typing import TYPE_CHECKING, Optional + +import httpx + +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.env.env import EnvCapabilityUnsupported, gateway_url_of +from agent_env.env.gateway.constants import CLOCK_WATCH_PATH, EXT_CLOCK_URI +from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for +from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url, sandbox_request_headers_for_url +from agent_env.task_step.context import StepContextError + +if TYPE_CHECKING: + from agent_env.env.env import DeployedEnv + from agent_env.task_step.context import DeployedAgent, TaskStepContext + +logger = logging.getLogger(__name__) + +#: What an agent's ``sync_time`` must take to run a follower of its own. +FOLLOWER_FIELDS = ("watch_url", "name", "read_token") +#: Bytes of randomness in an agent follower's read token. +READ_TOKEN_BYTES = 32 +#: How much of a failing agent's answer an error quotes. +ERROR_TEXT_CHARS = 600 +#: How long ``deploy_agent`` gives a late agent's declaration and ``sync_time``, which waits for its follower to connect. +LATE_FOLLOW_TIMEOUT_S = 60 +#: How long a teardown gives each gateway to retire an agent's follower. +RETIRE_TIMEOUT_S = 10 +#: ``metadata`` key of the env each agent follows: ``{slot: {"agent", "env_id", "follower"}}``, one slot per agent. +FOLLOWING_KEY = "agent_clock_following" + + +def follower_name(agent: DeployedAgent) -> str: + """The name the gateway knows ``agent``'s follower by, apart from the computers' names.""" + return f"agent-{agent.agent_name}" + + +def _sync_time(agent: DeployedAgent) -> tuple[Optional[dict], Optional[str]]: + return A2AAgent.operation(A2AAgent.find_extension(agent.a2a_card or {}, EXT_CLOCK_URI), "sync_time") + + +def _fields(method: dict) -> set[str]: + request = method.get("request") or {} + return {*request.get("required", ()), *request.get("optional", ()), *(request.get("properties") or {})} + + +def watch_url_for(agent: DeployedAgent, env: DeployedEnv) -> str: + """The gateway's watch URL as ``agent``'s sandbox reaches it, as its MCP URL is (``deploy_agent``).""" + url = gateway_url_of(env).rstrip("/") + CLOCK_WATCH_PATH + if not getattr(env, "sandbox_id", None): + return host_url_for(url, agent.sandbox_type) + return reachable_url(url, from_sandbox_type=env.sandbox_type, to_sandbox_type=agent.sandbox_type) + + +async def follow(agent: DeployedAgent, env: DeployedEnv, *, env_get_time_url: Optional[str], + timezone: Optional[str], timeout: float) -> dict: + """Have ``agent`` follow ``env``'s gateway clock. ``{"synced": False, "reason": ...}`` for an agent that can't: + its card serves no ``sync_time`` (``not_advertised``) or one without the follower's fields + (``no_follower_fields``), or the gateway takes no followers after it starts (``gateway_takes_no_followers``); raises + for one that serves it and fails, having retired the follower it declared, or can't take the task's zone, as for + a server.""" + method, path = _sync_time(agent) + if method is None: + return {"synced": False, "reason": "not_advertised"} + fields = _fields(method) + if not set(FOLLOWER_FIELDS) <= fields: + return {"synced": False, "reason": "no_follower_fields"} + if timezone is not None and "timezone" not in fields: + raise StepContextError(f"agent '{agent.agent_name}' serves {EXT_CLOCK_URI} sync_time without a timezone; " + f"rebuild its image or drop timezone={timezone!r} from the task") + name, token = follower_name(agent), secrets.token_urlsafe(READ_TOKEN_BYTES) + try: + await env.invoke(EXT_CLOCK_URI, "declare_follower", {"name": name, "read_token": token}, timeout=timeout) + except EnvCapabilityUnsupported: + return {"synced": False, "reason": "gateway_takes_no_followers"} + body = {"env_get_time_url": env_get_time_url or "", "watch_url": watch_url_for(agent, env), "name": name, + "read_token": token, **({"timezone": timezone} if timezone is not None else {})} + url = (agent.a2a_url or agent.api_url).rstrip("/") + path + try: + async with httpx.AsyncClient() as client: + answer = await client.post(url, json=body, timeout=timeout, headers=sandbox_request_headers_for_url(url)) + except httpx.HTTPError as e: + why = f"({type(e).__name__}): {e}" + else: + if answer.status_code < 400: + return {"synced": True, "follower": name, "result": answer.json()} + why = f"(HTTP {answer.status_code}): {answer.text}" + # Declared and not following: the gateway would wait for it on every later write, and nothing records it to retire. + await _retire(env, name, timeout=RETIRE_TIMEOUT_S) + raise RuntimeError(f"agent '{agent.agent_name}' doesn't follow the clock of env '{env.env_id}' " + f"{why[:ERROR_TEXT_CHARS]}") + + +async def _retire(env: DeployedEnv, follower: str, *, timeout: float) -> bool: + """Have ``env``'s gateway stop waiting for ``follower``; whether it did. Best-effort: a gateway that's gone or + takes no retirement is logged.""" + try: + await env.invoke(EXT_CLOCK_URI, "retire_follower", {"name": follower}, timeout=timeout) + except Exception as e: # noqa: BLE001 - best-effort: a teardown or a failed sync goes on either way + logger.warning(f"env {env.env_id}'s gateway didn't retire follower {follower!r}: {type(e).__name__}: {e}"[:300]) + return False + return True + + +def _slot(agent_name: str) -> str: + """``agent_name`` as a key of its own when the context is saved, where a ``.`` or a leading ``$`` would make one + save the whole map: percent-encoded, ``%`` first, so two names never share a key.""" + return agent_name.replace("%", "%25").replace(".", "%2E").replace("$", "%24") + + +def followed_env(context: TaskStepContext, agent: DeployedAgent) -> Optional[str]: + """The env whose clock ``agent`` follows in this run, if any.""" + return ((context.metadata.get(FOLLOWING_KEY) or {}).get(_slot(agent.agent_name)) or {}).get("env_id") + + +def torn_down(context: TaskStepContext, agent: DeployedAgent) -> bool: + """Whether ``teardown_sandboxes`` took ``agent``'s sandbox down: its record stays in the context, its server is gone.""" + from agent_env.task.teardown import TORN_DOWN_KEY + + return bool(agent.sandbox_id) and agent.sandbox_id in (context.metadata.get(TORN_DOWN_KEY) or ()) + + +def clock_env(agent: DeployedAgent) -> Optional[str]: + """The one env whose clock ``agent`` follows: the first it's registered with. Fixed by the agent alone, so two + steps arming two of its envs' clocks at once never both take its one follower.""" + return next(iter(agent.env_ids or ()), None) + + +async def follow_in(context: TaskStepContext, agent: DeployedAgent, env: DeployedEnv, *, + env_get_time_url: Optional[str], timezone: Optional[str], timeout: float) -> dict: + """``follow`` when ``env`` is the agent's ``clock_env``, recording it; another env's clock, which would take over + its follower while that gateway still waits for it, is reported and skipped (``follows_another_env``).""" + if env.env_id != clock_env(agent): + return {"synced": False, "reason": "follows_another_env", "follows": clock_env(agent)} + outcome = await follow(agent, env, env_get_time_url=env_get_time_url, timezone=timezone, timeout=timeout) + if outcome["synced"]: + context.metadata.setdefault(FOLLOWING_KEY, {})[_slot(agent.agent_name)] = { + "agent": agent.agent_name, "env_id": env.env_id, "follower": outcome["follower"]} + return outcome + + +def summary(outcome: dict) -> dict: + """What a log says of an outcome: never the agent's whole answer.""" + return {key: outcome[key] for key in ("synced", "reason", "follower", "follows") if key in outcome} + + +def _armed(context: TaskStepContext, env_id: str) -> Optional[dict]: + """The configuration of the last ``sync_env_clock`` that armed ``env_id``'s clock in this run, if any.""" + return next((c for c in reversed(context.metadata.get("clock_configurations") or []) + if c.get("env_id") == env_id and "env_get_time_url" in c), None) + + +async def follow_armed_clocks(context: TaskStepContext, agent: DeployedAgent, *, + timeout: float = LATE_FOLLOW_TIMEOUT_S) -> list[dict]: + """Have an agent ``deploy_agent`` just deployed follow the clock of each env registered with it that this run has + armed, as ``sync_env_clock`` has every agent then: one that can't is reported and skipped, or fails the step under + that step's ``require_agent_clock``.""" + outcomes = [] + for env_id in agent.env_ids: + if (armed := _armed(context, env_id)) is None: + continue + env = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if env is None or gateway_url_of(env) is None: + continue + outcome = await follow_in(context, agent, env, env_get_time_url=armed.get("env_get_time_url"), + timezone=armed.get("timezone"), timeout=timeout) + if not outcome["synced"] and armed.get("require_agent_clock", False): + raise StepContextError(f"agent '{agent.agent_name}' can't follow the clock of env '{env_id}': " + f"{outcome['reason']}") + logger.info(f"agent '{agent.agent_name}' on env {env_id}'s clock: {summary(outcome)}") + outcomes.append({"env_id": env_id, "agent": agent.agent_name, **outcome}) + return outcomes + + +async def retire_followers(context: TaskStepContext, agent_names, *, timeout: float = RETIRE_TIMEOUT_S) -> list[str]: + """Have the gateway whose clock each of ``agent_names`` follows in this run stop waiting for it, as its agent is + torn down, so later writes that wait for followers don't wait for it or answer 504 naming it. Best-effort, as a + teardown is: a gateway that's gone or takes no retirement is logged and its agent left recorded. The followers + retired.""" + following, retired = context.metadata.get(FOLLOWING_KEY) or {}, [] + for name in dict.fromkeys(agent_names): + if (record := following.get(_slot(name))) is None: + continue + env = next((d for d in context.deployed_envs if d.env_id == record["env_id"]), None) + if env is None: + continue + if not await _retire(env, record["follower"], timeout=timeout): + continue + del following[_slot(name)] + retired.append(record["follower"]) + return retired diff --git a/src/agent_env/a2a_agent/validator.py b/src/agent_env/a2a_agent/validator.py index 094f1dee6..f33ea166f 100644 --- a/src/agent_env/a2a_agent/validator.py +++ b/src/agent_env/a2a_agent/validator.py @@ -16,10 +16,14 @@ import asyncio import base64 import logging +import re import uuid from dataclasses import dataclass +from datetime import datetime from typing import Callable, Optional, Union +from zoneinfo import ZoneInfo +import httpx from agentenv_protocol.a2a_agent import ObjectChangelogApplyResponse from agent_env.a2a_agent.object_transfer import ( @@ -32,6 +36,7 @@ from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.bundle.run import _write_bundle from agent_env.config import get_config +from agent_env.env.gateway.constants import EXT_CLOCK_URI from agent_env.env.store import get_env_store from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases from agent_env.store.base import NotFoundError @@ -60,7 +65,10 @@ VIDEO_PROBE_MP4_B64, VIDEO_PROBE_PROMPT, ) +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container +from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep logger = logging.getLogger(__name__) @@ -126,6 +134,22 @@ class ModalityProbe: ), ] +#: clock/v1: the validation env's clock is armed frozen at the first time, in the zone, then a write that waits for the +#: agent moves it to the second; at each, the agent runs the command through its harness and must show that time. +CLOCK_PROBE_TIMES = ("2031-03-03T23:06:07-06:00", "2031-03-04T23:06:07-06:00") +CLOCK_PROBE_ZONE = "America/Chicago" +CLOCK_PROBE_COMMAND = "date +%Y-%m-%dT%H:%M:%S%z" +CLOCK_PROBE_PROMPT = f"Run exactly one shell command and nothing else: `{CLOCK_PROBE_COMMAND}`. Reply with only its output." +CLOCK_WAIT_FOLLOWERS_S = 10.0 +#: How much longer than the followers' wait the advance request may take: the gateway answers after it. +CLOCK_ADVANCE_MARGIN_S = 30 +#: How long one probe turn (the agent running ``date`` through its harness) may take. +CLOCK_PROBE_TURN_TIMEOUT_S = 180 +#: How much of a failure the validation result keeps as its error. +CLOCK_ERROR_CHARS = 600 +_CLOCK_STAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{4}") +_CLOCK_STAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z" + def write_validation_env(on_progress: Callable[[str], None]) -> None: """Write the env the validator deploys, ``A2AAgent.VALIDATION_ENV_ID``, when the store has none of that id: the env @@ -448,6 +472,9 @@ async def _run( on_progress=on_progress, ) + # clock/v1 after the DAG: arming the validation env's clock moves the clock of every agent that follows it. + await A2AAgentValidator._validate_clock(agent, context, task_id=task_id, on_progress=on_progress) + # install/v1: gated on the LIVE deployed card (not the stored one), so it # runs identically regardless of whether this agent was validated before. await A2AAgentValidator._validate_install( @@ -743,6 +770,65 @@ def record(*, supported, advertised, save_ok, apply_ok, roundtrip_ok, note=""): record(supported=roundtrip_ok, advertised=advertised, save_ok=save_ok, apply_ok=True, roundtrip_ok=roundtrip_ok) + @staticmethod + async def _validate_clock( + agent: "A2AAgent", + context, + task_id: str, + on_progress: Optional[Callable[[str], None]] = None, + ) -> None: + """Validate clock/v1 where the live card serves ``sync_time``: the validation env's clock armed frozen with the + agent required to follow it, its harness's ``date`` showing that time in the task's zone, a write that waits for + the agent acknowledged, and ``date`` showing the time it moved to.""" + log = on_progress or (lambda _msg: None) + main = next((a for a in context.deployed_agents if a.agent_name == TaskStep.DEFAULT_AGENT_NAME), None) + env = next((d for d in context.deployed_envs if d.env_id == agent.VALIDATION_ENV_ID), None) + advertised = main is not None and A2AAgent.extension_method(main.a2a_card or {}, EXT_CLOCK_URI, "sync_time") is not None + result: dict = {"supported": False, "advertised": advertised, "synced": False, "follows_writes": False, + "readings": []} + if not advertised or env is None: + result["skipped_reason"] = ("clock/v1 sync_time not advertised on live agent card" if not advertised + else f"validation env {agent.VALIDATION_ENV_ID!r} not deployed") + log(f"clock/v1 skipped: {result['skipped_reason']}") + else: + log("Validating clock/v1: the agent follows the validation env's clock...") + try: + await SyncEnvClockTaskStep( + id=f"{task_id}-clock", version=None, env_id=env.env_id, virtual_time=CLOCK_PROBE_TIMES[0], + virtual_seconds_per_real_second=0, timezone=CLOCK_PROBE_ZONE, require_agent_clock=True, + ).execute(context) + result["synced"] = True + result["readings"].append(await A2AAgentValidator._read_agent_clock(context, f"{task_id}-clock-1", CLOCK_PROBE_TIMES[0])) + await env.invoke(EXT_CLOCK_URI, "advance", + {"virtual_time": CLOCK_PROBE_TIMES[1], "wait_followers_s": CLOCK_WAIT_FOLLOWERS_S}, + timeout=int(CLOCK_WAIT_FOLLOWERS_S) + CLOCK_ADVANCE_MARGIN_S) + result["follows_writes"] = True + result["readings"].append(await A2AAgentValidator._read_agent_clock(context, f"{task_id}-clock-2", CLOCK_PROBE_TIMES[1])) + except httpx.HTTPStatusError as e: + result["error"] = f"HTTP {e.response.status_code}: {e.response.text}"[:CLOCK_ERROR_CHARS] + except Exception as e: # noqa: BLE001 - recorded as why the agent doesn't follow + result["error"] = f"{type(e).__name__}: {e}"[:CLOCK_ERROR_CHARS] + result["supported"] = (result["synced"] and result["follows_writes"] and len(result["readings"]) == 2 + and all(r["time"] and r["zone"] for r in result["readings"])) + log(f"clock/v1: supported={result['supported']} synced={result['synced']} " + f"follows_writes={result['follows_writes']} readings={[r['shown'] for r in result['readings']]}") + try: + fresh = A2AAgent.get(agent.id) + validated = dict(fresh.metadata.get("validated_a2a_extensions", {})) + validated[EXT_CLOCK_URI] = result + fresh.update_metadata({**fresh.metadata, "validated_a2a_extensions": validated}) + except Exception as e: # noqa: BLE001 + logger.warning(f"clock/v1: failed to persist validated_a2a_extensions: {e}") + context.metadata.setdefault("verifications", {})["a2a_clock"] = result + + @staticmethod + async def _read_agent_clock(context, prompt_id: str, at: str) -> dict: + """The agent's ``date``, run through its harness, read against the frozen clock at ``at``.""" + await PromptAgentTaskStep(id=prompt_id, version=None, prompt=CLOCK_PROBE_PROMPT, prompt_id=prompt_id, + timeout_seconds=CLOCK_PROBE_TURN_TIMEOUT_S).execute(context) + reply = next((p.response or "" for p in context.prompt_responses if p.prompt_id == prompt_id), "") + return {"prompt_id": prompt_id, **clock_reading(reply, at)} + @staticmethod def _upload_install_test_image_fixture(agent: "A2AAgent"): """Upload a minimal Dockerfile as a FileArtifactUniverse so the install @@ -879,3 +965,14 @@ async def _cleanup_sandboxes(context) -> None: await sandbox.terminate() except Exception as e: logger.warning(f"Failed to clean up agent sandbox {deployed_agent.sandbox_id}: {e}") + + +def clock_reading(reply: str, at: str, zone: str = CLOCK_PROBE_ZONE) -> dict: + """What the agent's ``date`` showed of the frozen clock at ``at``: the first timestamp in ``reply``, whether it is + that instant (``time``) and whether it carries ``zone``'s offset then (``zone``).""" + expected = datetime.fromisoformat(at).astimezone(ZoneInfo(zone)) + shown = next(iter(_CLOCK_STAMP.findall(reply or "")), None) + seen = datetime.strptime(shown, _CLOCK_STAMP_FORMAT) if shown else None + return {"expected": expected.strftime(_CLOCK_STAMP_FORMAT), "shown": shown, + "time": seen is not None and seen == expected, + "zone": seen is not None and seen.utcoffset() == expected.utcoffset()} diff --git a/src/agent_env/env/gateway/clock_followers.py b/src/agent_env/env/gateway/clock_followers.py index 5b89a351f..190c58b3c 100644 --- a/src/agent_env/env/gateway/clock_followers.py +++ b/src/agent_env/env/gateway/clock_followers.py @@ -1,4 +1,5 @@ -"""The followers of the gateway's clock (the computers on it): each long-polls ``/clock/watch`` with the version it +"""The followers of the gateway's clock (the computers on it, declared at its start, and the agents a run declares +after it, ``declare``, and retires when it tears one down, ``retire``): each long-polls ``/clock/watch`` with the version it shows, which is its acknowledgement, and a clock write can wait until every one shows the write's version.""" from __future__ import annotations @@ -41,9 +42,35 @@ def __init__(self, clock: Callable[[], Clock], tokens: Mapping[str, str]): self._tokens = dict(tokens) self._shown: dict[str, int | None] = dict.fromkeys(self._tokens) self._seen: dict[str, float] = {} + self._declared: set[str] = set() self._changed = asyncio.Event() self._acks = asyncio.Condition() + def declare(self, name: str, token: str) -> None: + """Declare ``name`` a follower, with the read token its watch carries, after the gateway started: an agent the + run deploys. Declared again, as a redeploy does, it takes the new token and is waited for afresh.""" + if not name or not token: + raise ClockError("a follower takes a name and a read token") + if name in self._tokens and name not in self._declared: + raise ClockError(f"{name!r} is a follower this gateway started with, which a declaration doesn't replace") + self._tokens[name] = token + self._shown[name] = None + self._seen.pop(name, None) + self._declared.add(name) + + async def retire(self, name: str) -> bool: + """Stop waiting for ``name``, a follower ``declare`` added, whose agent a run has torn down: its token no + longer acknowledges, no write waits for it or names it, and a write waiting for it now answers. Whether it was + one; a follower the gateway started with stays.""" + if name not in self._declared: + return False + async with self._acks: + self._declared.discard(name) + for followers in (self._tokens, self._shown, self._seen): + followers.pop(name, None) + self._acks.notify_all() + return True + def changed(self) -> None: """Answer every held watch: the clock has a new version.""" self._changed.set() @@ -71,11 +98,12 @@ async def watch(self, after: str, follower: str | None) -> None: if follower is not None: count = self._clock().count_of(after) async with self._acks: - shown = self._shown[follower] - if count is not None and (shown is None or count > shown): - self._shown[follower] = count - self._seen[follower] = time.monotonic() - self._acks.notify_all() + if follower in self._shown: # else retired while its acknowledgement waited for the lock + shown = self._shown[follower] + if count is not None and (shown is None or count > shown): + self._shown[follower] = count + self._seen[follower] = time.monotonic() + self._acks.notify_all() if after == self._clock().version: try: await asyncio.wait_for(changed.wait(), self.WATCH_S) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 58be95af0..c66934f07 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -79,6 +79,7 @@ def data_plane_load_timeout_s(size_bytes: int | None) -> int: COMPUTERS_ENV = "COMPUTERS" # The route a clock follower long-polls, with its read token. CLOCK_WATCH_PATH = "/clock/watch" +CLOCK_FOLLOWERS_PATH = "/clock/followers" # How long /clock/watch holds a watch naming the current version before answering it unchanged; a follower's request # timeout must be longer. The card's watch method advertises it as hold_s. CLOCK_WATCH_HOLD_S = 25.0 @@ -192,6 +193,7 @@ def sources(names: list[str]) -> str: f"/tools/{TOOL_DISABLE_ACTION}", f"/tools/{TOOL_ENABLE_ACTION}", "/triggers/register", "/triggers/remove", "/triggers/clear", "/notifications/config", "/notifications/take", "/notifications/clear", + CLOCK_FOLLOWERS_PATH, }) @@ -254,7 +256,7 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_CLOCK_URI, - "description": "Virtual clock (urn:agentenv:clock/v1): read the gateway's virtual time (wall-clock-derived, rate-scaled) via get_time; set_time/set_rate/advance/clear are the harness control path, each changing the clock's opaque version and, given wait_followers_s, answering 504 naming the followers that don't show it in time (set_rate keeps t0, as advance does, so time triggers keep their marks; rate 0 freezes the clock); a follower follows it through watch, a long poll naming the version it shows and held up to hold_s; state's followers block is diagnostic, not part of the contract.", + "description": "Virtual clock (urn:agentenv:clock/v1): read the gateway's virtual time (wall-clock-derived, rate-scaled) via get_time; set_time/set_rate/advance/clear are the harness control path, each changing the clock's opaque version and, given wait_followers_s, answering 504 naming the followers that don't show it in time (set_rate keeps t0, as advance does, so time triggers keep their marks; rate 0 freezes the clock); declare_follower adds a follower after the gateway started (an agent the run deploys), with the read token its watches carry, and retire_follower removes one so writes stop waiting for it (an agent the run tears down); a follower follows it through watch, a long poll naming the version it shows and held up to hold_s; state's followers block is diagnostic, not part of the contract.", "params": {"endpoint": "/clock/time", "methods": { "get_time": {"method": "GET", "endpoint": "/clock/time"}, "set_time": {"method": "PUT", "endpoint": "/clock/set-time", "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}, "virtual_seconds_per_real_second": {"type": "number"}, "time_zone": {"type": "string"}, **_WAIT_FOLLOWERS}, "required": ["virtual_time"]}}, @@ -262,6 +264,8 @@ def random_mcp_server_name() -> str: "advance": {"method": "PUT", "endpoint": "/clock/advance", "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}, **_WAIT_FOLLOWERS}, "required": ["virtual_time"]}}, "clear": {"method": "POST", "endpoint": "/clock/clear", "request": {"type": "object", "properties": {**_WAIT_FOLLOWERS}}}, "state": {"method": "GET", "endpoint": "/clock/state", "request": {"type": "object", "properties": {}}}, + "declare_follower": {"method": "PUT", "endpoint": CLOCK_FOLLOWERS_PATH, "request": {"type": "object", "properties": {"name": {"type": "string"}, "read_token": {"type": "string"}}, "required": ["name", "read_token"]}}, + "retire_follower": {"method": "DELETE", "endpoint": CLOCK_FOLLOWERS_PATH, "request": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}, "watch": {"method": "GET", "endpoint": CLOCK_WATCH_PATH, "hold_s": CLOCK_WATCH_HOLD_S, "request": {"type": "object", "properties": {"after": {"type": "string"}, "follower": {"type": "string"}}}}}}, "required": False, }, diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 1fd29608c..5c42fdb82 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -47,6 +47,7 @@ AGENT_ENV_SESSION_META_KEY, # noqa: F401 CARD_FETCH_TIMEOUT_S, CHANGES_PAGE_DEFAULT, + CLOCK_FOLLOWERS_PATH, CLOCK_WATCH_PATH, CHANGES_PAGE_MAX, CHILD_DISCOVERY_TIMEOUT_S, @@ -402,6 +403,31 @@ async def clock_watch(request: Request) -> Response: await self._clock_followers.watch(request.query_params.get("after", ""), follower) return self._json_response({**self._clock.state(), "received": received, "sent": time.time()}, 200) + @self._mcp.custom_route(CLOCK_FOLLOWERS_PATH, methods=["PUT"]) + async def clock_declare_follower(request: Request) -> Response: + try: + body = await request.json() + except Exception: + return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if not isinstance(body, dict): + return self._json_response({"ok": False, "error": "body must be a JSON object"}, 400) + try: + self._clock_followers.declare(str(body.get("name") or ""), str(body.get("read_token") or "")) + except ClockError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + return self._json_response({"ok": True, "followers": self._clock_followers.state()}, 200) + + @self._mcp.custom_route(CLOCK_FOLLOWERS_PATH, methods=["DELETE"]) + async def clock_retire_follower(request: Request) -> Response: + try: + body = await request.json() + except Exception: + return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if not isinstance(body, dict) or not isinstance(body.get("name"), str) or not body["name"]: + return self._json_response({"ok": False, "error": "body must be a JSON object naming the follower"}, 400) + retired = await self._clock_followers.retire(body["name"]) + return self._json_response({"ok": True, "retired": retired, "followers": self._clock_followers.state()}, 200) + @self._mcp.custom_route("/clock/time", methods=["GET"]) async def clock_time(request: Request) -> Response: try: diff --git a/src/agent_env/env/gateway/get_time.py b/src/agent_env/env/gateway/get_time.py index 24767249f..fa66a1a5a 100644 --- a/src/agent_env/env/gateway/get_time.py +++ b/src/agent_env/env/gateway/get_time.py @@ -18,7 +18,7 @@ GET_TIME_TOOL_NAME = "get_time" GET_TIME_TOOL_DESCRIPTION = ( "Returns the current date and time for this environment. The clock on the machine you are " - "running on is not synchronized with this environment, so do not use it and do not rely on " + "running on may not be synchronized with this environment, so do not rely on it or on " "your own assumption about today's date. Call this tool and derive from its value every " "absolute date, time or timestamp you pass to another tool, and every relative phrase you " 'resolve ("today", "this week", "overdue", "the last 30 days"). current_time is in UTC, as ' diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index fb24dc111..f5f16c2c9 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -546,6 +546,9 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas network_policy=deployed.network_policy, env_ids=list(self.env_ids), )) + # An agent deployed while a clock is armed follows it, as sync_env_clock had every agent then. + from agent_env.a2a_agent.clock import follow_armed_clocks + await follow_armed_clocks(context, context.deployed_agents[-1]) return context def _model_endpoint(self, context: TaskStepContext) -> dict[str, str]: diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index 8f574b058..d3f2a0777 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -5,6 +5,10 @@ env card lists (``legacy_protocol.backing_servers``) whose own card advertises it, at the route that card gives. An env whose card offers no set_time has no gateway clock to arm, as a computer deployed alone, and is refused saying to deploy it in a multi env; a record that kept no card is refused saying so. +Then has every agent registered with the env whose card serves clock/v1 sync_time follow the clock as a computer does +(``agent_env.a2a_agent.clock``), so every program it runs reads the env's time and later clock writes can wait for it. +An agent that can't is skipped, as every agent is until its image runs the follower, or fails the step with +require_agent_clock. Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. Place right before prompt_agent so the virtual timeline starts at agent-start; when setup needs the clock armed, arm it at rate 0 and start it with set_env_clock_rate right before prompt_agent. @@ -75,13 +79,17 @@ def __init__( fail_task_on_error: bool = True, timeout_seconds: int = 30, timezone: Optional[str] = None, + require_agent_clock: bool = False, ): """virtual_seconds_per_real_second = clock speed (1.0 real time, 0 frozen, 86400 cap = 1 real s -> 1 virtual day). For ~V virtual seconds over an ~R-real-second task, pass V/R. timezone = optional IANA name. Unset keeps today's behaviour. Set, a wall-clock virtual_time is local time in that zone, the gateway's clock is armed in it, so its get_time answers in it too, and every synced server is - told the zone (a server that cannot take it fails the step).""" + told the zone (a server that cannot take it fails the step). + + require_agent_clock = fail the step when an agent registered with the env can't follow its clock, rather than + skip it; apart from tolerate_missing_sync_time, which is about the env's servers.""" super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) if not isinstance(virtual_time, str): raise ValueError("virtual_time must be an RFC3339 string") @@ -89,6 +97,7 @@ def __init__( self.virtual_time = virtual_time self.virtual_seconds_per_real_second = virtual_seconds_per_real_second self.tolerate_missing_sync_time = tolerate_missing_sync_time + self.require_agent_clock = require_agent_clock self.timeout_seconds = timeout_seconds self.timezone = timezone.strip() if isinstance(timezone, str) else timezone self.armed_virtual_time = resolve_virtual_time(virtual_time, self.timezone) @@ -102,6 +111,8 @@ def to_dict(self) -> dict: base["timeout_seconds"] = self.timeout_seconds if self.timezone is not None: base["timezone"] = self.timezone + if self.require_agent_clock: + base["require_agent_clock"] = True return base @classmethod @@ -114,6 +125,7 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: tolerate_missing_sync_time=data.get("tolerate_missing_sync_time", True), timeout_seconds=data.get("timeout_seconds", 30), timezone=data.get("timezone"), + require_agent_clock=data.get("require_agent_clock", False), ) async def _backing_servers(self, deployed) -> list[BackingServer]: @@ -160,22 +172,45 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: entry = {"service": service, "environment": service, **({} if server.mcp else {"website": True}), **outcome} (synced if outcome.get("synced") else skipped).append(entry) + agents = await self._follow_agents(context, deployed, env_get_time_url) configuration = { "step_id": self.id, "env_id": self.env_id, "virtual_time": self.virtual_time, "virtual_seconds_per_real_second": self.virtual_seconds_per_real_second, - "env_get_time_url": env_get_time_url, "synced": synced, "skipped": skipped, + "env_get_time_url": env_get_time_url, "synced": synced, "skipped": skipped, "agents": agents, + "require_agent_clock": self.require_agent_clock, } if self.timezone is not None: configuration.update(timezone=self.timezone, armed_at=self.armed_virtual_time, get_time_zone=state.get("time_zone")) context.metadata.setdefault("clock_configurations", []).append(configuration) logger.info(f"sync_env_clock env={self.env_id}: armed rate={self.virtual_seconds_per_real_second}; " - f"synced={[s['service'] for s in synced]} skipped={[s['service'] for s in skipped]}") + f"synced={[s['service'] for s in synced]} skipped={[s['service'] for s in skipped]} " + f"agents={[(a['agent'], a['synced']) for a in agents]}") if not synced: logger.warning(f"sync_env_clock env={self.env_id}: clock armed but NO server synced " f"({[(s['service'], s.get('reason')) for s in skipped]}) — every tool still answers wall time") return context + async def _follow_agents(self, context: TaskStepContext, deployed, env_get_time_url: Optional[str]) -> list[dict]: + """Have every agent registered with the env follow its clock (``a2a_agent.clock.follow``): one that can't is + reported and skipped, or fails the step under ``require_agent_clock``.""" + from agent_env.a2a_agent.clock import follow_in, torn_down + + outcomes = [] + for agent in context.deployed_agents: + if self.env_id not in (agent.env_ids or []) or torn_down(context, agent): + continue # not the env's, or taken down by teardown_sandboxes: nothing to follow + outcome = await follow_in(context, agent, deployed, env_get_time_url=env_get_time_url, + timezone=self.timezone, timeout=self.timeout_seconds) + if not outcome["synced"]: + if self.require_agent_clock: + raise StepContextError(f"agent '{agent.agent_name}' can't follow the clock of env " + f"'{self.env_id}': {outcome['reason']}") + logger.info(f"sync_env_clock: agent '{agent.agent_name}' doesn't follow the clock " + f"({outcome['reason']}); skipping") + outcomes.append({"agent": agent.agent_name, **outcome}) + return outcomes + async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str], mcp: bool = True, tolerate: Optional[bool] = None, server: Optional[BackingServer] = None) -> dict: """Invoke sync_time on one server, an MCP server or with ``mcp=False`` a website, if its card advertises diff --git a/src/agent_env/task_step/task_steps/teardown_sandboxes.py b/src/agent_env/task_step/task_steps/teardown_sandboxes.py index 76af0ff98..0257566ce 100644 --- a/src/agent_env/task_step/task_steps/teardown_sandboxes.py +++ b/src/agent_env/task_step/task_steps/teardown_sandboxes.py @@ -27,6 +27,7 @@ tear_down, ) from agent_env.task_step.context import TaskStepContext +from agent_env.a2a_agent.clock import retire_followers from agent_env.task_step.task_step import TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -85,6 +86,10 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning(f"teardown_sandboxes: not deployed in this run, skipping: {', '.join(missing)}") already = set(context.metadata.get(TORN_DOWN_KEY) or []) await before_teardown(context, LAST_DRAIN_SECONDS, agent_names=self.agent_names) + # Before the agents go, however named: a gateway still running waits for a follower until it's told it's gone. + doomed = {sandbox.sandbox_id for sandbox in sandboxes} + await retire_followers(context, [*self.agent_names, *(a.agent_name for a in context.deployed_agents + if a.sandbox_id in doomed)]) report = await tear_down(context, sandboxes, envs, timeout=TERMINATE_TIMEOUT_SECONDS) for sandbox, why in report.failed: logger.warning(f"teardown_sandboxes: {sandbox.sandbox_id}: {why}"[:300]) diff --git a/tst/unit/a2a_agent/test_validator_clock.py b/tst/unit/a2a_agent/test_validator_clock.py new file mode 100644 index 000000000..672cdda56 --- /dev/null +++ b/tst/unit/a2a_agent/test_validator_clock.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import httpx +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.a2a_agent.validator import CLOCK_PROBE_TIMES, CLOCK_PROBE_ZONE, A2AAgentValidator, clock_reading +from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep + +CLOCK_CARD = {"capabilities": {"extensions": [{"uri": EXT_CLOCK_URI, "params": {"endpoint": "/ext/clock", "methods": { + "sync_time": {"method": "POST", "endpoint": "/ext/clock/sync-time"}}}}]}} + + +@pytest.mark.parametrize(("reply", "time", "zone"), [ + pytest.param("2031-03-03T23:06:07-0600", True, True, id="env-time-in-the-tasks-zone"), + pytest.param("2031-03-04T05:06:07+0000", True, False, id="env-time-without-the-zone"), + pytest.param("2026-11-17T11:20:00-0600", False, True, id="real-time"), + pytest.param("I can't run shell commands here.", False, False, id="no-reading"), +]) +def test_a_reading_is_the_frozen_instant_in_the_tasks_zone(reply, time, zone): + reading = clock_reading(f"Output:\n{reply}\n", CLOCK_PROBE_TIMES[0]) + assert (reading["expected"], reading["time"], reading["zone"]) == ("2031-03-03T23:06:07-0600", time, zone) + + +class _Env: + env_id = A2AAgent.VALIDATION_ENV_ID + + def __init__(self, write_status: int = 200): + self.write_status = write_status + self.calls: list[tuple[str, dict]] = [] + + async def invoke(self, uri, method, params=None, *, timeout=None): + self.calls.append((method, params)) + if self.write_status >= 400: + request = httpx.Request("PUT", "http://gateway/clock/advance") + response = httpx.Response(self.write_status, json={"ok": False, "stale": ["agent-agent"]}, request=request) + raise httpx.HTTPStatusError("not acknowledged", request=request, response=response) + return {"ok": True} + + +@pytest.fixture +def persisted(monkeypatch) -> SimpleNamespace: + agent = SimpleNamespace(metadata={}) + agent.update_metadata = lambda metadata: setattr(agent, "metadata", metadata) + monkeypatch.setattr(A2AAgent, "get", classmethod(lambda cls, agent_id, version=None: agent)) + return agent + + +@pytest.fixture +def synced(monkeypatch) -> list[SyncEnvClockTaskStep]: + armed: list[SyncEnvClockTaskStep] = [] + + async def execute(self, context): + armed.append(self) + return context + + monkeypatch.setattr(SyncEnvClockTaskStep, "execute", execute) + return armed + + +def _env_time(at: str) -> str: + return clock_reading("", at)["expected"] + + +def _real_time(at: str) -> str: + return "2026-11-17T11:20:00-0600" + + +def _shows(monkeypatch, reply_at) -> None: + times = iter(CLOCK_PROBE_TIMES) + + async def execute(self, context): + context.prompt_responses.append(PromptResponse(prompt_id=self.prompt_id, response=reply_at(next(times)))) + return context + + monkeypatch.setattr(PromptAgentTaskStep, "execute", execute) + + +async def _validate(card: dict, env: _Env) -> dict: + agent = SimpleNamespace(id="agent-under-test", VALIDATION_ENV_ID=A2AAgent.VALIDATION_ENV_ID) + context = TaskStepContext(deployed_envs=[env], deployed_agents=[DeployedAgent( + agent_name=TaskStep.DEFAULT_AGENT_NAME, api_url="http://agent", a2a_card=card, env_ids=[env.env_id])]) + await A2AAgentValidator._validate_clock(agent, context, task_id="validate") + return context.metadata["verifications"]["a2a_clock"] + + +@pytest.mark.asyncio +async def test_an_agent_whose_card_serves_no_sync_time_is_skipped_without_arming(persisted, synced): + result = await _validate({"capabilities": {"extensions": []}}, _Env()) + assert (result["supported"], result["advertised"], synced) == (False, False, []) + assert result["skipped_reason"] == "clock/v1 sync_time not advertised on live agent card" + assert persisted.metadata["validated_a2a_extensions"][EXT_CLOCK_URI] == result + + +@pytest.mark.asyncio +async def test_an_agent_that_follows_shows_each_time_the_clock_is_at(monkeypatch, persisted, synced): + _shows(monkeypatch, _env_time) + env = _Env() + result = await _validate(CLOCK_CARD, env) + assert (result["supported"], result["synced"], result["follows_writes"]) == (True, True, True) + assert [r["shown"] for r in result["readings"]] == ["2031-03-03T23:06:07-0600", "2031-03-04T23:06:07-0600"] + assert [(s.virtual_time, s.virtual_seconds_per_real_second, s.timezone, s.require_agent_clock) for s in synced] == [ + (CLOCK_PROBE_TIMES[0], 0, CLOCK_PROBE_ZONE, True)] + assert env.calls == [("advance", {"virtual_time": CLOCK_PROBE_TIMES[1], "wait_followers_s": 10.0})] + assert persisted.metadata["validated_a2a_extensions"][EXT_CLOCK_URI]["supported"] is True + + +@pytest.mark.asyncio +async def test_an_agent_whose_commands_read_real_time_fails(monkeypatch, persisted, synced): + _shows(monkeypatch, _real_time) + result = await _validate(CLOCK_CARD, _Env()) + assert (result["supported"], result["synced"], result["follows_writes"]) == (False, True, True) + assert [(r["time"], r["zone"]) for r in result["readings"]] == [(False, True), (False, True)] + + +@pytest.mark.asyncio +async def test_an_agent_that_doesnt_acknowledge_a_write_fails_naming_it(monkeypatch, persisted, synced): + _shows(monkeypatch, _env_time) + result = await _validate(CLOCK_CARD, _Env(write_status=504)) + assert (result["supported"], result["synced"], result["follows_writes"], len(result["readings"])) == (False, True, False, 1) + assert result["error"].startswith("HTTP 504") and "agent-agent" in result["error"] + + +@pytest.mark.asyncio +async def test_an_agent_that_cant_follow_fails_with_why(monkeypatch, persisted): + async def refuse(self, context): + raise StepContextError("agent 'agent' can't follow the clock of env 'mail': no_follower_fields") + + monkeypatch.setattr(SyncEnvClockTaskStep, "execute", refuse) + result = await _validate(CLOCK_CARD, _Env()) + assert (result["supported"], result["synced"], result["readings"]) == (False, False, []) + assert result["error"] == "StepContextError: agent 'agent' can't follow the clock of env 'mail': no_follower_fields" diff --git a/tst/unit/env/gateway/clock_watch_test.py b/tst/unit/env/gateway/clock_watch_test.py index 552a74f31..e18caa897 100644 --- a/tst/unit/env/gateway/clock_watch_test.py +++ b/tst/unit/env/gateway/clock_watch_test.py @@ -79,12 +79,15 @@ def test_the_card_offers_every_clock_method_with_the_waits_bound_and_the_watchs_ assert {name: (m["method"], m["endpoint"]) for name, m in methods.items()} == { "get_time": ("GET", "/clock/time"), "set_time": ("PUT", "/clock/set-time"), "set_rate": ("PUT", "/clock/set-rate"), "advance": ("PUT", "/clock/advance"), "clear": ("POST", "/clock/clear"), "state": ("GET", "/clock/state"), + "declare_follower": ("PUT", "/clock/followers"), "retire_follower": ("DELETE", "/clock/followers"), "watch": ("GET", "/clock/watch")} assert all(methods[name]["request"]["properties"]["wait_followers_s"] == wait for name in ("set_time", "set_rate", "advance", "clear")) assert methods["set_rate"]["request"]["required"] == ["virtual_seconds_per_real_second"] assert methods["set_time"]["request"]["required"] == ["virtual_time"] assert set(methods["watch"]["request"]["properties"]) == {"after", "follower"} + assert methods["declare_follower"]["request"]["required"] == ["name", "read_token"] + assert methods["retire_follower"]["request"]["required"] == ["name"] assert methods["watch"]["hold_s"] == CLOCK_WATCH_HOLD_S == ClockFollowers.WATCH_S == 25.0 @@ -432,3 +435,111 @@ async def test_a_write_stops_waiting_for_a_computer_when_it_goes_quiet_during_th waited = time.monotonic() - started assert answer.status_code == 504 and answer.json()["stale"] == ["laptop"], answer.text assert 0.5 <= waited < 3, waited + + +@pytest.mark.asyncio +async def test_an_agent_declared_after_the_gateway_started_is_waited_for_by_its_own_token(gateway): + gw = gateway(followers={}) + async with _client(gw) as client: + declared = await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "token-agent"}) + assert declared.status_code == 200 and declared.json()["followers"] == { + "agent-solver": {"version": None, "seen_s_ago": None}} + follower = asyncio.create_task(_follow(client, "agent-solver", "token-agent", applies_after_s=0.05)) + try: + await _until_shown(gw, "agent-solver") + written = await client.put("/clock/set-time", json={"virtual_time": T0, "wait_followers_s": 2}) + finally: + follower.cancel() + assert written.status_code == 200 + assert gw._clock_followers.state()["agent-solver"]["version"] == gw._clock.version + + +@pytest.mark.asyncio +async def test_an_agent_declared_again_takes_its_new_token_and_its_old_one_acknowledges_nothing(gateway): + gw = gateway(followers={}) + async with _client(gw) as client: + await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "first"}) + await client.put("/clock/set-time", json={"virtual_time": T0, "virtual_seconds_per_real_second": 0}) + await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "second"}) + old = asyncio.create_task(_watch(client, gw._clock.version, "agent-solver", "first")) + await asyncio.sleep(0.2) + by_the_old_token = gw._clock_followers.state()["agent-solver"]["version"] + new = asyncio.create_task(_watch(client, gw._clock.version, "agent-solver", "second")) + try: + await _until_shown(gw, "agent-solver") + finally: + old.cancel() + new.cancel() + assert by_the_old_token is None + + +@pytest.mark.parametrize("body", [{"name": "agent-solver"}, {"read_token": "t"}, {"name": "", "read_token": "t"}, [1]]) +@pytest.mark.asyncio +async def test_declaring_a_follower_takes_a_name_and_a_read_token(gateway, body): + gw = gateway(followers={}) + async with _client(gw) as client: + answer = await client.put("/clock/followers", json=body) + assert answer.status_code == 400 and gw._clock_followers.state() == {} + + +@pytest.mark.asyncio +async def test_only_the_envs_control_token_declares_or_retires_a_follower(gateway): + gw = gateway(followers={}, control_token="control") + control = {"Authorization": "Bearer control"} + async with _client(gw) as client: + refused = await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "t"}) + taken = await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "t"}, headers=control) + kept = await client.request("DELETE", "/clock/followers", json={"name": "agent-solver"}) + retired = await client.request("DELETE", "/clock/followers", json={"name": "agent-solver"}, headers=control) + assert (refused.status_code, taken.status_code, kept.status_code, retired.status_code) == (401, 200, 401, 200) + + +@pytest.mark.asyncio +async def test_a_retired_agent_is_no_longer_waited_for_or_named_and_its_token_acknowledges_nothing(gateway): + gw = gateway(followers={}) + async with _client(gw) as client: + await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "token-agent"}) + retired = await client.request("DELETE", "/clock/followers", json={"name": "agent-solver"}) + started = time.monotonic() + written = await client.put("/clock/set-time", json={"virtual_time": T0, "wait_followers_s": 5}) + waited = time.monotonic() - started + assert gw._clock_followers.follower_for("agent-solver", "token-agent") is None + assert retired.status_code == 200 and retired.json() == {"ok": True, "retired": True, "followers": {}} + assert written.status_code == 200 and waited < 1, (written.text, waited) + + +@pytest.mark.asyncio +async def test_a_follower_the_gateway_started_with_is_neither_retired_nor_redeclared(gateway): + gw = gateway(followers={"laptop": "token-laptop"}) + async with _client(gw) as client: + retired = await client.request("DELETE", "/clock/followers", json={"name": "laptop"}) + redeclared = await client.put("/clock/followers", json={"name": "laptop", "read_token": "mine"}) + unknown = await client.request("DELETE", "/clock/followers", json={"name": "agent-gone"}) + assert retired.json()["retired"] is False and unknown.json()["retired"] is False + assert redeclared.status_code == 400 and "started with" in redeclared.text + assert gw._clock_followers.follower_for("laptop", "token-laptop") == "laptop" + + +@pytest.mark.asyncio +async def test_a_write_waiting_for_an_agent_answers_once_the_agent_is_retired(gateway): + gw = gateway(followers={}) + async with _client(gw) as client: + await client.put("/clock/followers", json={"name": "agent-solver", "read_token": "token-agent"}) + started = time.monotonic() + write = asyncio.create_task(client.put("/clock/set-time", json={"virtual_time": T0, "wait_followers_s": 20})) + await asyncio.sleep(0.3) + await client.request("DELETE", "/clock/followers", json={"name": "agent-solver"}) + written = await asyncio.wait_for(write, 5) + waited = time.monotonic() - started + assert written.status_code == 200 and waited < 3, (written.text, waited) + + +@pytest.mark.asyncio +async def test_a_watch_by_a_follower_retired_meanwhile_acknowledges_nothing(gateway): + gw = gateway(followers={}) + gw._clock_followers.declare("agent-solver", "token-agent") + named = gw._clock_followers.follower_for("agent-solver", "token-agent") + await gw._clock_followers.retire("agent-solver") + gw._clock_followers.WATCH_S = 0.05 + await gw._clock_followers.watch("", named) + assert gw._clock_followers.state() == {} diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index 5b5458b0c..6a9a22771 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -20,6 +20,7 @@ "/tools/disable": "POST", "/tools/enable": "POST", "/triggers/register": "POST", "/triggers/remove": "POST", "/triggers/clear": "POST", "/notifications/config": "POST", "/notifications/take": "POST", "/notifications/clear": "POST", + "/clock/followers": "PUT", } OPEN = {"/state": "GET", "/step": "POST", "/clock/time": "GET", "/clock/state": "GET", "/triggers/state": "GET", "/notifications/state": "GET", "/trajectory": "GET", "/changes": "GET", "/.well-known/agent-env.json": "GET", diff --git a/tst/unit/task_step/agent_clock_test.py b/tst/unit/task_step/agent_clock_test.py new file mode 100644 index 000000000..089a28b24 --- /dev/null +++ b/tst/unit/task_step/agent_clock_test.py @@ -0,0 +1,321 @@ +"""Agents on the env's clock: once sync_env_clock has armed it, every agent registered with the env whose card serves +clock/v1 sync_time is declared at the gateway as a follower with a token of its own and handed the follower's config; +one that doesn't is reported and skipped, even when the env's servers must follow (tolerate_missing_sync_time false), +or fails the step under require_agent_clock; and an agent +deploy_agent deploys while the clock is armed follows the same way.""" +import json +from types import SimpleNamespace + +import pytest + +from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.task_step.context import DeployedAgent, StepContextError, TaskStepContext +from agent_env.a2a_agent import clock as agent_clock +from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep +from tst.unit.task_step.sync_env_clock_test import (CLOCK_URL, T0, _CARD, _CLOCK_EXT, _ZONE_EXT, _carded, _composed_card, _mock_http, + _step) + +AGENT_URL = "http://solver:8000" +SYNC_TIME = {"method": "POST", "endpoint": "/ext/clock/sync-time", + "request": {"required": ["env_get_time_url"], "optional": ["timezone", "watch_url", "name", "read_token"]}} + + +def _agent(name="solver", sync_time=SYNC_TIME, env_ids=("e1",)) -> DeployedAgent: + extensions = [{"uri": EXT_CLOCK_URI, "params": {"endpoint": "/ext/clock", "methods": {"sync_time": sync_time}}}] + return DeployedAgent(agent_name=name, api_url=AGENT_URL, a2a_url=AGENT_URL, env_ids=list(env_ids), + a2a_card={"capabilities": {"extensions": extensions if sync_time else []}}) + + +async def _context(*agents) -> TaskStepContext: + """An env whose one MCP server follows the clock, and ``agents``.""" + return TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))], + deployed_agents=list(agents)) + + +def _no_servers(monkeypatch): + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) + + +@pytest.mark.asyncio +async def test_an_agent_is_declared_a_follower_then_handed_its_config_with_the_same_token(monkeypatch): + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + context = await _context(_agent()) + context.deployed_envs = [_carded(await _composed_card({"mcp-email": [_ZONE_EXT]}))] + + await SyncEnvClockTaskStep(id="sync", version=1, env_id="e1", virtual_time="2026-01-01T06:00:00", + timezone="America/Chicago").execute(context) + + declared, synced = sent[3], sent[4] + assert (declared.method, str(declared.url)) == ("PUT", f"{_CARD}/clock/followers") + assert (synced.method, str(synced.url)) == ("POST", f"{AGENT_URL}/ext/clock/sync-time") + declaration, config = json.loads(declared.content), json.loads(synced.content) + assert declaration == {"name": "agent-solver", "read_token": config["read_token"]} and len(config["read_token"]) > 30 + assert config == {"env_get_time_url": CLOCK_URL, "watch_url": "http://gw/clock/watch", "name": "agent-solver", + "read_token": declaration["read_token"], "timezone": "America/Chicago"} + [agent] = context.metadata["clock_configurations"][0]["agents"] + assert (agent["agent"], agent["synced"], agent["follower"]) == ("solver", True, "agent-solver") + + +@pytest.mark.asyncio +async def test_only_the_agents_registered_with_the_env_follow_its_clock(monkeypatch): + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + + await _step().execute(await _context(_agent("elsewhere", env_ids=("e2",)))) + + assert [r.url.path for r in sent][3:] == [] + + +@pytest.mark.parametrize("sync_time, reason", [ + (None, "not_advertised"), + ({**SYNC_TIME, "request": {"required": ["env_get_time_url"], "optional": ["timezone"]}}, "no_follower_fields"), +]) +@pytest.mark.asyncio +async def test_an_agent_that_cant_follow_is_reported_and_skipped_or_fails_a_strict_step(monkeypatch, sync_time, reason): + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + context = await _context(_agent(sync_time=sync_time)) + + await _step().execute(context) + assert context.metadata["clock_configurations"][0]["agents"] == [{"agent": "solver", "synced": False, + "reason": reason}] + assert [r.url.path for r in sent][3:] == [] + strict = await _context(_agent(sync_time=sync_time)) + await _step(tolerate_missing_sync_time=False).execute(strict) + assert strict.metadata["clock_configurations"][0]["agents"][0]["synced"] is False + with pytest.raises(StepContextError, match=f"agent 'solver' can't follow the clock of env 'e1': {reason}"): + await _step(require_agent_clock=True).execute(await _context(_agent(sync_time=sync_time))) + + +@pytest.mark.asyncio +async def test_a_gateway_from_before_followers_after_start_is_reported_and_the_agent_skipped(monkeypatch): + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + context = await _context(_agent()) + methods = next(e for e in context.deployed_envs[0].environment_card["capabilities"]["extensions"] + if e["uri"] == EXT_CLOCK_URI)["params"]["methods"] + del methods["declare_follower"] + + await _step().execute(context) + + assert context.metadata["clock_configurations"][0]["agents"][0]["reason"] == "gateway_takes_no_followers" + assert [r.url.path for r in sent][3:] == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fails, why", [ + ("404", "\\(HTTP 404\\): .*isn't following"), + ("500", "\\(HTTP 500\\): .*boom"), + ("unreachable", "\\(ConnectError\\): .*refused"), +]) +async def test_an_agent_that_serves_sync_time_and_fails_fails_the_step_and_its_follower_is_retired(monkeypatch, fails, + why): + import httpx + + real, sent = httpx.AsyncClient, [] + + def handle(request: httpx.Request) -> httpx.Response: + sent.append((request.method, request.url.path)) + if request.url.path == "/ext/clock/sync-time": + if fails == "unreachable": + raise httpx.ConnectError("connection refused", request=request) + return httpx.Response(int(fails), json={"detail": "target_unavailable: this agent's clock follower isn't " + "following" if fails == "404" else "boom"}) + return httpx.Response(200, json={"env_get_time_url": CLOCK_URL} if request.url.path.endswith("/state") else {}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + _no_servers(monkeypatch) + context = await _context(_agent()) + + with pytest.raises(RuntimeError, match=f"agent 'solver' doesn't follow the clock of env 'e1' {why}"): + await _step().execute(context) + + # Declared, then retired: the gateway isn't left waiting for a follower that never watches. + assert [(method, "/" + "/".join(path.split("/")[-2:])) for method, path in sent[-3:]] == [ + ("PUT", "/clock/followers"), ("POST", "/clock/sync-time"), ("DELETE", "/clock/followers")] + assert agent_clock.followed_env(context, _agent()) is None + + +@pytest.mark.asyncio +async def test_an_agent_deployed_while_the_clock_is_armed_follows_it(monkeypatch): + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + context = await _context() + context.metadata["clock_configurations"] = [ + {"step_id": "sync", "env_id": "e1", "virtual_time": T0, "env_get_time_url": CLOCK_URL, "timezone": "Asia/Tokyo", + "require_agent_clock": False}, + {"step_id": "rate", "env_id": "e1", "virtual_seconds_per_real_second": 60, "virtual_time": T0}, + ] + late = _agent("late") + + outcomes = await agent_clock.follow_armed_clocks(context, late, timeout=5) + + assert [(r.method, str(r.url)) for r in sent] == [("PUT", f"{_CARD}/clock/followers"), + ("POST", f"{AGENT_URL}/ext/clock/sync-time")] + assert json.loads(sent[1].content)["timezone"] == "Asia/Tokyo" + assert [(o["env_id"], o["agent"], o["synced"]) for o in outcomes] == [("e1", "late", True)] + + +@pytest.mark.asyncio +async def test_an_agent_deployed_before_any_clock_is_armed_is_left_alone(monkeypatch): + sent = _mock_http(monkeypatch) + assert await agent_clock.follow_armed_clocks(await _context(), _agent(), timeout=5) == [] + assert sent == [] + + +@pytest.mark.asyncio +async def test_a_late_agent_that_cant_follow_fails_a_deploy_only_under_require_agent_clock(monkeypatch): + _mock_http(monkeypatch) + context = await _context() + context.metadata["clock_configurations"] = [{"env_id": "e1", "env_get_time_url": CLOCK_URL}] + [skipped] = await agent_clock.follow_armed_clocks(context, _agent("late", sync_time=None), timeout=5) + assert (skipped["synced"], skipped["reason"]) == (False, "not_advertised") + context.metadata["clock_configurations"][0]["require_agent_clock"] = True + with pytest.raises(StepContextError, match="agent 'late' can't follow the clock of env 'e1': not_advertised"): + await agent_clock.follow_armed_clocks(context, _agent("late", sync_time=None), timeout=5) + + +def test_require_agent_clock_round_trips_and_is_left_out_when_off(): + assert "require_agent_clock" not in _step().to_dict() + strict = _step(require_agent_clock=True).to_dict() + assert strict["require_agent_clock"] is True + assert SyncEnvClockTaskStep.from_dict(strict).require_agent_clock is True + + +@pytest.mark.asyncio +async def test_an_agent_follows_the_first_envs_clock_and_another_envs_is_reported_and_skipped(monkeypatch): + sent = _mock_http(monkeypatch) + context = await _context() # e1 alone is deployed and armed: whether e0 has been armed yet doesn't matter + context.metadata["clock_configurations"] = [{"env_id": "e1", "env_get_time_url": CLOCK_URL}] + + [outcome] = await agent_clock.follow_armed_clocks(context, _agent("late", env_ids=("e0", "e1")), timeout=5) + + assert (outcome["env_id"], outcome["synced"], outcome["reason"], outcome["follows"]) == ( + "e1", False, "follows_another_env", "e0") + assert sent == [] and agent_clock.FOLLOWING_KEY not in context.metadata + + +@pytest.mark.asyncio +async def test_a_teardown_retires_the_agents_follower_at_each_gateway_it_followed(monkeypatch): + from agent_env.task_step.task_steps.teardown_sandboxes import TeardownSandboxesTaskStep + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + context = await _context(_agent(), _agent("bystander")) + await _step().execute(context) + del sent[:] + + async def nothing(*a, **k): + return SimpleNamespace(terminated=[], failed=[]) + + monkeypatch.setattr("agent_env.task_step.task_steps.teardown_sandboxes.before_teardown", nothing) + monkeypatch.setattr("agent_env.task_step.task_steps.teardown_sandboxes.tear_down", nothing) + await TeardownSandboxesTaskStep(id="down", version=1, agent_names=["solver"]).execute(context) + + assert [(r.method, str(r.url), json.loads(r.content)) for r in sent] == [ + ("DELETE", f"{_CARD}/clock/followers", {"name": "agent-solver"})] + assert context.metadata[agent_clock.FOLLOWING_KEY] == { + "bystander": {"agent": "bystander", "env_id": "e1", "follower": "agent-bystander"}} + assert agent_clock.followed_env(context, _agent()) is None + + +@pytest.mark.asyncio +async def test_a_gateway_that_cant_retire_a_follower_doesnt_fail_the_teardown(monkeypatch): + _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + context = await _context(_agent()) + await _step().execute(context) + methods = next(e for e in context.deployed_envs[0].environment_card["capabilities"]["extensions"] + if e["uri"] == EXT_CLOCK_URI)["params"]["methods"] + del methods["retire_follower"] + + assert await agent_clock.retire_followers(context, ["solver"]) == [] + assert agent_clock.followed_env(context, _agent()) == "e1" + + +def test_a_log_of_an_outcome_leaves_out_the_agents_answer(): + assert agent_clock.summary({"synced": True, "follower": "agent-solver", "result": {"big": "x" * 10_000}}) == { + "synced": True, "follower": "agent-solver"} + + +@pytest.mark.asyncio +async def test_late_agents_deployed_at_once_each_save_only_their_own_record(monkeypatch): + import copy + + from agent_env.task_step.context_ops import build_context_update_ops + + _mock_http(monkeypatch) + pre = await _context() + pre.metadata["clock_configurations"] = [{"env_id": "e1", "env_get_time_url": CLOCK_URL}] + saved = {} + for name in ("first", "second"): # each from the same context, as two deploy_agent steps at once + post = copy.deepcopy(pre) + await agent_clock.follow_armed_clocks(post, _agent(name), timeout=5) + ops = build_context_update_ops(pre, post) + saved[name] = {*ops.sets, *ops.add_to_sets, *ops.unsets} + + for name, paths in saved.items(): + assert paths and all(p.startswith(f"context.metadata.{agent_clock.FOLLOWING_KEY}.{name}") for p in paths), paths + + +@pytest.mark.parametrize("names", [("team.solver", "team_solver", "team%2Esolver"), ("$x", "_x", "%24x")]) +def test_agent_names_that_look_alike_keep_keys_of_their_own(names): + slots = [agent_clock._slot(name) for name in names] + assert len(set(slots)) == len(names) and not any("." in s or s.startswith("$") for s in slots) + + +@pytest.mark.parametrize("by", ["sandbox_names", "env_ids"]) +@pytest.mark.asyncio +async def test_an_agent_whose_sandbox_goes_down_by_another_name_is_retired_too(monkeypatch, by): + from agent_env.task_step.context import DeployedSandbox + from agent_env.task_step.task_steps.teardown_sandboxes import TeardownSandboxesTaskStep + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + agent = _agent() + agent.sandbox_id = "sb-1" if by == "env_ids" else "sb-shared" + context = await _context(agent) + context.deployed_sandboxes = [DeployedSandbox(sandbox_name="box", sandbox_id="sb-shared", sandbox_mode="container", + sandbox_type="local")] + await _step().execute(context) + del sent[:] + + async def nothing(*a, **k): + return SimpleNamespace(terminated=[], failed=[]) + + monkeypatch.setattr("agent_env.task_step.task_steps.teardown_sandboxes.before_teardown", nothing) + monkeypatch.setattr("agent_env.task_step.task_steps.teardown_sandboxes.tear_down", nothing) + named = {"sandbox_names": ["box"]} if by == "sandbox_names" else {"env_ids": ["e1"]} + await TeardownSandboxesTaskStep(id="down", version=1, **named).execute(context) + + assert [(r.method, json.loads(r.content)) for r in sent] == [("DELETE", {"name": "agent-solver"})] + + +@pytest.mark.asyncio +async def test_tearing_down_one_of_two_look_alike_agents_retires_its_own_follower_alone(monkeypatch): + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + context = await _context(_agent("team.solver"), _agent("team_solver")) + await _step().execute(context) + del sent[:] + + assert await agent_clock.retire_followers(context, ["team.solver"]) == ["agent-team.solver"] + assert [json.loads(r.content) for r in sent] == [{"name": "agent-team.solver"}] + assert agent_clock.followed_env(context, _agent("team_solver")) == "e1" + + +@pytest.mark.asyncio +async def test_a_later_sync_leaves_an_agent_teardown_took_down_alone(monkeypatch): + from agent_env.task.teardown import TORN_DOWN_KEY + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + _no_servers(monkeypatch) + gone, kept = _agent("gone"), _agent("kept") + gone.sandbox_id, kept.sandbox_id = "sb-gone", "sb-kept" + context = await _context(gone, kept) + context.metadata[TORN_DOWN_KEY] = ["sb-gone"] + + await _step(require_agent_clock=True).execute(context) + + assert [a["agent"] for a in context.metadata["clock_configurations"][0]["agents"]] == ["kept"] + assert not any("agent-gone" in (r.content or b"").decode() for r in sent)