From d5629e0d7cf20ea7f0e111d560dadcf10c673910 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 17:18:18 -0800 Subject: [PATCH 1/8] fix(computer): a disk's computer on local Docker uses local grants, as a container does A Windows computer's recording never reached a local object store: its guest couldn't resolve host.docker.internal, the host local grants name, nor trust the local transfer CA. As lume's VMs do, the guest now maps that name to 10.0.2.2, QEMU's address for the launcher, which forwards the grant server's port to this machine (VM_HOST_PORTS), and the CA's trust files ride in the seed (VM_TRUST), where Linux's init and Windows' agentenv-session point TLS clients at them. The local provider passes both when the store hands out grants; lume's helper for them is now shared. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/README.md | 5 +- src/agent_env/computers/disk/linux/boot/init | 10 +++- .../computers/disk/linux/launcher/launch | 14 +++++- .../disk/windows/agentenv-session.py | 13 ++++- .../computers/disk/windows/setup.ps1 | 2 + .../local_docker/provider.py | 26 +++++++++- .../sandbox_providers/lume/provider.py | 17 ++----- tst/unit/env/envs/test_computer_windows.py | 16 +++++++ tst/unit/env/envs/test_disk_recipe.py | 48 ++++++++++++++++++- .../lume_sandbox_provider_test.py | 8 ++-- 10 files changed, 133 insertions(+), 26 deletions(-) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index ef1868fb9..6a8764286 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -39,7 +39,10 @@ a reset boots the disk afresh on a new overlay in the same launcher, at the same before reset in place started is refused a reset, to be redeployed. It boots on local Docker where Docker can pass it `/dev/kvm`, the disk image's files mounted read-only into the launcher where Docker mounts images (28 and later, on the containerd image store), so no copy of a disk, nor of a Windows disk's base, is made for each launcher, else copied into -an image of the launcher it keeps for the next boot; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as +an image of the launcher it keeps for the next boot. There its computer uses a local object store's grants as a container +does, at `host.docker.internal`: Linux's init and Windows' setup map that name to 10.0.2.2, QEMU's address for the +launcher, which forwards the grant server's port to this machine (`VM_HOST_PORTS`), and the local transfer CA's trust +files ride in the seed (`VM_TRUST`), where the init and agentenv-session point TLS clients at them; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as its image under nested virtualization: the provider exports the disk from the image store with this machine's Docker, compressed, once for each composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal uploads once for each content and the launcher expands onto the sandbox's disk; a thin disk, a Windows one, carries its diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init index 886180daf..07d38b977 100755 --- a/src/agent_env/computers/disk/linux/boot/init +++ b/src/agent_env/computers/disk/linux/boot/init @@ -2,6 +2,8 @@ # The disk form's first process: mounts the root filesystem the computer's image became, brings up the network, reads # the image's start (/etc/agentenv/vm/image.env, from its OCI config) and the deploy's environment (the NoCloud seed's # user-data), then becomes that start, as its user, as the container runtime would: PID 1 is the image's entrypoint. +# As a container finds this machine at host.docker.internal, it finds the launcher at QEMU's address for it, which +# forwards a local grant server's port, and trusts the local transfer CA the seed carries, as a container does. /bin/busybox --install -s /bin export PATH=/bin mount -t proc proc /proc @@ -35,11 +37,17 @@ while read -r line; do workdir) decode "$value"; workdir=$decoded ;; esac done < "$config" +if [ -f /seed/trust/ca-bundle.pem ]; then + cp /seed/trust/ca-bundle.pem /seed/trust/ca.pem /newroot/etc/agentenv/ || fail "can't copy the seed's trust files" + export SSL_CERT_FILE=/etc/agentenv/ca-bundle.pem REQUESTS_CA_BUNDLE=/etc/agentenv/ca-bundle.pem \ + NODE_EXTRA_CA_CERTS=/etc/agentenv/ca.pem +fi [ -n "$seed" ] && { seed_environment /seed || fail "$refused"; } hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data) hostname "${hostname:-computer}" hostname > /newroot/etc/hostname -printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n' "$(hostname)" > /newroot/etc/hosts +printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n10.0.2.2\thost.docker.internal\n' \ + "$(hostname)" > /newroot/etc/hosts [ -s /resolv.conf ] && cp /resolv.conf /newroot/etc/resolv.conf entry=$(awk -F: -v u="${user%%:*}" '$1 == u || $3 == u' /newroot/etc/passwd | head -n 1) [ -n "$entry" ] || fail "the image's user ${user%%:*} isn't in /etc/passwd" diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index d3acc6516..d2ac70abb 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -14,6 +14,9 @@ # VM_PORTS guest TCP ports, each served on the same port here by socat, whose listen backlog takes a burst of # connections, in front of QEMU's forward on 127.0.0.1 at the port plus 10000, whose backlog is 1; # socat retries a connection that forward drops every half second, not on the kernel's SYN backoff +# VM_HOST_PORTS ports of this machine's host the guest reaches, as host.docker.internal, at 10.0.2.2, QEMU's address +# for here: socat serves each on 127.0.0.1 here, through to the host, as a local grant server's +# VM_TRUST a folder of trust files, a local transfer CA's, that go into the seed as its trust/ folder # VM_ARCH the disk's arch, amd64 or arm64; this machine's by default # VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default # VM_REQUIRE_KVM when set, fail rather than emulate where KVM can't run the disk @@ -108,6 +111,14 @@ for port in ${VM_PORTS:-}; do socat "TCP-LISTEN:$port,fork,reuseaddr,backlog=1024" "TCP:127.0.0.1:$behind,connect-timeout=0.5,retry=600,interval=0.05" & fronts+=("$!") done +for port in ${VM_HOST_PORTS:-}; do + if [[ " ${VM_PORTS:-} " == *" $port "* || " ${VM_PORTS:-} " == *" $((port - 10000)) "* ]]; then + echo "launch: host port $port can't be forwarded: a guest port's proxy or its forward here takes it" >&2 + exit 2 + fi + socat "TCP-LISTEN:$port,bind=127.0.0.1,fork,reuseaddr" "TCP:host.docker.internal:$port" & + fronts+=("$!") +done trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null; exit 143' TERM INT if [[ "$disk" == *.zst ]]; then @@ -135,8 +146,9 @@ boot() { qemu-img create -q -f qcow2 -b "$disk" -F qcow2 "$booted" fi seed "$work/seed" + [ -z "${VM_TRUST:-}" ] || cp -r "$VM_TRUST" "$work/seed/trust" mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null - mcopy -i "$work/seed.img" "$work/seed/"* ::/ + mcopy -s -i "$work/seed.img" "$work/seed/"* ::/ if [ "$guest" = windows ]; then cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd "$work/vars.fd" mkdir "$work/tpm" diff --git a/src/agent_env/computers/disk/windows/agentenv-session.py b/src/agent_env/computers/disk/windows/agentenv-session.py index e29fa2f26..0cb7d89f1 100644 --- a/src/agent_env/computers/disk/windows/agentenv-session.py +++ b/src/agent_env/computers/disk/windows/agentenv-session.py @@ -3,7 +3,8 @@ in its prefix with the build-phase settings the disk fixes, the deploy's environment over them, read from the NoCloud seed the launcher attaches (the CIDATA volume; agentenv_protocol.computer.seed, version 1), and its own environment over that; one that exits, or couldn't start, starts again after a pause, what kept it from starting written to its log. It -waits for the seed, and refuses one that carries only versions it doesn't read, as an init does.""" +waits for the seed, and refuses one that carries only versions it doesn't read, as an init does. A seed's trust folder, +a local transfer CA's, is what TLS clients are pointed at, as a container's are.""" import base64 import ctypes @@ -64,7 +65,7 @@ def main() -> None: raise SystemExit(f"agentenv-session: no CIDATA seed after {WAIT_S} s") time.sleep(1) fixed = json.loads(BUILD_SETTINGS.read_text()) if BUILD_SETTINGS.is_file() else {} - base = {**os.environ, **fixed, **environment(folder)} + base = {**os.environ, **fixed, **trust(folder / "trust"), **environment(folder)} specs = [json.loads(path.read_text()) for path in sorted(SERVICES.glob("*.json"))] running = {spec["name"]: start(spec, base) for spec in specs} while True: @@ -74,6 +75,14 @@ def main() -> None: running[spec["name"]] = start(spec, base) +def trust(folder: Path) -> dict[str, str]: + """The variables that point TLS clients at the trust files in ``folder``, none when it has none.""" + if not (folder / "ca-bundle.pem").is_file(): + return {} + return {"SSL_CERT_FILE": str(folder / "ca-bundle.pem"), "REQUESTS_CA_BUNDLE": str(folder / "ca-bundle.pem"), + "NODE_EXTRA_CA_CERTS": str(folder / "ca.pem")} + + def start(spec: dict, base: dict[str, str]) -> subprocess.Popen | None: """``spec``'s service started, its output to its log; None when it couldn't be, the reason in its log if that opens.""" diff --git a/src/agent_env/computers/disk/windows/setup.ps1 b/src/agent_env/computers/disk/windows/setup.ps1 index 1be6b9142..373e6ad02 100644 --- a/src/agent_env/computers/disk/windows/setup.ps1 +++ b/src/agent_env/computers/disk/windows/setup.ps1 @@ -41,6 +41,8 @@ try { icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r /grant '*S-1-5-32-544:F' /grant 'SYSTEM:F' | Out-Null # QEMU's user network reaches the guest at the ports the launcher forwards alone, which the layers' services listen on. Set-NetFirewallProfile -Profile Domain, Private, Public -Enabled False + # The launcher, at 10.0.2.2, forwards a local grant server's port to the machine it runs on, as host.docker.internal. + Add-Content -Path "$env:SystemRoot\System32\drivers\etc\hosts" -Value "`r`n10.0.2.2 host.docker.internal" -Encoding ASCII Set-Service sshd -StartupType Automatic Start-Service sshd diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index ab4efc79e..506acd96f 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -607,6 +607,16 @@ def local_grant_trust() -> Path | None: return None +def local_grants() -> Optional[tuple[int, Path]]: + """The port of the configured object store's grant server, started, and the local transfer CA's trust files, when + the store is a local one handing out grants; None otherwise.""" + trust = local_grant_trust() + if trust is None: + return None + store = config.get_config().get_object_store() + return (store.local if isinstance(store, LocalRunObjectStore) else store).grant_port(), trust + + async def start_trusting(sandbox: VmSandbox, container: str, trust_dir: Path) -> None: """Copy ``trust_dir`` into the created ``container`` where ``LOCAL_TRUST_ENV`` points, as the sandbox's host can reach it, then start it. A container that cannot be given the files or started is removed, so its name is free @@ -733,10 +743,11 @@ async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, ports = list(sandbox._port_map) publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port) for arg in ("-p", spec)] + grants = await asyncio.to_thread(local_grants) running = asyncio.ensure_future(asyncio.to_thread( docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", - *publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]), machine.image, - timeout=_DOCKER_QUERY_SECONDS)) + *publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]), + *(_grant_args(*grants) if grants else ()), machine.image, timeout=_DOCKER_QUERY_SECONDS)) try: await asyncio.shield(running) except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it @@ -959,6 +970,17 @@ def _host_ips() -> tuple[str, ...]: timeout=_DOCKER_QUERY_SECONDS)) +def _grant_args(port: int, trust: Path) -> list[str]: + """The ``docker run`` arguments that let a disk's computer use this machine's local grants, as lume's VMs do: it + finds ``host.docker.internal`` at QEMU's address for the launcher, where the launcher forwards the grant server's + ``port`` to this machine, and trusts the local transfer CA, whose ``trust`` files the launcher puts in its seed.""" + return [*(arg for entry in _EXTRA_HOSTS for arg in ("--add-host", entry)), + "--mount", f"type=bind,src={trust},dst={_LAUNCHER_TRUST},readonly", + "-e", f"VM_HOST_PORTS={port}", "-e", f"VM_TRUST={_LAUNCHER_TRUST}"] + + _DOCKER_QUERY_SECONDS = 30 #: How long a reset may take: longer than the minute the launcher's reset waits for the computer to start again. _RESET_SECONDS = 90 +#: Where a disk's launcher finds the local transfer CA's trust files. +_LAUNCHER_TRUST = "/agentenv-trust" diff --git a/src/agent_env/providers/sandbox_providers/lume/provider.py b/src/agent_env/providers/sandbox_providers/lume/provider.py index f1e2da52f..da032a1c9 100644 --- a/src/agent_env/providers/sandbox_providers/lume/provider.py +++ b/src/agent_env/providers/sandbox_providers/lume/provider.py @@ -38,7 +38,7 @@ from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.local_docker.provider import ( LocalSandboxProvider, - local_grant_trust, + local_grants, trust_env, ) from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox @@ -49,7 +49,6 @@ ) from agent_env.store.image_store.registry_api import pin_digest from agent_env.store.local_state import awaiting_lock -from agent_env.store.routing import LocalRunObjectStore logger = logging.getLogger(__name__) @@ -203,7 +202,7 @@ async def reset_vm(self, sandbox: Any, *, image: str, env: dict[str, str]) -> No async def get_sandbox(self, sandbox_id: str) -> LumeSandbox: """The VM ``sandbox_id``, with this process's grant server forwarded into it, as another process reconnects.""" - grants = await asyncio.to_thread(_local_grants) + grants = await asyncio.to_thread(local_grants) async with self._changing(sandbox_id): state = self._state(sandbox_id) if "address" in state and grants is not None: @@ -349,7 +348,7 @@ async def _started(self, name: str, state: dict, env: dict[str, str]) -> None: if sizes: await self._lume("set", name, *[part for pair in zip(sizes[::2], sizes[1::2]) for part in pair]) folder = self._folder(name) - grants = None if state.get("build") else await asyncio.to_thread(_local_grants) + grants = None if state.get("build") else await asyncio.to_thread(local_grants) if grants is not None: env = {**trust_env(f"{SHARED_FILES}/{SEED_FOLDER}/{TRUST_FOLDER}"), **env} share = (f"{state['build']}:rw" if state.get("build") @@ -586,13 +585,3 @@ def _alive(pid: Optional[int]) -> bool: def _kill(pid: int) -> None: with contextlib.suppress(ProcessLookupError, PermissionError): os.killpg(pid, signal.SIGTERM) - - -def _local_grants() -> Optional[tuple[int, Path]]: - """The port of the configured object store's grant server, started, and the local transfer CA's trust files, when - the store is a local one handing out grants; None otherwise.""" - trust = local_grant_trust() - if trust is None: - return None - store = get_config().get_object_store() - return (store.local if isinstance(store, LocalRunObjectStore) else store).grant_port(), trust diff --git a/tst/unit/env/envs/test_computer_windows.py b/tst/unit/env/envs/test_computer_windows.py index 3803fc587..95c368afa 100644 --- a/tst/unit/env/envs/test_computer_windows.py +++ b/tst/unit/env/envs/test_computer_windows.py @@ -7,6 +7,7 @@ agentenv-session. The scripts the Windows test bundle's verifiers run inside the computer are Python.""" import ast +import importlib.util import json import threading import time @@ -443,3 +444,18 @@ def test_the_scripts_the_windows_bundles_verifiers_run_inside_the_computer_are_p if isinstance(node, ast.Assign) and [target.id for target in node.targets] == [script]] ast.parse(text) + + +def test_the_session_points_tls_clients_at_a_seeds_trust_folder_and_at_nothing_without_one(tmp_path): + spec = importlib.util.spec_from_file_location("agentenv_session", windows_disk.RECIPE / "agentenv-session.py") + session = importlib.util.module_from_spec(spec) + spec.loader.exec_module(session) + trust = tmp_path / "trust" + + assert session.trust(trust) == {} + trust.mkdir() + (trust / "ca-bundle.pem").write_text("roots and the local CA") + (trust / "ca.pem").write_text("the local CA") + assert session.trust(trust) == {"SSL_CERT_FILE": str(trust / "ca-bundle.pem"), + "REQUESTS_CA_BUNDLE": str(trust / "ca-bundle.pem"), + "NODE_EXTRA_CA_CERTS": str(trust / "ca.pem")} diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index e85d4ddd4..b25b3dd5f 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -222,6 +222,35 @@ def test_the_launcher_serves_each_port_through_socat_in_front_of_qemus_forward_o assert running.wait(10) == 143 +def test_the_launcher_forwards_each_host_port_from_loopback_here_to_the_host_and_seeds_the_trust_files(launcher, + tmp_path): + """The guest finds host.docker.internal at 10.0.2.2, which QEMU's network gives this container's loopback, and + finds a local transfer CA's trust files in its seed's trust folder.""" + launch, calls = launcher + trust = tmp_path / "trust" + trust.mkdir() + + running = launch(VM_PORTS="18765", VM_HOST_PORTS="41234", VM_TRUST=str(trust)) + + assert "socat TCP-LISTEN:41234,bind=127.0.0.1,fork,reuseaddr TCP:host.docker.internal:41234" in _logged( + calls, "socat", 2) + assert f"cp -r {trust} {tmp_path}/state/boot/seed/trust" in _logged(calls, "cp -r") + assert _logged(calls, "mcopy")[0].startswith(f"mcopy -s -i {tmp_path}/state/boot/seed.img {tmp_path}/state/boot/seed/") + running.terminate() + assert running.wait(10) == 143 + + +@pytest.mark.parametrize("host_ports", ["18765", "28765"], ids=["a-guest-ports-proxy", "its-forward"]) +def test_the_launcher_refuses_a_host_port_a_guest_ports_proxy_or_forward_takes(launcher, host_ports): + launch, calls = launcher + + refused = launch(VM_PORTS="18765", VM_HOST_PORTS=host_ports) + + assert refused.wait(10) == 2 + assert f"host port {host_ports} can't be forwarded: a guest port's proxy or its forward here takes it" in ( + refused.stderr.read()) + + @pytest.mark.parametrize("ports, why", [("55536", "above 65535"), ("6080 16080", "another of VM_PORTS")]) def test_the_launcher_refuses_a_port_whose_forward_behind_the_proxy_has_no_port_of_its_own(launcher, ports, why): launch, calls = launcher @@ -359,7 +388,7 @@ def test_a_windows_install_boots_its_iso_first_pressing_enter_and_finds_its_answ assert "-device ide-hd,drive=disk,bus=ide.0,bootindex=1,rotation_rate=1 " in qemu assert "-device usb-storage,bus=usb.0,drive=files,removable=on " in qemu assert _logged(calls, "socat - UNIX-CONNECT:") - assert _logged(calls, "mcopy -s")[0].startswith(f"mcopy -s -i {tmp_path}/state/boot/files.img {answers}/") + assert f"mcopy -s -i {tmp_path}/state/boot/files.img {answers}/autounattend.xml ::/" in _logged(calls, "mcopy -s", 2) running.terminate() assert running.wait(10) == 143 pressed = calls.read_text().count("socat - UNIX-CONNECT:") @@ -444,6 +473,7 @@ def launched(monkeypatch): monkeypatch.setattr(local_sandbox, "machine_for", AsyncMock(return_value=local_disk.Machine("agentenv-vm:0123"))) monkeypatch.setattr(local_sandbox, "_free_host_port", lambda: 20001) monkeypatch.setattr(local_sandbox, "_host_ips", lambda: ("127.0.0.1",)) + monkeypatch.setattr(local_sandbox, "local_grants", lambda: None) monkeypatch.setattr(local_sandbox, "docker", lambda *args, input=None, timeout=None: ran.append( args if input is None else (*args, input, timeout)) or "") monkeypatch.setattr(LocalSandbox, "pull_images", AsyncMock()) @@ -471,6 +501,22 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container +@pytest.mark.asyncio +async def test_with_a_local_stores_grants_a_disks_launcher_forwards_their_port_and_seeds_their_trust(launched, + monkeypatch): + monkeypatch.setattr(local_sandbox, "local_grants", lambda: (41234, Path("/state/tls/trust/ca"))) + + sandbox = await LocalSandboxProvider().create_vm(image="localhost:5000/disk@sha256:" + "a" * 64, + exposed_ports=[18765], env={}, cpu=2, memory=4096) + + (run,) = launched + hosts = [arg for entry in local_sandbox._EXTRA_HOSTS for arg in ("--add-host", entry)] + assert run[-7 - len(hosts):-1] == ( + *hosts, "--mount", "type=bind,src=/state/tls/trust/ca,dst=/agentenv-trust,readonly", + "-e", "VM_HOST_PORTS=41234", "-e", "VM_TRUST=/agentenv-trust") + assert run[-1] == "agentenv-vm:0123" and sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} + + @pytest.mark.asyncio @pytest.mark.parametrize("returncode, refused", [ (UNRESETTABLE, "the computer in container .* was booted by an older agent-env, whose launcher can't reset it in " diff --git a/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py b/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py index 1e1979367..f7a37ff2c 100644 --- a/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py +++ b/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py @@ -122,7 +122,7 @@ def mac(tmp_path, monkeypatch): monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) monkeypatch.setattr(lume, "_free_port", lambda: 48123) monkeypatch.setattr(lume, "_POLL_S", 0.05) - monkeypatch.setattr(lume, "_local_grants", lambda: None) + monkeypatch.setattr(lume, "local_grants", lambda: None) provider = LumeSandboxProvider(lume=str(bin_ / "lume"), ssh=str(bin_ / "ssh"), relay_python=str(bin_ / "relay"), ssh_password="s3cret-pw", boot_timeout_s=10) provider.fake = tmp_path @@ -183,7 +183,7 @@ async def test_a_local_stores_grants_reach_a_vm_at_its_loopback_through_ssh_forw trust.mkdir() (trust / "ca-bundle.pem").write_text("roots and the CA") (trust / "ca.pem").write_text("the CA") - monkeypatch.setattr(lume, "_local_grants", lambda: (54321, trust)) + monkeypatch.setattr(lume, "local_grants", lambda: (54321, trust)) sandbox = await mac.create_vm(image="local:bench", env={"ENVIRONMENT_NAME": "desk"}, exposed_ports=[18765]) @@ -200,7 +200,7 @@ async def test_a_local_stores_grants_reach_a_vm_at_its_loopback_through_ssh_forw assert LumeSandboxProvider.REACHES_HOST_AT == "127.0.0.1" ports = iter([54322, 54323]) - monkeypatch.setattr(lume, "_local_grants", lambda: (next(ports), trust)) + monkeypatch.setattr(lume, "local_grants", lambda: (next(ports), trust)) await asyncio.gather(mac.get_sandbox(sandbox.sandbox_id), mac.get_sandbox(sandbox.sandbox_id)) assert {call["args"][-2] for call in _waited(lambda: _forwards(mac.fake, 3))[1:]} == { @@ -208,7 +208,7 @@ async def test_a_local_stores_grants_reach_a_vm_at_its_loopback_through_ssh_forw forwards = json.loads((folder.parent / "state.json").read_text())["forwards"] assert set(forwards) == {"54321", "54322", "54323"} - monkeypatch.setattr(lume, "_local_grants", lambda: (54324, trust)) + monkeypatch.setattr(lume, "local_grants", lambda: (54324, trust)) monkeypatch.setenv("FAKE_LUME_SLOW_DELETE", "1") _, reconnect = await asyncio.gather(sandbox.terminate(), mac.get_sandbox(sandbox.sandbox_id), return_exceptions=True) From 29d5ffb074f6822a18707d03249420a900a5f11a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 19:04:00 -0800 Subject: [PATCH 2/8] fix(computer): another process reconnecting to a disk's computer forwards its own grant port Each process that hands a computer grants serves them on a port of its own, so a reconnect forwards this process's into the launcher, as lume's VMs get theirs. Co-Authored-By: Claude Opus 5.5 --- .../sandbox_providers/local_docker/provider.py | 16 +++++++++++++++- tst/unit/env/envs/test_disk_recipe.py | 11 ++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 506acd96f..512d576ac 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -907,13 +907,19 @@ async def create_sandbox( ) async def get_sandbox(self, sandbox_id: str) -> LocalSandbox: + """The sandbox ``sandbox_id``; a disk's, with this process's grant server forwarded into its launcher, as + lume's VMs get theirs as another process reconnects: each process that hands it grants serves them on a port + of its own.""" work_dir = LocalSandbox.find_work_dir(sandbox_id) if work_dir is None: raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") port_map = work_dir / _PORT_MAP_FILE ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None # VM mode even for a container it owns: exec runs on this host, so steps must `docker exec` into it. - return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) + sandbox = LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) + if (work_dir / _VM_SIZE_FILE).exists() and (grants := await asyncio.to_thread(local_grants)) is not None: + await asyncio.to_thread(_forward_grants, sandbox.container_name, grants[0]) + return sandbox @classmethod def shares_network_with(cls, sandbox_type: Optional[str]) -> bool: @@ -979,6 +985,14 @@ def _grant_args(port: int, trust: Path) -> list[str]: "-e", f"VM_HOST_PORTS={port}", "-e", f"VM_TRUST={_LAUNCHER_TRUST}"] +def _forward_grants(container: str, port: int) -> None: + """``port`` forwarded in a disk's launcher ``container`` as ``VM_HOST_PORTS`` forwards it, unless it is already, as + a second listener on it exits; a container that's gone has nothing to forward to.""" + with contextlib.suppress(DockerFailed): + docker("exec", "-d", container, "socat", f"TCP-LISTEN:{port},bind=127.0.0.1,fork,reuseaddr", + f"TCP:host.docker.internal:{port}", timeout=_DOCKER_QUERY_SECONDS) + + _DOCKER_QUERY_SECONDS = 30 #: How long a reset may take: longer than the minute the launcher's reset waits for the computer to start again. _RESET_SECONDS = 90 diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index b25b3dd5f..82019252b 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -502,19 +502,24 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r @pytest.mark.asyncio -async def test_with_a_local_stores_grants_a_disks_launcher_forwards_their_port_and_seeds_their_trust(launched, - monkeypatch): +async def test_a_disks_launcher_forwards_this_processs_grant_port_and_seeds_the_trust_and_a_reconnect_forwards_its_own( + launched, monkeypatch): monkeypatch.setattr(local_sandbox, "local_grants", lambda: (41234, Path("/state/tls/trust/ca"))) sandbox = await LocalSandboxProvider().create_vm(image="localhost:5000/disk@sha256:" + "a" * 64, exposed_ports=[18765], env={}, cpu=2, memory=4096) - (run,) = launched + monkeypatch.setattr(local_sandbox, "local_grants", lambda: (52345, Path("/state/tls/trust/ca"))) + await LocalSandboxProvider().get_sandbox(sandbox.sandbox_id) + + run, forward = launched hosts = [arg for entry in local_sandbox._EXTRA_HOSTS for arg in ("--add-host", entry)] assert run[-7 - len(hosts):-1] == ( *hosts, "--mount", "type=bind,src=/state/tls/trust/ca,dst=/agentenv-trust,readonly", "-e", "VM_HOST_PORTS=41234", "-e", "VM_TRUST=/agentenv-trust") assert run[-1] == "agentenv-vm:0123" and sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} + assert forward == ("exec", "-d", sandbox.container_name, "socat", + "TCP-LISTEN:52345,bind=127.0.0.1,fork,reuseaddr", "TCP:host.docker.internal:52345") @pytest.mark.asyncio From c16f82c5b06c43a187bcd3773593c6f3457bd7cd Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 19:48:57 -0800 Subject: [PATCH 3/8] fix(computer): sandboxed apps on a Windows computer read and run their own program C:\agentenv granted only Administrators, SYSTEM and Users, so Chrome's sandbox, whose processes run as the app package identities, couldn't read chrome.exe there: its network service crashed and no page loaded. The lockdown now grants ALL APPLICATION PACKAGES and ALL RESTRICTED APPLICATION PACKAGES read and run, as Program Files does. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/README.md | 3 ++- .../env/envs/computer/utils/windows_disk.py | 13 +++++++++---- tst/unit/env/envs/test_computer_windows.py | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 6a8764286..671428765 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -184,7 +184,8 @@ with 4 vCPUs and 8 GB, on local Docker with KVM and on `modal_vm`; it has comput protocol's Windows defaults, so the probe runs its fixture there as on Linux, with Python one-liners for the commands it would give a shell. On `modal_vm` it runs in your own Modal workspace, under the evaluation's terms or a licence of your own; whether they allow it there is yours to check. Composing leaves `C:\agentenv` the admins', as `/opt/agentenv` is -root's on Linux: every user reads and runs what's there, and the session user writes only the services' logs and each +root's on Linux: every user, and every sandboxed process (Chrome's, as Program Files allows it), reads and runs +what's there, and the session user writes only the services' logs and each app's `.agentenv` folder. An app's own server runs in the session as its services do, on a loopback port it writes with a token to that folder, which only the session user, SYSTEM and Administrators reach; it launches the app in a Job object, which ends whatever the app started. Composing signs in as `agentadmin`, so diff --git a/src/agent_env/env/envs/computer/utils/windows_disk.py b/src/agent_env/env/envs/computer/utils/windows_disk.py index 3e5526397..1721e20b1 100644 --- a/src/agent_env/env/envs/computer/utils/windows_disk.py +++ b/src/agent_env/env/envs/computer/utils/windows_disk.py @@ -75,6 +75,9 @@ #: The groups and the account the folders under ``ROOT`` grant access to, by their well-known SIDs, as icacls takes #: them. ADMINISTRATORS, SYSTEM, USERS = "*S-1-5-32-544", "*S-1-5-18", "*S-1-5-32-545" +#: ALL APPLICATION PACKAGES and ALL RESTRICTED APPLICATION PACKAGES: the identities a sandboxed process runs as, Chrome's +#: and its kind's, which read and run their own program where they're installed, as Program Files lets them. +APP_PACKAGES, RESTRICTED_APP_PACKAGES = "*S-1-15-2-1", "*S-1-15-2-2" #: What every service runs with: Python reading and writing files as UTF-8, as on Linux, not in Windows' code page. UTF8 = {"PYTHONUTF8": "1"} #: What a base's install prints once, as it fetches Microsoft's media. @@ -350,10 +353,12 @@ def write(build: Path, parts: list[Part], *, user: str, build_settings: dict[str def _locked(user: str) -> list[str]: - """The lines that leave ``ROOT`` the admins', as on Linux ``/opt/agentenv`` is root's: every user reads and runs - what's there, none of the access ``C:\\`` gives every user to change it is left, and the session user ``user`` - writes only the services' logs, and each app's own folder, which keeps its own access.""" - return [_granted_alone(ROOT, {ADMINISTRATORS: "F", SYSTEM: "F", USERS: "RX"}), + """The lines that leave ``ROOT`` the admins', as on Linux ``/opt/agentenv`` is root's: every user, and every + sandboxed process, reads and runs what's there, none of the access ``C:\\`` gives every user to change it is left, + and the session user ``user`` writes only the services' logs, and each app's own folder, which keeps its own + access.""" + return [_granted_alone(ROOT, {ADMINISTRATORS: "F", SYSTEM: "F", USERS: "RX", APP_PACKAGES: "RX", + RESTRICTED_APP_PACKAGES: "RX"}), f"Checked {{ icacls '{ROOT / 'logs'}' /grant {_ps(f'{user}:(OI)(CI)M')} | Out-Null }}"] diff --git a/tst/unit/env/envs/test_computer_windows.py b/tst/unit/env/envs/test_computer_windows.py index 95c368afa..b359995a4 100644 --- a/tst/unit/env/envs/test_computer_windows.py +++ b/tst/unit/env/envs/test_computer_windows.py @@ -190,7 +190,7 @@ def test_a_windows_env_composes_a_thin_disk_on_its_base_here_the_driver_installe "driver.py"], "env": {"PYTHONUTF8": "1", "MCP_PORT": "18765"}} assert lines[-3:] == [ "Checked { icacls 'C:\\agentenv' /inheritance:r /grant:r '*S-1-5-32-544:(OI)(CI)F' '*S-1-5-18:(OI)(CI)F' " - "'*S-1-5-32-545:(OI)(CI)RX' | Out-Null }", + "'*S-1-5-32-545:(OI)(CI)RX' '*S-1-15-2-1:(OI)(CI)RX' '*S-1-15-2-2:(OI)(CI)RX' | Out-Null }", "Checked { icacls 'C:\\agentenv\\logs' /grant 'user:(OI)(CI)M' | Out-Null }", ""] assert {"provision.ps1", "context/driver/driver.py", "context/driver/install.ps1"} <= set(files) computer = env.computer_artifact() From d3cf4959855f6503cd58382b5afdea74e4bb8e55 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:10:17 -0800 Subject: [PATCH 4/8] fix(protocol): data/get of a file a Windows app holds stops the app, copies it, and relaunches it Windows lets an app hold a file so that no other process may read it, as Chrome and Electron hold their profiles, and a pause doesn't release it, so data/get, snapshots and gate 3's probe failed with WinError 32 on such an app. Where the paused copy hits a sharing or lock violation, the data plane stops the app, copies, and relaunches it, with data/get as the launch's cause; an app holding nothing keeps the pause. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_data.py | 37 +++++++++++++++---- .../tests/test_computer_app_data.py | 29 ++++++++++++++- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 00856062f..d81354786 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -1,7 +1,9 @@ """An app's data plane as its own server serves it (``AppDataPlane``), by the app's data hooks or, with none, the defaults on the paths it keeps its data in, under the session user's home: a load writes the files it's given under them, an export returns the files there, and a reset puts back the copy its server kept when it first started on this -boot, which is the app as installed, since every boot starts from the computer's image.""" +boot, which is the app as installed, since every boot starts from the computer's image. An export copies the files with +the app paused, or, where the app holds one so that no other process may read it, as Windows lets an app hold a file +and a pause doesn't release, with it stopped and relaunched after.""" from __future__ import annotations import asyncio @@ -17,7 +19,7 @@ from ..agent_env_environment import OP_ADD_DATA, OP_GET_DATA, OP_RESET_DATA from ..transfers import WriteNamespaceGrant -from ..types import METHOD_ADD, METHOD_RESET, FilePart, uploaded_object_path +from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, FilePart, uploaded_object_path from .app_runtime import paused_app from .dataplane import data_files, file_parts, part_bytes @@ -27,15 +29,24 @@ logger = logging.getLogger(__name__) #: The cause a launch after each data call logs. -_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_RESET_DATA: METHOD_RESET} +_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_GET_DATA: METHOD_GET, OP_RESET_DATA: METHOD_RESET} #: Where an app's server keeps its paths as installed, in the app's folder. INITIAL = Path(".agentenv/initial") +#: Windows' errors for a file another process holds so that no other may read it: a sharing violation, and a lock on +#: the part read. +_HELD = (32, 33) + + +class HeldByApp(OSError): + """A file under the app's paths that the running app holds so that no other process may read it, which stopping + the app releases and pausing it doesn't.""" class AppDataPlane: """An app's data plane, served around its contract: one data call or launch at a time; a load and a reset stop the running app first and relaunch it after, but for those the app serves live, and a load that fails is followed by a - reset, when the app says so. A call fails once it runs + reset, when the app says so; an export the app holds a file of (``HeldByApp``) stops it and relaunches it too. A + call fails once it runs past ``timeout_s`` and is cancelled: what awaits stops at once, but a thread can't be stopped, so a sync hook's or a default's thread is waited for, as are the reset and relaunch after it and the next call.""" @@ -90,7 +101,11 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. running = op != OP_GET_DATA and op not in self.live and await self.runtime.stop_app() ran = op in self.live and self.runtime.running() try: - result = await self._within(op, served, args, answer) + try: + result = await self._within(op, served, args, answer) + except HeldByApp: + running = await self.runtime.stop_app() + result = await self._within(op, served, args, answer) except Exception: if op == OP_ADD_DATA and self.resets_failed_add(): if op in self.live: @@ -185,11 +200,17 @@ async def add(parts: list, paths: list[str]) -> None: async def get(paths: list[str], write_namespace: Optional[WriteNamespaceGrant]) -> list: """A part for each file under ``paths``, copied with the app this server launched paused (``paused_app``), so they - are of one moment, and sent from the copy once it runs again; an app's own data/get hook calls it so too.""" + are of one moment, and sent from the copy once it runs again; an app's own data/get hook calls it so too. A file + the paused app holds raises ``HeldByApp``, for its data plane to copy them with it stopped.""" INITIAL.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=INITIAL.parent) as staging: - with paused_app(): - await in_thread(_stage, Path.home(), paths, Path(staging)) + try: + with paused_app(): + await in_thread(_stage, Path.home(), paths, Path(staging)) + except OSError as e: + if getattr(e, "winerror", None) in _HELD: + raise HeldByApp(f"the running app holds {e.filename or 'a file'} so that nothing else may read it") from e + raise return await file_parts(Path(staging), paths, write_namespace) diff --git a/packages/agentenv-protocol/tests/test_computer_app_data.py b/packages/agentenv-protocol/tests/test_computer_app_data.py index 00b601504..d10267922 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_data.py +++ b/packages/agentenv-protocol/tests/test_computer_app_data.py @@ -2,7 +2,8 @@ the defaults: a load writes files under them, through no link out of them, an export returns them, and a reset puts back the copy its server kept before it served; its card declares a file tree. Hooks override paths, and read the parts they get with part_bytes. Around a load and a reset the app is stopped and relaunched, a failed load is followed by a reset, -and data calls take turns; an app that loads live is left running through a load, and a failed one is reset with it +and data calls take turns; an export of a file the paused app holds stops it and relaunches it after; an app that +loads live is left running through a load, and a failed one is reset with it stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past data_timeout_s fails then and is cancelled, but a sync hook runs in a thread, which its reset and relaunch, and the next call, wait for. The app serves its skill file.""" @@ -210,6 +211,32 @@ async def test_a_load_and_a_reset_stop_the_running_app_and_relaunch_it_and_an_ex ("stop", 4244), ("launch", 4248)] +@pytest.mark.asyncio +async def test_an_export_of_a_file_the_paused_app_holds_stops_the_app_copies_it_and_relaunches_it(home, desktop, + monkeypatch): + """As Windows lets an app hold a file that no other process may read, which a pause doesn't release.""" + (home / "Mail").mkdir() + (home / "Mail/inbox.txt").write_text("held") + mail = PathsMail() + mail._computer_app_data_plane().keep_initial() + await mail._computer_app.launch_app() + stage = app_data._stage + + def held_while_running(*args): + if desktop[-1][0] == "pause": + held = PermissionError(13, "The process cannot access the file", str(home / "Mail/inbox.txt")) + held.winerror = 32 + raise held + stage(*args) + + monkeypatch.setattr(app_data, "_stage", held_while_running) + async with _http(mail) as http: + exported = _files(await _rpc(http, "data/get", {})) + + assert [event[0] for event in desktop] == ["launch", "pause", "continue", "stop", "launch"] + assert desktop[3] == ("stop", 4242) and exported["Mail/inbox.txt"] == b"held" + + @pytest.mark.asyncio async def test_an_export_copies_the_paths_with_the_app_paused_and_sends_the_copy_once_it_runs_again(home, desktop, monkeypatch): From 04cbe35c75c6d751cb7351e894ebb4ddd553b818 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:11:36 -0800 Subject: [PATCH 5/8] test(protocol): the fake desktop resumes a paused app when the block raises, as both sessions do Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/tests/test_computer_app_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/agentenv-protocol/tests/test_computer_app_data.py b/packages/agentenv-protocol/tests/test_computer_app_data.py index d10267922..e20d00620 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_data.py +++ b/packages/agentenv-protocol/tests/test_computer_app_data.py @@ -107,8 +107,10 @@ async def stop(pid, timeout_s): @contextlib.contextmanager def paused(pid): events.append(("pause", pid)) - yield - events.append(("continue", pid)) + try: + yield + finally: + events.append(("continue", pid)) monkeypatch.setattr(session, "launch", launch) monkeypatch.setattr(session, "stop", stop) From e0889d0dd8dc11fb6f56aaa27d0969ef3e716791 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:28:26 -0800 Subject: [PATCH 6/8] refactor(computer): one forward in the launcher, the guests' trust pinned to a container's, and named checks The launcher's forward is one function, behind launch forward PORT, which a reconnect runs rather than spelling socat out again, guarded for a launcher older than it; a test pins both guests' trust variables to trust_env's; the provider names its disk check, logs a forward it couldn't make and finds the granting store in one place; the data plane's held-file retry is a helper of its own. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_data.py | 22 +++++++--- .../computers/disk/linux/launcher/launch | 16 +++++-- .../sandbox_providers/disk_launcher.py | 7 +++ .../local_docker/provider.py | 41 ++++++++++------- tst/unit/env/envs/test_disk_recipe.py | 44 ++++++++++++++++--- 5 files changed, 98 insertions(+), 32 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index d81354786..022406d1b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -101,11 +101,7 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. running = op != OP_GET_DATA and op not in self.live and await self.runtime.stop_app() ran = op in self.live and self.runtime.running() try: - try: - result = await self._within(op, served, args, answer) - except HeldByApp: - running = await self.runtime.stop_app() - result = await self._within(op, served, args, answer) + result = await self._within_held(op, served, args, answer) except Exception: if op == OP_ADD_DATA and self.resets_failed_add(): if op in self.live: @@ -125,6 +121,22 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. else: answer.set_result(result) + async def _within_held(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any: + """``served``'s result within ``timeout_s``; where the app holds a file of its paths (``HeldByApp``), again with + the app stopped, which is relaunched after, as a load's is.""" + try: + return await self._within(op, served, args, answer) + except HeldByApp: + stopped = await self.runtime.stop_app() + try: + result = await self._within(op, served, args, answer) + finally: + if stopped: + await self.runtime.launch_app(_CAUSES[op]) + if stopped and self.relaunched is not None: + await self.relaunched(op) + return result + async def _undo(self, answer: asyncio.Future) -> None: """Reset the app's data after a load that failed.""" if (reset := self._served(OP_RESET_DATA)) is None: diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index d2ac70abb..9780e3ace 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -29,7 +29,16 @@ # VM_FILES for a windows guest, a folder whose files it finds on a removable FAT drive # launch reset: boot the running computer afresh on a new overlay, its environment the VM_ENV_= lines on # stdin, returning once its new QEMU has started; its ports stay up throughout. +# launch forward PORT: serve PORT of this machine's host to the running computer as VM_HOST_PORTS does, for another +# process's grant server, until stopped. set -euo pipefail + +# forward PORT: PORT of this machine's host served on 127.0.0.1 here, where the guest reaches it at 10.0.2.2. +forward() { exec socat "TCP-LISTEN:$1,bind=127.0.0.1,fork,reuseaddr" "TCP:host.docker.internal:$1"; } + +if [ "${1:-}" = forward ]; then + forward "$2" +fi state=${VM_STATE:-/var/lib/agentenv-vm} # A reset and the boot loop meet here: the reset writes next.env, creates reset and stops the QEMU qemu.pid names; the # loop, finding reset, takes next.env's environment, removes reset and boots afresh; the reset returns once qemu.pid @@ -99,9 +108,10 @@ case "$guest/$arch" in echo "launch: no architecture $arch" >&2 exit 2 ;; esac -forwards="" fronts=() +forwards="" fronts=() taken="" for port in ${VM_PORTS:-}; do behind=$((port + 10000)) + taken="$taken $port $behind" if [ "$behind" -gt 65535 ] || [[ " $VM_PORTS " == *" $behind "* ]]; then echo "launch: port $port can't be forwarded: QEMU's forward behind its proxy would take port $behind, which" \ "is $([ "$behind" -gt 65535 ] && echo "above 65535" || echo "another of VM_PORTS")" >&2 @@ -112,11 +122,11 @@ for port in ${VM_PORTS:-}; do fronts+=("$!") done for port in ${VM_HOST_PORTS:-}; do - if [[ " ${VM_PORTS:-} " == *" $port "* || " ${VM_PORTS:-} " == *" $((port - 10000)) "* ]]; then + if [[ "$taken " == *" $port "* ]]; then echo "launch: host port $port can't be forwarded: a guest port's proxy or its forward here takes it" >&2 exit 2 fi - socat "TCP-LISTEN:$port,bind=127.0.0.1,fork,reuseaddr" "TCP:host.docker.internal:$port" & + forward "$port" & fronts+=("$!") done trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT diff --git a/src/agent_env/providers/sandbox_providers/disk_launcher.py b/src/agent_env/providers/sandbox_providers/disk_launcher.py index d0812b707..ee20a2b60 100644 --- a/src/agent_env/providers/sandbox_providers/disk_launcher.py +++ b/src/agent_env/providers/sandbox_providers/disk_launcher.py @@ -18,6 +18,13 @@ RESET = ("sh", "-c", f"[ -d {CONTROL} ] || exit {UNRESETTABLE}; exec {LAUNCH} reset") +def forward(port: int) -> tuple[str, ...]: + """The command that serves ``port`` of this machine's host to the running computer, as ``VM_HOST_PORTS`` does: the + launcher's ``launch forward``, which a launcher older than it would take for a second boot, so it isn't run there, + whose computer has no use for it.""" + return ("sh", "-c", f"grep -q '^forward()' {LAUNCH} || exit 0; exec {LAUNCH} forward {port}") + + def launcher_variables(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> dict[str, str]: """The environment that has the launcher size its VM, forward ``ports``, refuse to boot without KVM and hand the computer ``env``, each value base64'd, so one of several lines reaches the computer whole.""" diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 512d576ac..658c34a2e 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -39,7 +39,7 @@ from agent_env.attribution import Attribution from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here -from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, reset_input, unresettable +from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, forward, reset_input, unresettable from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_for from agent_env.providers.sandbox_providers.local_docker.relay import relay_image @@ -599,22 +599,22 @@ def _remove_staged(sandbox_id: str) -> None: def local_grant_trust() -> Path | None: """The trust files a container on this host needs to use the configured object store's grants: the local transfer CA's, when the store is a local one that hands out grants; None otherwise.""" - store = config.get_config().get_object_store() - if isinstance(store, LocalRunObjectStore): - store = store.local - if isinstance(store, LocalFilesystemObjectStore) and store.supports_transfer_grants: - return local_ca().trust_dir - return None + return local_ca().trust_dir if _granting_store() is not None else None def local_grants() -> Optional[tuple[int, Path]]: """The port of the configured object store's grant server, started, and the local transfer CA's trust files, when the store is a local one handing out grants; None otherwise.""" - trust = local_grant_trust() - if trust is None: - return None + store = _granting_store() + return (store.grant_port(), local_ca().trust_dir) if store is not None else None + + +def _granting_store() -> Optional[LocalFilesystemObjectStore]: + """The configured object store, or the local one a run routes to, when it's a local one that hands out grants.""" store = config.get_config().get_object_store() - return (store.local if isinstance(store, LocalRunObjectStore) else store).grant_port(), trust + if isinstance(store, LocalRunObjectStore): + store = store.local + return store if isinstance(store, LocalFilesystemObjectStore) and store.supports_transfer_grants else None async def start_trusting(sandbox: VmSandbox, container: str, trust_dir: Path) -> None: @@ -917,7 +917,7 @@ async def get_sandbox(self, sandbox_id: str) -> LocalSandbox: ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None # VM mode even for a container it owns: exec runs on this host, so steps must `docker exec` into it. sandbox = LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) - if (work_dir / _VM_SIZE_FILE).exists() and (grants := await asyncio.to_thread(local_grants)) is not None: + if _is_disk(work_dir) and (grants := await asyncio.to_thread(local_grants)) is not None: await asyncio.to_thread(_forward_grants, sandbox.container_name, grants[0]) return sandbox @@ -985,12 +985,19 @@ def _grant_args(port: int, trust: Path) -> list[str]: "-e", f"VM_HOST_PORTS={port}", "-e", f"VM_TRUST={_LAUNCHER_TRUST}"] +def _is_disk(work_dir: Path) -> bool: + """Whether the sandbox in ``work_dir`` is a disk's computer: ``create_vm`` records the size of the VM it boots.""" + return (work_dir / _VM_SIZE_FILE).exists() + + def _forward_grants(container: str, port: int) -> None: - """``port`` forwarded in a disk's launcher ``container`` as ``VM_HOST_PORTS`` forwards it, unless it is already, as - a second listener on it exits; a container that's gone has nothing to forward to.""" - with contextlib.suppress(DockerFailed): - docker("exec", "-d", container, "socat", f"TCP-LISTEN:{port},bind=127.0.0.1,fork,reuseaddr", - f"TCP:host.docker.internal:{port}", timeout=_DOCKER_QUERY_SECONDS) + """``port`` forwarded in a disk's launcher ``container`` (``launch forward``), unless it is already, as a second + listener on it exits; one that can't be is logged, the computer then reaching no grant of this process's.""" + try: + docker("exec", "-d", container, *forward(port), timeout=_DOCKER_QUERY_SECONDS) + except DockerFailed as e: + logger.warning("Couldn't forward grant port %s into %s, so grants from this process won't reach its computer: " + "%s", port, container, e) _DOCKER_QUERY_SECONDS = 30 diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index 82019252b..cc9457aff 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -7,6 +7,7 @@ import asyncio import base64 import hashlib +import importlib.util import json import os import re @@ -26,7 +27,10 @@ from agent_env.providers.sandbox_providers.local_docker import disk as local_disk from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed -from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider +from agent_env.env.envs.computer.utils import windows_disk +from agent_env.providers.sandbox_providers import disk_launcher +from agent_env.providers.sandbox_providers.local_docker.provider import (LOCAL_TRUST_DIR, LocalSandbox, + LocalSandboxProvider, trust_env) from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL TRICKY = "a value with spaces, = signs, 'quotes', $dollars and\na second line" @@ -251,6 +255,31 @@ def test_the_launcher_refuses_a_host_port_a_guest_ports_proxy_or_forward_takes(l refused.stderr.read()) +def test_launch_forward_serves_a_host_port_to_the_running_computer_as_the_boot_does(launcher): + launch, calls = launcher + + forwarding = launch("forward", "52345") + + assert _logged(calls, "socat") == [ + "socat TCP-LISTEN:52345,bind=127.0.0.1,fork,reuseaddr TCP:host.docker.internal:52345"] + forwarding.terminate() + + +def test_a_disks_guests_point_tls_clients_at_the_trust_files_a_containers_variables_name(tmp_path): + """Neither guest can import the framework, so each keeps its own copy of trust_env's variables and files.""" + spec = importlib.util.spec_from_file_location("agentenv_session", windows_disk.RECIPE / "agentenv-session.py") + session = importlib.util.module_from_spec(spec) + spec.loader.exec_module(session) + init = (DISK / "boot" / "init").read_text() + expected = trust_env(LOCAL_TRUST_DIR) + + assert all(f"{name}={value}" in init for name, value in expected.items()) + assert all(f"/seed/trust/{PurePosixPath(value).name}" in init for value in expected.values()) + (tmp_path / "ca-bundle.pem").write_text("") + assert {name: Path(value).name for name, value in session.trust(tmp_path).items()} == { + name: PurePosixPath(value).name for name, value in expected.items()} + + @pytest.mark.parametrize("ports, why", [("55536", "above 65535"), ("6080 16080", "another of VM_PORTS")]) def test_the_launcher_refuses_a_port_whose_forward_behind_the_proxy_has_no_port_of_its_own(launcher, ports, why): launch, calls = launcher @@ -514,12 +543,13 @@ async def test_a_disks_launcher_forwards_this_processs_grant_port_and_seeds_the_ run, forward = launched hosts = [arg for entry in local_sandbox._EXTRA_HOSTS for arg in ("--add-host", entry)] - assert run[-7 - len(hosts):-1] == ( - *hosts, "--mount", "type=bind,src=/state/tls/trust/ca,dst=/agentenv-trust,readonly", - "-e", "VM_HOST_PORTS=41234", "-e", "VM_TRUST=/agentenv-trust") - assert run[-1] == "agentenv-vm:0123" and sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} - assert forward == ("exec", "-d", sandbox.container_name, "socat", - "TCP-LISTEN:52345,bind=127.0.0.1,fork,reuseaddr", "TCP:host.docker.internal:52345") + assert run == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {}, 2, 4096), *hosts, + "--mount", "type=bind,src=/state/tls/trust/ca,dst=/agentenv-trust,readonly", + "-e", "VM_HOST_PORTS=41234", "-e", "VM_TRUST=/agentenv-trust", "agentenv-vm:0123") + assert forward == ("exec", "-d", sandbox.container_name, *disk_launcher.forward(52345)) + assert disk_launcher.forward(52345)[-1] == (f"grep -q '^forward()' {LAUNCH} || exit 0; " + f"exec {LAUNCH} forward 52345") @pytest.mark.asyncio From 50952a36bb01aa50fb90141413580d36af11bb0a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 21:03:09 -0800 Subject: [PATCH 7/8] fix(protocol): an export retried with the app stopped gets only what's left of data_timeout_s The retry after a held file had a second full window, so a large export that met a held file late could take nearly twice the app's data_timeout_s. Both attempts now share one deadline. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_data.py | 16 +++++---- .../tests/test_computer_app_data.py | 34 ++++++++++++++++--- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 022406d1b..bd81cf97b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -123,13 +123,15 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. async def _within_held(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any: """``served``'s result within ``timeout_s``; where the app holds a file of its paths (``HeldByApp``), again with - the app stopped, which is relaunched after, as a load's is.""" + the app stopped, in what's left of it, and the app relaunched after, as a load's is.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + self.timeout_s try: return await self._within(op, served, args, answer) except HeldByApp: stopped = await self.runtime.stop_app() try: - result = await self._within(op, served, args, answer) + result = await self._within(op, served, args, answer, deadline - loop.time()) finally: if stopped: await self.runtime.launch_app(_CAUSES[op]) @@ -167,12 +169,14 @@ def _served(self, op: str) -> Optional[Callable]: OP_GET_DATA: functools.partial(get, self.paths), OP_RESET_DATA: functools.partial(reset, self.paths)}[op] - async def _within(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any: - """``served``'s result within ``timeout_s``; past it, ``answer`` fails at once and the call is cancelled, and - this raises the same once it has ended: at once for what awaits, after any thread it runs.""" + async def _within(self, op: str, served: Callable, args: tuple, answer: asyncio.Future, + left_s: Optional[float] = None) -> Any: + """``served``'s result within ``left_s``, ``timeout_s`` by default; past it, ``answer`` fails at once and the + call is cancelled, and this raises the same once it has ended: at once for what awaits, after any thread it + runs.""" work = asyncio.ensure_future(_called(served, *args)) try: - return await asyncio.wait_for(asyncio.shield(work), self.timeout_s) + return await asyncio.wait_for(asyncio.shield(work), max(0.0, self.timeout_s if left_s is None else left_s)) except TimeoutError: late = TimeoutError(f"{op} ran past the app's data_timeout_s, {self.timeout_s:g} s") if not answer.done(): diff --git a/packages/agentenv-protocol/tests/test_computer_app_data.py b/packages/agentenv-protocol/tests/test_computer_app_data.py index e20d00620..06b6e2ba6 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_data.py +++ b/packages/agentenv-protocol/tests/test_computer_app_data.py @@ -2,9 +2,9 @@ the defaults: a load writes files under them, through no link out of them, an export returns them, and a reset puts back the copy its server kept before it served; its card declares a file tree. Hooks override paths, and read the parts they get with part_bytes. Around a load and a reset the app is stopped and relaunched, a failed load is followed by a reset, -and data calls take turns; an export of a file the paused app holds stops it and relaunches it after; an app that -loads live is left running through a load, and a failed one is reset with it -stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past +and data calls take turns; an export of a file the paused app holds stops it, copies it in what's left of its +timeout, and relaunches it after; an app that loads live is left running through a load, and a failed one is reset +with it stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past data_timeout_s fails then and is cancelled, but a sync hook runs in a thread, which its reset and relaunch, and the next call, wait for. The app serves its skill file.""" from __future__ import annotations @@ -21,7 +21,7 @@ import httpx # noqa: E402 -from agentenv_protocol import add_data, client, reset_data # noqa: E402 +from agentenv_protocol import add_data, client, get_data, reset_data # noqa: E402 from agentenv_protocol.agent_env_environment import OP_ADD_DATA # noqa: E402 from agentenv_protocol.computer import ComputerAppEnvironment, Launch, linux, part_bytes # noqa: E402 from agentenv_protocol.computer import app, app_data, app_runtime, session # noqa: E402 @@ -443,6 +443,32 @@ def wipe(self) -> None: ("launch", 4248), ("stop", 4248), ("load", 1), ("loaded", 1), ("launch", 4252)] +@pytest.mark.asyncio +async def test_an_export_retried_with_the_app_stopped_gets_only_what_is_left_of_its_timeout(home, desktop): + @linux(launch=Launch(command=["held"])) + class Held(ComputerAppEnvironment): + data_timeout_s = 1.0 + + @get_data + async def export(self) -> list: + if desktop[-1][0] == "launch": + await asyncio.sleep(0.8) + raise app_data.HeldByApp("the running app holds a file") + await asyncio.Event().wait() + + held = Held() + await held._computer_app.launch_app() + started = time.monotonic() + async with _http(held) as http: + late = await http.post(RPC_PATH, json=_call("data/get", {})) + answered = time.monotonic() - started + await asyncio.gather(*held._computer_app_data_plane().changing) + + assert late.json()["error"]["message"] == "get_data ran past the app's data_timeout_s, 1 s" + assert answered < 1.5 + assert desktop == [("launch", 4242), ("stop", 4242), ("launch", 4244)] + + @pytest.mark.asyncio async def test_it_serves_its_skill_file_and_no_skill_without_one(home, tmp_path): (tmp_path / "folder" / "SKILL.md").write_text("---\ndescription: Reading and sending mail.\n---\n\n# Mail\nUse it.\n") From eb3af1ef1c02b00ca5cffd26556d7445fb0967db Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 21:19:24 -0800 Subject: [PATCH 8/8] fix(local): a teardown terminates a disk's computer without forwarding a grant port into it terminate_sandbox reconnected through get_sandbox, which now forwards this process's grant port into a disk's launcher, so every local disk run ended by forwarding into a computer it was removing, and warned when the launcher was already gone. The local provider terminates by id without it. Co-Authored-By: Claude Opus 5.5 --- .../local_docker/provider.py | 25 +++++++++++++------ tst/unit/env/envs/test_disk_recipe.py | 8 +++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 658c34a2e..7a3c32c0f 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -910,17 +910,15 @@ async def get_sandbox(self, sandbox_id: str) -> LocalSandbox: """The sandbox ``sandbox_id``; a disk's, with this process's grant server forwarded into its launcher, as lume's VMs get theirs as another process reconnects: each process that hands it grants serves them on a port of its own.""" - work_dir = LocalSandbox.find_work_dir(sandbox_id) - if work_dir is None: - raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") - port_map = work_dir / _PORT_MAP_FILE - ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None - # VM mode even for a container it owns: exec runs on this host, so steps must `docker exec` into it. - sandbox = LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) - if _is_disk(work_dir) and (grants := await asyncio.to_thread(local_grants)) is not None: + sandbox = _found(sandbox_id) + if _is_disk(sandbox.work_dir) and (grants := await asyncio.to_thread(local_grants)) is not None: await asyncio.to_thread(_forward_grants, sandbox.container_name, grants[0]) return sandbox + async def terminate_sandbox(self, sandbox_id: str) -> None: + """Terminate the sandbox ``sandbox_id``, forwarding no grant port into a disk's launcher it is removing.""" + await _found(sandbox_id).terminate() + @classmethod def shares_network_with(cls, sandbox_type: Optional[str]) -> bool: """Local sandboxes never share a Docker network. The agent runs via ``docker run`` on the @@ -985,6 +983,17 @@ def _grant_args(port: int, trust: Path) -> list[str]: "-e", f"VM_HOST_PORTS={port}", "-e", f"VM_TRUST={_LAUNCHER_TRUST}"] +def _found(sandbox_id: str) -> LocalSandbox: + """The sandbox ``sandbox_id`` from its work directory, in VM mode even for a container it owns: exec runs on this + host, so steps must ``docker exec`` into it.""" + work_dir = LocalSandbox.find_work_dir(sandbox_id) + if work_dir is None: + raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") + port_map = work_dir / _PORT_MAP_FILE + ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None + return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) + + def _is_disk(work_dir: Path) -> bool: """Whether the sandbox in ``work_dir`` is a disk's computer: ``create_vm`` records the size of the VM it boots.""" return (work_dir / _VM_SIZE_FILE).exists() diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index cc9457aff..744040ffa 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -531,7 +531,7 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r @pytest.mark.asyncio -async def test_a_disks_launcher_forwards_this_processs_grant_port_and_seeds_the_trust_and_a_reconnect_forwards_its_own( +async def test_a_disk_gets_this_processs_grant_port_and_the_trust_and_a_reconnect_its_own_but_not_a_teardown( launched, monkeypatch): monkeypatch.setattr(local_sandbox, "local_grants", lambda: (41234, Path("/state/tls/trust/ca"))) @@ -551,6 +551,12 @@ async def test_a_disks_launcher_forwards_this_processs_grant_port_and_seeds_the_ assert disk_launcher.forward(52345)[-1] == (f"grep -q '^forward()' {LAUNCH} || exit 0; " f"exec {LAUNCH} forward 52345") + launched.clear() + monkeypatch.setattr(LocalSandbox, "terminate", AsyncMock()) + await LocalSandboxProvider().terminate_sandbox(sandbox.sandbox_id) + + assert launched == [] and LocalSandbox.terminate.await_count == 1 + @pytest.mark.asyncio @pytest.mark.parametrize("returncode, refused", [