Skip to content

Commit d5a7cb4

Browse files
perf(computer): boot disks with their image mounted into the launcher, no copy for each launcher (#536)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 1df8f31 commit d5a7cb4

6 files changed

Lines changed: 159 additions & 32 deletions

File tree

‎src/agent_env/computers/README.md‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,18 @@ partition kept, so nothing is emulated or privileged. A disk boots where `/dev/k
3434
KVM probes it. A deploy boots it with `env_params = { form = "disk" }`. The launcher serves each of the computer's ports
3535
through socat, whose listen backlog takes a burst of connections, in front of QEMU's own forward, whose backlog is 1, and
3636
a reset boots the disk afresh on a new overlay in the same launcher, at the same URLs; a container an agent-env from
37-
before reset in place started is refused a reset, to be redeployed. It boots on local Docker where
38-
Docker can pass it `/dev/kvm`, and on `modal_vm`, amd64 only, whose sandbox runs the launcher as its image under nested
39-
virtualization: the provider exports the disk from the image store with this machine's Docker, compressed, once for each
40-
composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal uploads once for each
41-
content and the launcher expands onto the sandbox's disk; a thin disk, a Windows one, carries its base too, exported
42-
once for each base and expanded beside it there, so another computer on that base adds its layer alone. Such a sandbox
43-
is ended once it has been idle, with no connection open, for the provider's `idle_timeout`, 30 minutes unless its config
44-
sets another, as a running QEMU isn't activity. A gate 3 probe that ends with the connection to the driver dropped,
45-
refused or timed out runs again on a fresh boot, up to three in all, and the record keeps each attempt; one with a
46-
failed check is recorded as it is.
37+
before reset in place started is refused a reset, to be redeployed. It boots on local Docker where Docker can pass it
38+
`/dev/kvm`, the disk image's files mounted read-only into the launcher where Docker mounts images (28 and later, on the
39+
containerd image store), so no copy of a disk, nor of a Windows disk's base, is made for each launcher, else copied into
40+
an image of the launcher it keeps for the next boot; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as
41+
its image under nested virtualization: the provider exports the disk from the image store with this machine's Docker,
42+
compressed, once for each composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal
43+
uploads once for each content and the launcher expands onto the sandbox's disk; a thin disk, a Windows one, carries its
44+
base too, exported once for each base and expanded beside it there, so another computer on that base adds its layer
45+
alone. Such a sandbox is ended once it has been idle, with no connection open, for the provider's `idle_timeout`, 30
46+
minutes unless its config sets another, as a running QEMU isn't activity. A gate 3 probe that ends with the connection
47+
to the driver dropped, refused or timed out runs again on a fresh boot, up to three in all, and the record keeps each
48+
attempt; one with a failed check is recorded as it is.
4749

4850
A put, `env build` and the release compose every form; a run composes only the forms its sandbox's deploys boot: on each
4951
link of a chain, the first form it boots natively, as a deploy picks, so local Docker composes containers, and a sandbox

‎src/agent_env/env/envs/computer/utils/windows_disk.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
from agent_env.env.envs.computer.utils.install import WindowsInstall
3535
from agent_env.providers.sandbox_providers.host_images import present
3636
from agent_env.providers.sandbox_providers.local_docker.cli import docker
37-
from agent_env.providers.sandbox_providers.local_docker.disk import KVM, launcher, machine_image
37+
from agent_env.providers.sandbox_providers.local_docker.disk import KVM, launcher, machine_for
3838
from agent_env.store.image_store.image_contents import labels as image_labels
3939
from agent_env.store.image_store.image_store import ImageStore
4040
from agent_env.store.image_store.registry_api import pin_digest, pin_if_served
@@ -284,8 +284,8 @@ def compose(base_: Base, build: Path, key: str, labels: dict[str, str], say: Cal
284284
out.mkdir()
285285
name = f"agentenv-windows-compose-{_short(key)}"
286286
say(f"booting its base in {name}")
287-
machine = asyncio.run(machine_image(base_.image))
288-
container = _launched(name, {"VM_KEEP": "/out/disk.img"}, ["-v", f"{out}:/out"], machine)
287+
machine = asyncio.run(machine_for(base_.image))
288+
container = _launched(name, {"VM_KEEP": "/out/disk.img"}, ["-v", f"{out}:/out", *machine.args], machine.image)
289289
try:
290290
port, deadline = _ssh_port(container), time.monotonic() + COMPOSE_TIMEOUT_S
291291
while ssh(port, base_.keys, "echo ready", check=False, timeout=60).returncode != 0:
@@ -303,7 +303,7 @@ def compose(base_: Base, build: Path, key: str, labels: dict[str, str], say: Cal
303303
_shut_down(container, port, base_.keys)
304304
finally:
305305
subprocess.run(["docker", "rm", "-f", container], capture_output=True)
306-
_converted(out, machine, backing=f"/{DISK_BASE_FILE}")
306+
_converted(out, machine.image, backing=f"/{DISK_BASE_FILE}", args=machine.args)
307307
subprocess.run(["docker", "run", "--rm", "-v", f"{out}:/out", "--entrypoint", "qemu-img", launcher(), "rebase",
308308
"-u", "-b", PureWindowsPath(DISK_BASE_FILE).name, "-F", "qcow2", "/out/disk.img"],
309309
check=True, capture_output=True)
@@ -419,12 +419,12 @@ def _compacted(port: int, keys: Path, say: Callable[[str], None]) -> None:
419419
say(f"trimming exited {done.returncode}, so its image keeps what it held: {(done.stderr + done.stdout)[-300:]}")
420420

421421

422-
def _converted(folder: Path, image: str, backing: str | None = None) -> None:
423-
"""``folder``'s ``disk.img`` rewritten by ``qemu-img convert`` in ``image``, which holds its backing file, keeping
424-
only the clusters it uses: a base's whole, a layer's over ``backing``, the backing file ``image`` holds."""
422+
def _converted(folder: Path, image: str, backing: str | None = None, args: tuple[str, ...] = ()) -> None:
423+
"""``folder``'s ``disk.img`` rewritten by ``qemu-img convert`` in ``image`` run with ``args``, which give it its
424+
backing file, keeping only the clusters it uses: a base's whole, a layer's over ``backing``."""
425425
over = ["-B", backing, "-F", "qcow2"] if backing else []
426-
subprocess.run(["docker", "run", "--rm", "-v", f"{folder}:/out", "--entrypoint", "qemu-img", image, "convert", "-O",
427-
"qcow2", *over, "/out/disk.img", "/out/compact.img"], check=True, capture_output=True)
426+
subprocess.run(["docker", "run", "--rm", "-v", f"{folder}:/out", *args, "--entrypoint", "qemu-img", image, "convert",
427+
"-O", "qcow2", *over, "/out/disk.img", "/out/compact.img"], check=True, capture_output=True)
428428
(folder / "compact.img").replace(folder / "disk.img")
429429

430430

‎src/agent_env/providers/sandbox_providers/local_docker/disk.py‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
"""Booting a computer's disk form on this machine's Docker: the QEMU launcher (the published image for the sources
2-
installed, else built here from its folder) with the disk copied in, as an image this machine keeps for the next boot,
3-
run with /dev/kvm; and a disk exported from the image store by this machine's Docker, for a provider elsewhere to
4-
carry."""
2+
installed, else built here from its folder) run with /dev/kvm, the disk image's files mounted into it where this
3+
machine's Docker mounts images, else copied into an image of it this machine keeps for the next boot; and a disk
4+
exported from the image store by this machine's Docker, for a provider elsewhere to carry."""
55

66
import asyncio
7+
import functools
78
import hashlib
89
import os
910
import shutil
@@ -24,6 +25,8 @@
2425
from agent_env.utils.docker.build import build_image
2526

2627
KVM = "/dev/kvm"
28+
#: How long asking this machine's Docker whether it mounts images may take, before a boot copies the disk instead.
29+
_DOCKER_INFO_SECONDS = 30
2730
#: What ``docker cp`` says of a path its container lacks: Docker 23 and later, then earlier.
2831
_NO_SUCH_PATH = ("Could not find the file", "No such container:path")
2932

@@ -37,6 +40,49 @@ def kvm_platforms() -> frozenset[str]:
3740
return frozenset({here, "windows/amd64"} if here == "linux/amd64" else {here})
3841

3942

43+
@dataclass(frozen=True)
44+
class Machine:
45+
"""How the launcher boots a disk on this machine: ``image`` run with ``args``. Where Docker mounts images (28 and
46+
later, on the containerd image store), the launcher itself with the disk image's folder of files mounted read-only
47+
where the launcher finds them, so no copy of a disk, nor of its base, is made for each launcher; else a machine
48+
image of the launcher with the disk copied in."""
49+
50+
image: str
51+
args: tuple[str, ...] = ()
52+
53+
54+
async def machine_for(disk: str) -> Machine:
55+
if await asyncio.to_thread(mounts_images):
56+
folder = PurePosixPath(DISK_FILE).parent
57+
return Machine(await asyncio.to_thread(launcher),
58+
("--mount", f"type=image,source={disk},target=/{folder},image-subpath={folder}"))
59+
return Machine(await machine_image(disk))
60+
61+
62+
def mounts_images() -> bool:
63+
"""Whether this machine's Docker mounts an image's files into a container (``--mount type=image``, with its
64+
``image-subpath``): its client, which parses the option, and its server both 28 or later, on the containerd image
65+
store. A Docker that fails to say, or doesn't within ``_DOCKER_INFO_SECONDS``, gets the copy this time and is asked
66+
again next time; an answer is kept for the process."""
67+
try:
68+
return _docker_mounts_images()
69+
except (DockerFailed, subprocess.TimeoutExpired):
70+
return False
71+
72+
73+
@functools.cache
74+
def _docker_mounts_images() -> bool:
75+
client, server, status = docker("info", "--format",
76+
"{{.ClientInfo.Version}} {{.ServerVersion}} {{json .DriverStatus}}",
77+
timeout=_DOCKER_INFO_SECONDS).split(" ", 2)
78+
return _major(client) >= 28 and _major(server) >= 28 and "io.containerd.snapshotter.v1" in status
79+
80+
81+
def _major(version: str) -> int:
82+
head = version.split(".", 1)[0]
83+
return int(head) if head.isdigit() else 0
84+
85+
4086
async def machine_image(disk: str) -> str:
4187
"""An image of the launcher holding ``disk``'s files for this machine's platform, built unless held: one for each
4288
image ``disk`` names, by its id, so a disk built again under its tag gets a machine of its own. A thin disk's base

‎src/agent_env/providers/sandbox_providers/local_docker/provider.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here
4242
from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, reset_input, unresettable
4343
from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker
44-
from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_image
44+
from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_for
4545
from agent_env.providers.sandbox_providers.local_docker.relay import relay_image
4646
from agent_env.providers.sandbox_providers.sandbox import (
4747
SANDBOX_LABEL,
@@ -729,13 +729,14 @@ async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str,
729729
"""The launcher booting ``image`` as the sandbox's container, its ports published as a container's are."""
730730
size = json.loads((sandbox.work_dir / _VM_SIZE_FILE).read_text())
731731
await sandbox.pull_images([image])
732-
machine = await machine_image(image)
732+
machine = await machine_for(image)
733733
ports = list(sandbox._port_map)
734734
publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port)
735735
for arg in ("-p", spec)]
736736
running = asyncio.ensure_future(asyncio.to_thread(
737737
docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}",
738-
*publish, *launcher_args(ports, env, size["cpu"], size["memory"]), machine, timeout=_DOCKER_QUERY_SECONDS))
738+
*publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]), machine.image,
739+
timeout=_DOCKER_QUERY_SECONDS))
739740
try:
740741
await asyncio.shield(running)
741742
except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it

‎tst/unit/env/envs/test_computer_windows.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -325,9 +325,11 @@ def test_a_windows_computers_display_is_1920x1080_for_now(local_stores, cli_rout
325325
"display: a Windows computer's display is 1920x1080 for now"]
326326

327327

328-
@pytest.mark.parametrize("backing, over", [(None, []), ("/disk/base.img", ["-B", "/disk/base.img", "-F", "qcow2"])],
329-
ids=["a-base", "a-layer-over-its-base"])
330-
def test_a_disk_is_rewritten_keeping_only_the_clusters_it_uses(monkeypatch, tmp_path, backing, over):
328+
@pytest.mark.parametrize("backing, over, args", [
329+
(None, [], ()), ("/disk/base.img", ["-B", "/disk/base.img", "-F", "qcow2"], ()),
330+
("/disk/base.img", ["-B", "/disk/base.img", "-F", "qcow2"], ("--mount", "type=image,source=base:1,target=/disk"))],
331+
ids=["a-base", "a-layer-over-its-base", "a-layer-over-its-base-mounted"])
332+
def test_a_disk_is_rewritten_keeping_only_the_clusters_it_uses(monkeypatch, tmp_path, backing, over, args):
331333
ran = []
332334

333335
def run(argv, **kwargs):
@@ -337,8 +339,8 @@ def run(argv, **kwargs):
337339
(tmp_path / "disk.img").write_text("as Windows left it")
338340
monkeypatch.setattr(windows_disk.subprocess, "run", run)
339341

340-
windows_disk._converted(tmp_path, "machine:1", backing=backing)
342+
windows_disk._converted(tmp_path, "machine:1", backing=backing, args=args)
341343

342-
assert ran == [["docker", "run", "--rm", "-v", f"{tmp_path}:/out", "--entrypoint", "qemu-img", "machine:1", "convert",
343-
"-O", "qcow2", *over, "/out/disk.img", "/out/compact.img"]]
344+
assert ran == [["docker", "run", "--rm", "-v", f"{tmp_path}:/out", *args, "--entrypoint", "qemu-img", "machine:1",
345+
"convert", "-O", "qcow2", *over, "/out/disk.img", "/out/compact.img"]]
344346
assert (tmp_path / "disk.img").read_text() == "compacted" and not (tmp_path / "compact.img").exists()

‎tst/unit/env/envs/test_disk_recipe.py‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,7 @@ def launched(monkeypatch):
441441
ran = []
442442
monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"}))
443443
monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64")
444-
monkeypatch.setattr(local_sandbox, "machine_image", AsyncMock(return_value="agentenv-vm:0123"))
444+
monkeypatch.setattr(local_sandbox, "machine_for", AsyncMock(return_value=local_disk.Machine("agentenv-vm:0123")))
445445
monkeypatch.setattr(local_sandbox, "_free_host_port", lambda: 20001)
446446
monkeypatch.setattr(local_sandbox, "_host_ips", lambda: ("127.0.0.1",))
447447
monkeypatch.setattr(local_sandbox, "docker", lambda *args, input=None, timeout=None: ran.append(
@@ -526,6 +526,82 @@ async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypat
526526
assert launched == [] and LocalSandboxProvider.disk_image_platforms() == frozenset()
527527

528528

529+
@pytest.mark.asyncio
530+
async def test_a_disk_boots_with_its_image_mounted_into_the_launcher_where_docker_mounts_images(launched, monkeypatch):
531+
mounted = local_disk.Machine("launcher:1", ("--mount", "type=image,source=disk:1,target=/disk,image-subpath=disk"))
532+
monkeypatch.setattr(local_sandbox, "machine_for", AsyncMock(return_value=mounted))
533+
534+
await LocalSandboxProvider().create_vm(image="disk:1", exposed_ports=[18765], env={}, cpu=2, memory=4096)
535+
536+
(run,) = launched
537+
assert run[-1] == "launcher:1" and run[run.index("--mount"):run.index("--mount") + 2] == mounted.args
538+
539+
540+
@pytest.mark.asyncio
541+
@pytest.mark.parametrize("mounts", [True, False], ids=["mounted", "copied"])
542+
async def test_a_disk_is_mounted_into_the_launcher_where_docker_mounts_images_else_copied_into_a_machine_image(
543+
monkeypatch, mounts):
544+
"""No copy of a disk, nor of a Windows disk's base, is made for each launcher where Docker can mount it."""
545+
monkeypatch.setattr(local_disk, "mounts_images", lambda: mounts)
546+
monkeypatch.setattr(local_disk, "launcher", lambda: "launcher:1")
547+
monkeypatch.setattr(local_disk, "machine_image", AsyncMock(return_value="agentenv-vm:1"))
548+
549+
machine = await local_disk.machine_for("disk:1")
550+
551+
assert machine == (local_disk.Machine("launcher:1", (
552+
"--mount", "type=image,source=disk:1,target=/disk,image-subpath=disk")) if mounts else local_disk.Machine(
553+
"agentenv-vm:1"))
554+
assert local_disk.machine_image.await_count == (0 if mounts else 1)
555+
556+
557+
@pytest.mark.parametrize("info, mounts", [
558+
('29.9.3 29.9.3 [["driver-type","io.containerd.snapshotter.v1"]]', True),
559+
('28.0.0 28.0.0 [["driver-type","io.containerd.snapshotter.v1"]]', True),
560+
('27.5.1 27.5.1 [["driver-type","io.containerd.snapshotter.v1"]]', False),
561+
('27.5.1 29.9.3 [["driver-type","io.containerd.snapshotter.v1"]]', False),
562+
('29.9.3 29.9.3 [["Backing Filesystem","extfs"],["Supports d_type","true"]]', False),
563+
("4.9.3 4.9.3 null", False),
564+
(DockerFailed("docker info failed: template: :1: can't evaluate field ClientInfo", 1), False),
565+
(subprocess.TimeoutExpired(["docker", "info"], 30), False)],
566+
ids=["29-containerd", "28-containerd", "27", "27-client-on-a-29-server", "29-overlay2", "podman",
567+
"a-client-too-old-to-say", "no-answer"])
568+
def test_docker_mounts_images_from_28_on_both_ends_on_the_containerd_image_store(monkeypatch, info, mounts):
569+
"""A Docker that fails to say, or doesn't in time, gets the copy."""
570+
local_disk._docker_mounts_images.cache_clear()
571+
asked = []
572+
573+
def docker(*args, timeout=None):
574+
asked.append(timeout)
575+
if isinstance(info, Exception):
576+
raise info
577+
return info
578+
579+
monkeypatch.setattr(local_disk, "docker", docker)
580+
581+
assert local_disk.mounts_images() is mounts and asked == [local_disk._DOCKER_INFO_SECONDS]
582+
local_disk._docker_mounts_images.cache_clear()
583+
584+
585+
def test_a_docker_that_failed_to_say_is_asked_again_and_an_answer_is_kept(monkeypatch):
586+
"""A daemon busy for a moment doesn't leave a long-lived process copying disks for good."""
587+
local_disk._docker_mounts_images.cache_clear()
588+
answers = [subprocess.TimeoutExpired(["docker", "info"], 30),
589+
'29.9.3 29.9.3 [["driver-type","io.containerd.snapshotter.v1"]]']
590+
asked = []
591+
592+
def docker(*args, timeout=None):
593+
asked.append(args)
594+
answer = answers.pop(0)
595+
if isinstance(answer, Exception):
596+
raise answer
597+
return answer
598+
599+
monkeypatch.setattr(local_disk, "docker", docker)
600+
601+
assert [local_disk.mounts_images() for _ in range(3)] == [False, True, True] and len(asked) == 2
602+
local_disk._docker_mounts_images.cache_clear()
603+
604+
529605
@pytest.mark.asyncio
530606
@pytest.mark.parametrize("layers, dockerfile", [
531607
("1", "FROM launcher:1\nCOPY --from=disk:1 /disk/ /disk/\n"),

0 commit comments

Comments
 (0)