Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
"""An app's data plane as its own server serves it (``AppDataPlane``), by the app's data hooks or, with none, the
defaults on the paths it keeps its data in, under the session user's home: a load writes the files it's given under
them, an export returns the files there, and a reset puts back the copy its server kept when it first started on this
boot, which is the app as installed, since every boot starts from the computer's image."""
boot, which is the app as installed, since every boot starts from the computer's image. An export copies the files with
the app paused, or, where the app holds one so that no other process may read it, as Windows lets an app hold a file
and a pause doesn't release, with it stopped and relaunched after."""
from __future__ import annotations

import asyncio
Expand All @@ -17,7 +19,7 @@

from ..agent_env_environment import OP_ADD_DATA, OP_GET_DATA, OP_RESET_DATA
from ..transfers import WriteNamespaceGrant
from ..types import METHOD_ADD, METHOD_RESET, FilePart, uploaded_object_path
from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, FilePart, uploaded_object_path
from .app_runtime import paused_app
from .dataplane import data_files, file_parts, part_bytes

Expand All @@ -27,15 +29,24 @@
logger = logging.getLogger(__name__)

#: The cause a launch after each data call logs.
_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_RESET_DATA: METHOD_RESET}
_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_GET_DATA: METHOD_GET, OP_RESET_DATA: METHOD_RESET}
#: Where an app's server keeps its paths as installed, in the app's folder.
INITIAL = Path(".agentenv/initial")
#: Windows' errors for a file another process holds so that no other may read it: a sharing violation, and a lock on
#: the part read.
_HELD = (32, 33)


class HeldByApp(OSError):
"""A file under the app's paths that the running app holds so that no other process may read it, which stopping
the app releases and pausing it doesn't."""


class AppDataPlane:
"""An app's data plane, served around its contract: one data call or launch at a time; a load and a reset stop the
running app first and relaunch it after, but for those the app serves live, and a load that fails is followed by a
reset, when the app says so. A call fails once it runs
reset, when the app says so; an export the app holds a file of (``HeldByApp``) stops it and relaunches it too. A
call fails once it runs
past ``timeout_s`` and is cancelled: what awaits stops at once, but a thread can't be stopped, so a sync hook's or
a default's thread is waited for, as are the reset and relaunch after it and the next call."""

Expand Down Expand Up @@ -90,7 +101,7 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio.
running = op != OP_GET_DATA and op not in self.live and await self.runtime.stop_app()
ran = op in self.live and self.runtime.running()
try:
result = await self._within(op, served, args, answer)
result = await self._within_held(op, served, args, answer)
except Exception:
if op == OP_ADD_DATA and self.resets_failed_add():
if op in self.live:
Expand All @@ -110,6 +121,24 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio.
else:
answer.set_result(result)

async def _within_held(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any:
"""``served``'s result within ``timeout_s``; where the app holds a file of its paths (``HeldByApp``), again with
the app stopped, in what's left of it, and the app relaunched after, as a load's is."""
loop = asyncio.get_running_loop()
deadline = loop.time() + self.timeout_s
try:
return await self._within(op, served, args, answer)
except HeldByApp:
stopped = await self.runtime.stop_app()
try:
result = await self._within(op, served, args, answer, deadline - loop.time())
finally:
if stopped:
await self.runtime.launch_app(_CAUSES[op])
if stopped and self.relaunched is not None:
await self.relaunched(op)
return result

async def _undo(self, answer: asyncio.Future) -> None:
"""Reset the app's data after a load that failed."""
if (reset := self._served(OP_RESET_DATA)) is None:
Expand Down Expand Up @@ -140,12 +169,14 @@ def _served(self, op: str) -> Optional[Callable]:
OP_GET_DATA: functools.partial(get, self.paths),
OP_RESET_DATA: functools.partial(reset, self.paths)}[op]

async def _within(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any:
"""``served``'s result within ``timeout_s``; past it, ``answer`` fails at once and the call is cancelled, and
this raises the same once it has ended: at once for what awaits, after any thread it runs."""
async def _within(self, op: str, served: Callable, args: tuple, answer: asyncio.Future,
left_s: Optional[float] = None) -> Any:
"""``served``'s result within ``left_s``, ``timeout_s`` by default; past it, ``answer`` fails at once and the
call is cancelled, and this raises the same once it has ended: at once for what awaits, after any thread it
runs."""
work = asyncio.ensure_future(_called(served, *args))
try:
return await asyncio.wait_for(asyncio.shield(work), self.timeout_s)
return await asyncio.wait_for(asyncio.shield(work), max(0.0, self.timeout_s if left_s is None else left_s))
except TimeoutError:
late = TimeoutError(f"{op} ran past the app's data_timeout_s, {self.timeout_s:g} s")
if not answer.done():
Expand Down Expand Up @@ -185,11 +216,17 @@ async def add(parts: list, paths: list[str]) -> None:

async def get(paths: list[str], write_namespace: Optional[WriteNamespaceGrant]) -> list:
"""A part for each file under ``paths``, copied with the app this server launched paused (``paused_app``), so they
are of one moment, and sent from the copy once it runs again; an app's own data/get hook calls it so too."""
are of one moment, and sent from the copy once it runs again; an app's own data/get hook calls it so too. A file
the paused app holds raises ``HeldByApp``, for its data plane to copy them with it stopped."""
INITIAL.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=INITIAL.parent) as staging:
with paused_app():
await in_thread(_stage, Path.home(), paths, Path(staging))
try:
with paused_app():
await in_thread(_stage, Path.home(), paths, Path(staging))
except OSError as e:
if getattr(e, "winerror", None) in _HELD:
raise HeldByApp(f"the running app holds {e.filename or 'a file'} so that nothing else may read it") from e
raise
return await file_parts(Path(staging), paths, write_namespace)


Expand Down
65 changes: 60 additions & 5 deletions packages/agentenv-protocol/tests/test_computer_app_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
the defaults: a load writes files under them, through no link out of them, an export returns them, and a reset puts back
the copy its server kept before it served; its card declares a file tree. Hooks override paths, and read the parts they
get with part_bytes. Around a load and a reset the app is stopped and relaunched, a failed load is followed by a reset,
and data calls take turns; an app that loads live is left running through a load, and a failed one is reset with it
stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past
and data calls take turns; an export of a file the paused app holds stops it, copies it in what's left of its
timeout, and relaunches it after; an app that loads live is left running through a load, and a failed one is reset
with it stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past
data_timeout_s fails then and is cancelled, but a sync hook runs in a thread, which its reset and relaunch, and the next
call, wait for. The app serves its skill file."""
from __future__ import annotations
Expand All @@ -20,7 +21,7 @@

import httpx # noqa: E402

from agentenv_protocol import add_data, client, reset_data # noqa: E402
from agentenv_protocol import add_data, client, get_data, reset_data # noqa: E402
from agentenv_protocol.agent_env_environment import OP_ADD_DATA # noqa: E402
from agentenv_protocol.computer import ComputerAppEnvironment, Launch, linux, part_bytes # noqa: E402
from agentenv_protocol.computer import app, app_data, app_runtime, session # noqa: E402
Expand Down Expand Up @@ -106,8 +107,10 @@ async def stop(pid, timeout_s):
@contextlib.contextmanager
def paused(pid):
events.append(("pause", pid))
yield
events.append(("continue", pid))
try:
yield
finally:
events.append(("continue", pid))

monkeypatch.setattr(session, "launch", launch)
monkeypatch.setattr(session, "stop", stop)
Expand Down Expand Up @@ -210,6 +213,32 @@ async def test_a_load_and_a_reset_stop_the_running_app_and_relaunch_it_and_an_ex
("stop", 4244), ("launch", 4248)]


@pytest.mark.asyncio
async def test_an_export_of_a_file_the_paused_app_holds_stops_the_app_copies_it_and_relaunches_it(home, desktop,
monkeypatch):
"""As Windows lets an app hold a file that no other process may read, which a pause doesn't release."""
(home / "Mail").mkdir()
(home / "Mail/inbox.txt").write_text("held")
mail = PathsMail()
mail._computer_app_data_plane().keep_initial()
await mail._computer_app.launch_app()
stage = app_data._stage

def held_while_running(*args):
if desktop[-1][0] == "pause":
held = PermissionError(13, "The process cannot access the file", str(home / "Mail/inbox.txt"))
held.winerror = 32
raise held
stage(*args)

monkeypatch.setattr(app_data, "_stage", held_while_running)
async with _http(mail) as http:
exported = _files(await _rpc(http, "data/get", {}))

assert [event[0] for event in desktop] == ["launch", "pause", "continue", "stop", "launch"]
assert desktop[3] == ("stop", 4242) and exported["Mail/inbox.txt"] == b"held"


@pytest.mark.asyncio
async def test_an_export_copies_the_paths_with_the_app_paused_and_sends_the_copy_once_it_runs_again(home, desktop,
monkeypatch):
Expand Down Expand Up @@ -414,6 +443,32 @@ def wipe(self) -> None:
("launch", 4248), ("stop", 4248), ("load", 1), ("loaded", 1), ("launch", 4252)]


@pytest.mark.asyncio
async def test_an_export_retried_with_the_app_stopped_gets_only_what_is_left_of_its_timeout(home, desktop):
@linux(launch=Launch(command=["held"]))
class Held(ComputerAppEnvironment):
data_timeout_s = 1.0

@get_data
async def export(self) -> list:
if desktop[-1][0] == "launch":
await asyncio.sleep(0.8)
raise app_data.HeldByApp("the running app holds a file")
await asyncio.Event().wait()

held = Held()
await held._computer_app.launch_app()
started = time.monotonic()
async with _http(held) as http:
late = await http.post(RPC_PATH, json=_call("data/get", {}))
answered = time.monotonic() - started
await asyncio.gather(*held._computer_app_data_plane().changing)

assert late.json()["error"]["message"] == "get_data ran past the app's data_timeout_s, 1 s"
assert answered < 1.5
assert desktop == [("launch", 4242), ("stop", 4242), ("launch", 4244)]


@pytest.mark.asyncio
async def test_it_serves_its_skill_file_and_no_skill_without_one(home, tmp_path):
(tmp_path / "folder" / "SKILL.md").write_text("---\ndescription: Reading and sending mail.\n---\n\n# Mail\nUse it.\n")
Expand Down
8 changes: 6 additions & 2 deletions src/agent_env/computers/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ a reset boots the disk afresh on a new overlay in the same launcher, at the same
before reset in place started is refused a reset, to be redeployed. It boots on local Docker where Docker can pass it
`/dev/kvm`, the disk image's files mounted read-only into the launcher where Docker mounts images (28 and later, on the
containerd image store), so no copy of a disk, nor of a Windows disk's base, is made for each launcher, else copied into
an image of the launcher it keeps for the next boot; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as
an image of the launcher it keeps for the next boot. There its computer uses a local object store's grants as a container
does, at `host.docker.internal`: Linux's init and Windows' setup map that name to 10.0.2.2, QEMU's address for the
launcher, which forwards the grant server's port to this machine (`VM_HOST_PORTS`), and the local transfer CA's trust
files ride in the seed (`VM_TRUST`), where the init and agentenv-session point TLS clients at them; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as
its image under nested virtualization: the provider exports the disk from the image store with this machine's Docker,
compressed, once for each composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal
uploads once for each content and the launcher expands onto the sandbox's disk; a thin disk, a Windows one, carries its
Expand Down Expand Up @@ -181,7 +184,8 @@ with 4 vCPUs and 8 GB, on local Docker with KVM and on `modal_vm`; it has comput
protocol's Windows defaults, so the probe runs its fixture there as on Linux, with Python one-liners for the commands it
would give a shell. On `modal_vm` it runs in your own Modal workspace, under the evaluation's terms or a licence of your
own; whether they allow it there is yours to check. Composing leaves `C:\agentenv` the admins', as `/opt/agentenv` is
root's on Linux: every user reads and runs what's there, and the session user writes only the services' logs and each
root's on Linux: every user, and every sandboxed process (Chrome's, as Program Files allows it), reads and runs
what's there, and the session user writes only the services' logs and each
app's `.agentenv` folder. An app's own server runs in the session as its services do, on a loopback port it writes
with a token to that folder, which only the session user, SYSTEM and Administrators reach; it launches the app in a
Job object, which ends whatever the app started. Composing signs in as `agentadmin`, so
Expand Down
10 changes: 9 additions & 1 deletion src/agent_env/computers/disk/linux/boot/init
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# The disk form's first process: mounts the root filesystem the computer's image became, brings up the network, reads
# the image's start (/etc/agentenv/vm/image.env, from its OCI config) and the deploy's environment (the NoCloud seed's
# user-data), then becomes that start, as its user, as the container runtime would: PID 1 is the image's entrypoint.
# As a container finds this machine at host.docker.internal, it finds the launcher at QEMU's address for it, which
# forwards a local grant server's port, and trusts the local transfer CA the seed carries, as a container does.
/bin/busybox --install -s /bin
export PATH=/bin
mount -t proc proc /proc
Expand Down Expand Up @@ -35,11 +37,17 @@ while read -r line; do
workdir) decode "$value"; workdir=$decoded ;;
esac
done < "$config"
if [ -f /seed/trust/ca-bundle.pem ]; then
cp /seed/trust/ca-bundle.pem /seed/trust/ca.pem /newroot/etc/agentenv/ || fail "can't copy the seed's trust files"
export SSL_CERT_FILE=/etc/agentenv/ca-bundle.pem REQUESTS_CA_BUNDLE=/etc/agentenv/ca-bundle.pem \
NODE_EXTRA_CA_CERTS=/etc/agentenv/ca.pem
fi
[ -n "$seed" ] && { seed_environment /seed || fail "$refused"; }
hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data)
hostname "${hostname:-computer}"
hostname > /newroot/etc/hostname
printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n' "$(hostname)" > /newroot/etc/hosts
printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n10.0.2.2\thost.docker.internal\n' \
"$(hostname)" > /newroot/etc/hosts
[ -s /resolv.conf ] && cp /resolv.conf /newroot/etc/resolv.conf
entry=$(awk -F: -v u="${user%%:*}" '$1 == u || $3 == u' /newroot/etc/passwd | head -n 1)
[ -n "$entry" ] || fail "the image's user ${user%%:*} isn't in /etc/passwd"
Expand Down
26 changes: 24 additions & 2 deletions src/agent_env/computers/disk/linux/launcher/launch
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
# VM_PORTS guest TCP ports, each served on the same port here by socat, whose listen backlog takes a burst of
# connections, in front of QEMU's forward on 127.0.0.1 at the port plus 10000, whose backlog is 1;
# socat retries a connection that forward drops every half second, not on the kernel's SYN backoff
# VM_HOST_PORTS ports of this machine's host the guest reaches, as host.docker.internal, at 10.0.2.2, QEMU's address
# for here: socat serves each on 127.0.0.1 here, through to the host, as a local grant server's
# VM_TRUST a folder of trust files, a local transfer CA's, that go into the seed as its trust/ folder
# VM_ARCH the disk's arch, amd64 or arm64; this machine's by default
# VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default
# VM_REQUIRE_KVM when set, fail rather than emulate where KVM can't run the disk
Expand All @@ -26,7 +29,16 @@
# VM_FILES for a windows guest, a folder whose files it finds on a removable FAT drive
# launch reset: boot the running computer afresh on a new overlay, its environment the VM_ENV_<NAME>=<base64> lines on
# stdin, returning once its new QEMU has started; its ports stay up throughout.
# launch forward PORT: serve PORT of this machine's host to the running computer as VM_HOST_PORTS does, for another
# process's grant server, until stopped.
set -euo pipefail

# forward PORT: PORT of this machine's host served on 127.0.0.1 here, where the guest reaches it at 10.0.2.2.
forward() { exec socat "TCP-LISTEN:$1,bind=127.0.0.1,fork,reuseaddr" "TCP:host.docker.internal:$1"; }

if [ "${1:-}" = forward ]; then
forward "$2"
fi
state=${VM_STATE:-/var/lib/agentenv-vm}
# A reset and the boot loop meet here: the reset writes next.env, creates reset and stops the QEMU qemu.pid names; the
# loop, finding reset, takes next.env's environment, removes reset and boots afresh; the reset returns once qemu.pid
Expand Down Expand Up @@ -96,9 +108,10 @@ case "$guest/$arch" in
echo "launch: no architecture $arch" >&2
exit 2 ;;
esac
forwards="" fronts=()
forwards="" fronts=() taken=""
for port in ${VM_PORTS:-}; do
behind=$((port + 10000))
taken="$taken $port $behind"
if [ "$behind" -gt 65535 ] || [[ " $VM_PORTS " == *" $behind "* ]]; then
echo "launch: port $port can't be forwarded: QEMU's forward behind its proxy would take port $behind, which" \
"is $([ "$behind" -gt 65535 ] && echo "above 65535" || echo "another of VM_PORTS")" >&2
Expand All @@ -108,6 +121,14 @@ for port in ${VM_PORTS:-}; do
socat "TCP-LISTEN:$port,fork,reuseaddr,backlog=1024" "TCP:127.0.0.1:$behind,connect-timeout=0.5,retry=600,interval=0.05" &
fronts+=("$!")
done
for port in ${VM_HOST_PORTS:-}; do
if [[ "$taken " == *" $port "* ]]; then
echo "launch: host port $port can't be forwarded: a guest port's proxy or its forward here takes it" >&2
exit 2
fi
forward "$port" &
fronts+=("$!")
done
trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT
trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null; exit 143' TERM INT
if [[ "$disk" == *.zst ]]; then
Expand Down Expand Up @@ -135,8 +156,9 @@ boot() {
qemu-img create -q -f qcow2 -b "$disk" -F qcow2 "$booted"
fi
seed "$work/seed"
[ -z "${VM_TRUST:-}" ] || cp -r "$VM_TRUST" "$work/seed/trust"
mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null
mcopy -i "$work/seed.img" "$work/seed/"* ::/
mcopy -s -i "$work/seed.img" "$work/seed/"* ::/
if [ "$guest" = windows ]; then
cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd "$work/vars.fd"
mkdir "$work/tpm"
Expand Down
Loading
Loading