Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 28 additions & 12 deletions src/agent_env/task_step/task_steps/run_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import json
import logging
import os
import re
import shlex
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
Expand All @@ -55,14 +54,14 @@
_COMPUTER_RUN_DIR = ".agentenv/run_code"
# How much longer than the script's timeout a computer lets the runner run, so the runner stops the script first.
_RUNNER_SLACK_S = 10
# What a POSIX extended regular expression, as `pkill -f` reads one, gives a meaning to outside brackets.
_ERE_SPECIAL = re.compile(r"[.\[\\()*+?{|^$]")

# The bootstrap we run in the sandbox: `python3 runner.py <run_dir> <entrypoint> <script_file> <timeout_seconds>`.
_RUNNER_SOURCE = (Path(__file__).parent / "run_code_runner.py").read_text(encoding="utf-8")
# Inside a computer, the runner's own `--clear` and `--size` stand in for the shell tools its OS may not have, isolated
# from the home it runs in.
_RUNNER_TOOL = ("python3", "-I", "-c", _RUNNER_SOURCE)

# A universe member with one of these names would shadow run_code's own staging files.
_RESERVED_MEMBERS = frozenset({"runner.py", "input.json", "output.json"})
_RESERVED_MEMBERS = frozenset({"runner.py", "input.json", "output.json", "runner.lock", "runner.pid"})

# Where a single-file FileArtifact is always staged, regardless of its own filename.
_STAGED_SCRIPT = "script.py"
Expand Down Expand Up @@ -373,10 +372,11 @@ async def _read_output(self, target: _ExecTarget, run_dir: str) -> Any:
# rejected here rather than slurped into the worker — and we never decode a
# truncated file (which would crash strict UTF-8 decode mid-codepoint).
path = f"{run_dir}/output.json"
exit_code, size_out, stderr = await target.run("wc", "-c", path)
if exit_code != 0:
raise self._exec_error(f"wrote no result file: {self._stderr_tail(stderr)}")
if int(size_out.split()[0]) > self.MAX_OUTPUT_BYTES:
try:
size = await target.size(path)
except OSError as exc:
raise self._exec_error(f"wrote no result file: {self._stderr_tail(str(exc))}") from exc
if size > self.MAX_OUTPUT_BYTES:
raise self._validation_error(
f"result exceeds {self.MAX_OUTPUT_BYTES}-byte cap"
)
Expand Down Expand Up @@ -554,12 +554,20 @@ async def _cat(target: _ExecTarget, path: str) -> str:
return stdout


async def _size(target: _ExecTarget, *command: str) -> int:
exit_code, stdout, stderr = await target.run(*command)
if exit_code != 0:
raise OSError(stderr)
return int(stdout.split()[0])


class _ExecTarget(Protocol):
def run_dir(self, step_id: str) -> str: ...
def endpoints(self, deployed: DeployedSandboxEnv) -> dict: ...
async def reset_dir(self, path: str) -> None: ...
async def write_file(self, path: str, content: str) -> None: ...
async def read_file(self, path: str) -> str: ...
async def size(self, path: str) -> int: ...
async def run(self, *command: str) -> tuple[int, str, str]: ...


Expand Down Expand Up @@ -587,6 +595,9 @@ async def write_file(self, path: str, content: str) -> None:
async def read_file(self, path: str) -> str:
return await _cat(self, path)

async def size(self, path: str) -> int:
return await _size(self, "wc", "-c", path)

async def run(self, *command: str) -> tuple[int, str, str]:
# Files are written as root, so run as root too.
return await self.sandbox.exec_with_output("sudo", *command)
Expand Down Expand Up @@ -618,6 +629,9 @@ async def write_file(self, path: str, content: str) -> None:
async def read_file(self, path: str) -> str:
return await _cat(self, path)

async def size(self, path: str) -> int:
return await _size(self, "wc", "-c", path)

async def run(self, *command: str) -> tuple[int, str, str]:
return await self.sandbox.exec_with_output(*self.prefix, *command)

Expand All @@ -639,15 +653,17 @@ def endpoints(self, deployed: DeployedSandboxEnv) -> dict:
return inside.loopback(deployed)

async def reset_dir(self, path: str) -> None:
"""End whatever still runs from the folder, as a run whose driver died leaves going, then remove it."""
await self.run("pkill", "-f", _ERE_SPECIAL.sub(r"\\\g<0>", f"{path}/runner.py"))
await self.remove_dir(path)

async def remove_dir(self, path: str) -> None:
exit_code, _, stderr = await self.run("rm", "-rf", path)
"""End whatever still runs from the folder, as a run whose driver died leaves going, then remove it."""
exit_code, _, stderr = await self.run(*_RUNNER_TOOL, "--clear", path)
if exit_code != 0:
raise OSError(stderr)

async def size(self, path: str) -> int:
return await _size(self, *_RUNNER_TOOL, "--size", path)

async def write_file(self, path: str, content: str) -> None:
await inside.add_files(self.deployed, {path: content.encode("utf-8")})

Expand Down
94 changes: 90 additions & 4 deletions src/agent_env/task_step/task_steps/run_code_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@

Given a fourth argument, `timeout_seconds`, it first supervises: it reruns itself without
it in a new session and stops that whole process group at the limit, so no target needs
GNU `timeout`.
GNU `timeout`. Windows has no process groups to signal, so there it ends the rerun's process
tree with `taskkill`. While it supervises, it holds a lock on `runner.lock` in `run_dir`, its pid
in `runner.pid`.

`--clear <run_dir>` ends the supervisor holding that lock, with all it started, then removes the
folder; `--size <path>` prints a file's size. run_code runs these inside a computer, as its OS
may have no POSIX tools to do it with.

Both the entry script's own directory and `run_dir` go on sys.path, so a multi-file
artifact can import its siblings whether it is laid out flat or under a subdirectory.
Expand All @@ -18,9 +24,17 @@
_TIMEOUT_EXIT_CODE = 124 # GNU timeout's, which run_code reports as a timeout
_GRACE_SECONDS = 5 # between SIGTERM and SIGKILL
_POLL_SECONDS = 0.1
_LOCK, _PID = "runner.lock", "runner.pid"


def _end_tree(pid):
subprocess.run(["taskkill", "/T", "/F", "/PID", str(pid)], capture_output=True)


def _signal_group(pgid, signum):
if os.name == "nt":
_end_tree(pgid)
return
try:
os.killpg(pgid, signum)
except (ProcessLookupError, PermissionError): # gone, or (macOS) only zombies are left
Expand Down Expand Up @@ -49,8 +63,9 @@ def forward(signum, _frame):

# Caught before the child starts, as GNU timeout does, so it gets their defaults: one ignored here (a background
# job's SIGINT) would stay ignored in it.
for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
signal.signal(signum, forward)
for name in ("SIGTERM", "SIGINT", "SIGHUP"): # Windows has no SIGHUP
if hasattr(signal, name):
signal.signal(getattr(signal, name), forward)
child = subprocess.Popen(command, start_new_session=True)
for signum in early:
_signal_group(child.pid, signum)
Expand All @@ -67,13 +82,78 @@ def forward(signum, _frame):
except subprocess.TimeoutExpired:
pass
if kill_at is not None:
_signal_group(child.pid, signal.SIGKILL)
_signal_group(child.pid, getattr(signal, "SIGKILL", signal.SIGTERM))
status = child.wait()
if timed_out:
return _TIMEOUT_EXIT_CODE
return status if status >= 0 else 128 - status


def _hold(run_dir):
"""Lock ``run_dir`` for this supervisor's life, its pid beside the lock."""
with open(os.path.join(run_dir, _PID), "w") as pid:
pid.write(str(os.getpid()))
lock = open(os.path.join(run_dir, _LOCK), "wb")
if os.name == "nt":
msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
else:
fcntl.flock(lock, fcntl.LOCK_EX)
return lock


def _held(path):
"""Whether another process holds the lock at ``path``."""
with open(path, "r+b") as lock:
try:
if os.name == "nt":
msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
else:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
return True
return False


def _clear(run_dir):
"""End the supervisor still running from ``run_dir``, as a run whose caller died leaves one, with all it started,
then remove the folder. A supervisor passes a SIGTERM on to its group; Windows ends the tree at once."""
lock = os.path.join(run_dir, _LOCK)
if os.path.exists(lock) and _held(lock):
with open(os.path.join(run_dir, _PID)) as pid_file:
pid = int(pid_file.read())
if os.name == "nt":
_end_tree(pid)
else:
try:
os.kill(pid, signal.SIGTERM)
except (ProcessLookupError, PermissionError):
pass
deadline = time.monotonic() + _GRACE_SECONDS * 2
while _held(lock) and time.monotonic() < deadline:
time.sleep(_POLL_SECONDS)
if os.path.isdir(run_dir):
shutil.rmtree(run_dir)


if sys.argv[1:2] in (["--clear"], ["--size"]):
import os
import shutil
import signal
import subprocess
import time

if os.name == "nt":
import msvcrt
else:
import fcntl

if sys.argv[1] == "--size":
print(os.path.getsize(sys.argv[2]))
else:
_clear(sys.argv[2])
sys.exit(0)

if len(sys.argv) == 5:
# Only the supervisor imports these, and not from the run dir (sys.path[0]): an artifact member named after one
# would otherwise run here, outside the limit. The script's own process imports just what it always has.
Expand All @@ -86,6 +166,12 @@ def forward(signum, _frame):
import subprocess
import time

if os.name == "nt":
import msvcrt
else:
import fcntl

lock = _hold(sys.argv[1])
sys.exit(_supervise([sys.executable] + sys.argv[:4], float(sys.argv[4])))

# Below the supervisor, which must not import these.
Expand Down
20 changes: 20 additions & 0 deletions tst/integration/computer/windows/artifacts/check_inside/run.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""A run_code script inside the Windows computer: it runs in the session user's home, and from there reaches the driver
on loopback, whose card says Windows, and its readiness with the computer's token."""
import json
import os
import urllib.request
from pathlib import Path


def run(input):
with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response:
card = json.load(response)
computer = next(ext["params"] for ext in card["capabilities"]["extensions"]
if ext["uri"] == "urn:agentenv:computer/v1")
status = urllib.request.Request(f"{input['env']['url']}/agentenv/ext/computer-ready/status",
headers=input["env"]["headers"])
with urllib.request.urlopen(status, timeout=30) as response:
ready = json.load(response)
if computer["platform"]["os"] != "windows" or not Path.cwd().samefile(Path.home()) or not ready.get("ready"):
raise AssertionError(f"os {computer['platform']['os']}, cwd {Path.cwd()}, ready {ready}")
return {"user": os.environ.get("USERNAME"), "home": str(Path.home())}
4 changes: 3 additions & 1 deletion tst/integration/computer/windows/tasks/notes.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
"verifier_id": "before"},
{"id": "session", "type": "env_outcome_verifier", "env_id": "notes-desk", "file_artifact_id": "check_session",
"verifier_id": "session", "depends_on": ["before"]},
{"id": "reset", "type": "reset_env", "env_id": "notes-desk", "depends_on": ["session"]},
{"id": "inside", "type": "run_code", "env_id": "notes-desk", "script_artifact_id": "check_inside",
"depends_on": ["session"]},
{"id": "reset", "type": "reset_env", "env_id": "notes-desk", "depends_on": ["inside"]},
{"id": "after", "type": "env_outcome_verifier", "env_id": "notes-desk", "file_artifact_id": "check_notes",
"verifier_id": "after", "depends_on": ["reset"]}
]
4 changes: 2 additions & 2 deletions tst/integration/computer/windows_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
passes the conformance probe at full, cua-driver's known failure on Windows waived, before and after a reset in place;
and an agent drives Notepad on it through cua-driver's tools. And the bundle's notes app, composed onto it as a layer,
runs on its deploy's settings, its window up and its install done as its Windows install says, its own folder and
server the session user's alone, before and after a reset; and on it the protocol's Windows session launches, pauses
and stops apps, and all they start, as an app's server does. Needs Docker, KVM and the ISO; the agent's task, a model
server the session user's alone, before and after a reset; on it the protocol's Windows session launches, pauses
and stops apps, and all they start, as an app's server does; and run_code runs a script inside it. Needs Docker, KVM and the ISO; the agent's task, a model
endpoint too."""

from pathlib import Path
Expand Down
5 changes: 1 addition & 4 deletions tst/unit/computer_home.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""A computer at its driver's boundary, for steps that reach inside one: its home a folder on this machine, ``data/add``
writing there, ``data/get`` reading there, and computer-exec/v1 ``run`` running commands there; each call recorded,
with the headers it sent, and the address each ``data/add`` is sent to.
``pkill`` matches nothing, so one test never ends another's processes on this machine."""
with the headers it sent, and the address each ``data/add`` is sent to."""

from __future__ import annotations

Expand Down Expand Up @@ -74,8 +73,6 @@ async def invoke_extension(self, base_url, card, uri, params=None, timeout=30, v
return await asyncio.to_thread(self._run, params)

def _run(self, params: dict) -> dict:
if params["argv"][0] == "pkill":
return {"exit_code": 1, "stdout": "", "stderr": "", "truncated": False, "timed_out": False}
try:
done = subprocess.run(params["argv"], cwd=self.home, env={**os.environ, **(params.get("env") or {})},
capture_output=True, text=True, timeout=params["timeout_s"])
Expand Down
Loading
Loading