Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions src/agent_env/env/envs/computer/utils/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,13 @@
from agent_env.config import get_config
from agent_env.env.envs.computer.utils import layers_in_image
from agent_env.env.published_images import COMPUTER_IMAGES
from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag
from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag, pull_lock
from agent_env.providers.sandbox_providers.sandbox import upload_vm_file
from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM
from agent_env.store.base import ObjectAlreadyExistsError
from agent_env.store.image_store.oci_registry_credentials import (
is_loopback_host, names_registry, registry_host_from_ref)
from agent_env.store.local_state import holding_locks
from agent_env.utils.docker.build import DockerBuildError, build_image

if TYPE_CHECKING:
Expand Down Expand Up @@ -243,10 +244,12 @@ def _docker_here(link: SandboxProvider) -> bool:
@functools.cache
def _runs_here(platform: str, image: str) -> bool:
"""Whether this machine's Docker runs ``platform``'s programs, natively or emulated, as a container of ``image`` for
it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64."""
it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64. Its pull
of ``image`` takes turns with the others (``pull_lock``), as a compose's other arch resolves on the same base."""
try:
return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image],
capture_output=True).returncode == 0
with holding_locks([pull_lock(image)]):
return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image],
capture_output=True).returncode == 0
except FileNotFoundError:
return False

Expand Down Expand Up @@ -285,15 +288,17 @@ def __init__(self, name: str, work: Path, base: DockerImageArtifact, platform: s
def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict:
"""With the build context and what the resolver reads copied into its container, and what it wrote copied out,
rather than bind-mounted: a Docker in a VM, as on a Mac, mounts only the folders the VM shares, colima's only the
home, and keeps no owner on them."""
home, and keeps no owner on them. The container's pull of ``source`` takes turns with the others
(``pull_lock``), its wait for its turn within ``timeout`` too."""
out = self.work / RESOLVED / "out"
out.mkdir(parents=True)
(out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]}))
shutil.copy(RESOLVER, out / "resolve.py")
container, deadline = f"agentenv-resolve-{uuid.uuid4().hex[:12]}", time.monotonic() + timeout
try:
_docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint",
"python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline)
with holding_locks([pull_lock(source)], deadline=deadline):
_docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint",
"python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline)
_docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx", deadline=deadline)
_docker("cp", f"{out}/.", f"{container}:/out", deadline=deadline)
ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True,
Expand All @@ -303,6 +308,8 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict:
_docker("cp", f"{container}:/out/.", str(out), deadline=deadline)
except subprocess.TimeoutExpired as e:
raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None
except TimeoutError:
raise ResolveError(f"resolving took over {timeout} s, waiting for another pull of {source}") from None
finally:
subprocess.run(["docker", "rm", "--force", container], capture_output=True)
return json.loads((out / "result.json").read_text())
Expand Down
9 changes: 8 additions & 1 deletion src/agent_env/providers/sandbox_providers/host_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,18 @@ async def build_here(image: DockerImageArtifact) -> None:
await asyncio.to_thread(build, image)


def pull_lock(image_name: str) -> str:
"""The lock a pull of ``image_name`` on this machine holds, ``docker pull``'s or that of a container of an image
Docker lacks: two at once, even of two of its platforms, can leave Docker's containerd image store holding a
platform's manifest without all its layers, and creating a container of it then fails."""
return f"image-pull:{image_name}"


def pull(image: DockerImageArtifact) -> None:
"""Have this machine's Docker hold ``image``, one with an ``image_digest``, under its name, unless it already holds
it (``holds_digest``): pulled by its digest, or, when that fails, built from its build context. Raises RuntimeError
when it can't be pulled or built."""
with holding_locks([f"image-pull:{image.pinned_name}"]):
with holding_locks([pull_lock(image.pinned_name)]):
if holds_digest(image):
return
pulled = subprocess.run(["docker", "pull", image.pinned_name], capture_output=True, text=True, errors="replace")
Expand Down
22 changes: 19 additions & 3 deletions src/agent_env/store/local_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ def ensure_state_dir(path: Path) -> None:


@contextmanager
def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) -> Iterator[None]:
def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None,
deadline: float | None = None) -> Iterator[None]:
"""Hold a lock on each of ``ids`` in the per-user state root, so another process writing any of them waits
here, calling ``on_wait`` first. The locks are taken in one order, so two holders can't each hold one the
other waits for."""
other waits for. With a ``deadline`` (``time.monotonic()``), a held lock is tried again every ``_LOCK_POLL_S``
until then, and TimeoutError raised after."""
if fcntl is None:
yield
return
Expand All @@ -65,13 +67,27 @@ def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None)
if on_wait is not None and not waited:
on_wait()
waited = True
fcntl.flock(fds[-1], fcntl.LOCK_EX)
if deadline is None:
fcntl.flock(fds[-1], fcntl.LOCK_EX)
else:
_lock_by(fds[-1], deadline)
yield
finally:
for fd in fds:
os.close(fd)


def _lock_by(fd: int, deadline: float) -> None:
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
return
except BlockingIOError:
if time.monotonic() >= deadline:
raise TimeoutError("another holder kept the lock past the deadline") from None
time.sleep(min(_LOCK_POLL_S, max(deadline - time.monotonic(), 0)))


@asynccontextmanager
async def awaiting_lock(id: str, waiting: Callable[[float], Awaitable[None]] | None = None,
every_s: float | None = None) -> AsyncIterator[None]:
Expand Down
19 changes: 17 additions & 2 deletions tst/integration/cli/run_bundle_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from agent_env.cli import cli
from agent_env.config import configure, reset_config
from agent_env.config.paths import state_root
from agent_env.providers.sandbox_providers.host_images import present
from agent_env.providers.sandbox_providers.local_docker.provider import (
LocalSandboxProvider,
local_sandbox_ids,
Expand All @@ -36,6 +37,7 @@
from agent_env.store import Filter, LocalSqliteDocumentStore
from agent_env.store.routing import namespace_routing
from agent_env.task import Task
from tst.integration.utils.registry import REGISTRY_IMAGE
from tst.util.capabilities import skip_without_gnu_stat

pytestmark = pytest.mark.integration
Expand Down Expand Up @@ -254,13 +256,26 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co
assert not _alive(int(nap.read_text()))


def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path):
@pytest.fixture(scope="module")
def held_image():
"""The image the tests' throwaway registries run, which the workflow pulls before the tests, under a name pinned to
its digest, so a local sandbox starts a container of it pulling nothing: a pull from a public registry at the test's
time failed whenever ECR Public refused the runner's anonymous pulls."""
if not present(REGISTRY_IMAGE):
subprocess.run(["docker", "pull", "--quiet", REGISTRY_IMAGE], check=True, capture_output=True)
inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .RepoDigests}}", REGISTRY_IMAGE],
check=True, capture_output=True, text=True)
return json.loads(inspected.stdout)[0]


def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path,
held_image):
napping = tmp_path / "napping"
(napping / "tasks").mkdir(parents=True)
(napping / "tasks/nap.json").write_text(json.dumps([
{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local"},
{"id": "web", "type": "deploy_sandbox", "sandbox_name": "web", "sandbox_mode": "container",
"sandbox_type": "local", "image": "public.ecr.aws/nginx/nginx:stable-alpine", "port": 80},
"sandbox_type": "local", "image": held_image, "port": 5000},
{"id": "nap", "type": "verify_sandbox", "sandbox_name": "box", "base_dir": "/app", "verifier_id": "nap",
"depends_on": ["box", "web"], "criteria": [{"type": "bash_cmd_succeeds", "criterion": "naps",
"bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]},
Expand Down
59 changes: 57 additions & 2 deletions tst/unit/env/envs/test_computer_builder.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Where and how a computer env's pairs build: this machine's Docker, natively or emulated, or a VM of the put's
sandbox provider, each arch on the first link of a chain that runs it natively; what builds when emulation is off
or no link can build; a VM the provider loses; the VM's shell helper; and what a put says as it goes. Docker,
the registry and the VMs are faked."""
or no link can build; a VM the provider loses; pulls of a base taking turns; the VM's shell helper; and what a put says
as it goes. Docker, the registry and the VMs are faked."""

import hashlib
import io
Expand All @@ -13,6 +13,7 @@
import sys
import tarfile
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace

Expand All @@ -25,6 +26,8 @@
from agent_env.env import Env
from agent_env.env.envs.computer.utils import builder, composer, layers_in_image, packages
from agent_env.env.envs.computer.utils.composer import Composing
from agent_env.providers.sandbox_providers import host_images
from agent_env.store.local_state import holding_locks
from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider
from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider
Expand Down Expand Up @@ -164,6 +167,58 @@ def docker(command, **kwargs):
assert failed.value.log == "Pulling from acme/base"


#: How long a fake pull of the base takes: long enough that pulls not taking turns would overlap.
PULL_S = 0.2


def test_a_resolves_pull_of_its_base_takes_turns_with_a_platform_checks_and_a_deploys(tmp_path, monkeypatch):
"""A compose resolves one arch in a container of the base as it checks the other's with another, each pulled when
Docker lacks the base, and two pulls of one image at once left Docker's containerd image store without layers of
it, which every container of it then failed on."""
base, digest = "ghcr.io/acme/base:1", "sha256:" + "b" * 64
pulling, ran, most = [], [], []

def docker(command, **kwargs):
if f"{base}@{digest}" in command and command[1] in ("create", "run", "pull"):
pulling.append(command[1])
most.append(len(pulling))
time.sleep(PULL_S)
pulling.remove(command[1])
ran.append(command[1])
if command[1] == "cp" and command[3] == str(tmp_path / "work" / builder.RESOLVED / "out"):
(Path(command[3]) / "result.json").write_text(json.dumps({"apt": [], "wheels": {}}))
return subprocess.CompletedProcess(command, 0, "", "")

_, local = _resolving(tmp_path, monkeypatch, docker)
deployed = DockerImageArtifact(id="base-image", version=1, description="", image_name=base, image_digest=digest)
builder._runs_here.cache_clear()
with ThreadPoolExecutor(max_workers=3) as pool:
calls = [pool.submit(local.resolve, {"apt": [], "python": []}, f"{base}@{digest}", "amd64", 60),
pool.submit(builder._runs_here, "linux/arm64", f"{base}@{digest}"),
pool.submit(host_images.pull, deployed)]
results = [call.result() for call in calls]
builder._runs_here.cache_clear()

assert results == [{"apt": [], "wheels": {}}, True, None]
assert sorted(ran) == ["create", "pull", "run"] and max(most) == 1


def test_a_resolve_waiting_for_another_pull_of_its_base_gives_up_at_its_own_timeout(tmp_path, monkeypatch):
ran = []

def docker(command, **kwargs):
ran.append(command[1])
return subprocess.CompletedProcess(command, 0, "", "")

_, local = _resolving(tmp_path, monkeypatch, docker)
started = time.monotonic()
with holding_locks([host_images.pull_lock("base")]), \
pytest.raises(builder.ResolveError, match="resolving took over 1 s, waiting for another pull of base"):
local.resolve({"apt": [], "python": []}, "base", "amd64", 1)

assert 1 <= time.monotonic() - started < 5 and ran == ["rm"]


def _resolving(tmp_path, monkeypatch, docker):
"""A builder of this machine's Docker in ``tmp_path``, its build context a driver's, ``docker`` its subprocess.run."""
work = tmp_path / "work"
Expand Down
3 changes: 2 additions & 1 deletion tst/unit/env/gateway/user_chat_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,8 @@ def test_get_time_reaches_a_configured_role_through_a_deny_all_while_the_clock_i
_req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]})
assert "get_time" not in _names(gateway, "psi_agent") # no clock set, so no get_time at all

assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0}).status_code == 200
assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0, "virtual_seconds_per_real_second": 0}
).status_code == 200
assert "get_time" in _names(gateway, "psi_agent") and "get_time" not in _names(gateway, "other")
called = _call(gateway, "get_time", {}, "psi_agent")
assert called.status_code == 200 and T0 in called.text
Expand Down
Loading