From a0a559a7c0d58b9eb4690c0c81d7af4017d836dc Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:56:16 -0700 Subject: [PATCH 1/4] Run unmodified task images: guest injected as an OCI image volume Until now a template's container image had to be the ate-env-guest image itself, so running a task image meant rebaking it with the guest as the entrypoint, once per image. Substrate's image volumes allow the other split: the task image stays unmodified and the guest is mounted into it read-only and started from the mount. Two ways to get such a template: - ate-env manifest template --task-image writes a template whose container is the task image, with the guest image as a volume at /ate and the command re-rooted to /ate/ko-app/ate-env-guest. --workspace passes -workspace to the guest in either mode. - CreateEnvironment gains an image field. ate-env-api takes the request's template (default-template by default) as the base, derives "-<12 hex digits of the digest>" from it on first use and reuses it for later environments on the same image; the response reports it. A derived name already taken by a template that runs something else is a FailedPrecondition, as is a missing base; an unpinned image, a derived name over 63 characters, or a base command that is not an absolute path to the guest is an InvalidArgument. Concurrent creates racing on one derived template both succeed. The derivation lives in internal/apiservice/imagetemplate.go and accepts either kind of base, guest-as-image or already injected, so a second runtime can reuse it. internal/ate gains Get/CreateActorTemplate and the fake control plane stores templates. The CLI gains create --image, the Python client create(image=), and the full-stack e2e suite a create-from-image test gated on ATE_ENV_TASK_IMAGE that checks the guest comes from the volume, the rootfs is the image's, and a second environment reuses the derived template. docs/task-images/README.md is the how-to (both flows, what an image needs, errors, cleanup) and docs/task-images/DESIGN.md the design (derivation rules, naming and idempotency, failure semantics, security, alternatives, future work). Stubs regenerated: Go with buf and protoc-gen-go v1.36.11, Python with grpcio-tools 1.61.3, keeping the committed license headers. --- README.md | 22 +- clients/python/README.md | 16 +- .../ate_env/_gen/ateenv/v1alpha/env_pb2.py | 40 ++-- .../ate_env/_gen/ateenv/v1alpha/env_pb2.pyi | 6 +- clients/python/src/ate_env/client.py | 10 +- clients/python/tests/e2e/test_full_stack.py | 87 ++++++++ clients/python/tests/fakes.py | 7 + clients/python/tests/test_client.py | 24 +++ cmd/ate-env/main.go | 3 + cmd/ate-env/main_test.go | 32 +++ cmd/ate-env/manifest.go | 37 +++- cmd/ate-env/manifest_test.go | 64 ++++++ docs/task-images/DESIGN.md | 195 ++++++++++++++++++ docs/task-images/README.md | 137 ++++++++++++ internal/apiservice/imagetemplate.go | 164 +++++++++++++++ internal/apiservice/imagetemplate_test.go | 172 +++++++++++++++ internal/apiservice/server.go | 53 +++++ internal/apiservice/server_test.go | 128 ++++++++++++ internal/ate/client.go | 36 ++++ .../internaltest/fakecontrol/fakecontrol.go | 69 ++++++- proto/ateenv/v1alpha/env.pb.go | 24 ++- proto/ateenv/v1alpha/env.proto | 10 + 22 files changed, 1306 insertions(+), 30 deletions(-) create mode 100644 docs/task-images/DESIGN.md create mode 100644 docs/task-images/README.md create mode 100644 internal/apiservice/imagetemplate.go create mode 100644 internal/apiservice/imagetemplate_test.go diff --git a/README.md b/README.md index 4f0b948..bc12088 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,21 @@ ate-env manifest template \ --snapshots-bucket gs://$GOOGLE_CLOUD_PROJECT/ate-env/ | kubectl-ate create actor-template -f - ``` +That template's container is the guest image itself. To run another image unmodified, name +it with `--task-image`: the guest is then mounted into it as a read-only OCI image volume at +`/ate` and started from there, so the task image is never rebuilt. + +```bash +ate-env manifest template --template py312 \ + --task-image docker.io/library/python@sha256: \ + --guest-image /ate-env-guest@sha256: \ + --snapshots-bucket | kubectl-ate create actor-template -f - +``` + +See [docs/task-images/README.md](docs/task-images/README.md) for the full guide, including +creating environments from an image on demand, and [docs/task-images/DESIGN.md](docs/task-images/DESIGN.md) +for the design. + Then create and use an environment: ```bash @@ -75,6 +90,11 @@ kubectl port-forward -n ate-env svc/ate-env-api 7777:7777 & # Create an environment. ate-env create dev1 +# Or run any digest-pinned image unmodified. ate-env-api derives a template +# from default-template on first use (guest mounted in as an image volume) +# and reuses it for later environments on the same image. +ate-env create py1 --image docker.io/library/python@sha256: + # Execute a shell command inside the environment. ate-env dev1 shell 'echo hello > /note.txt' @@ -156,7 +176,7 @@ Manages the lifecycle of isolated execution environments (defined in [`proto/ate | RPC | Description | | --- | ----------- | -| `CreateEnvironment` | Creates and starts a new environment actor from an ActorTemplate | +| `CreateEnvironment` | Creates and starts a new environment actor from an ActorTemplate, or from a digest-pinned `image` on top of one: the template becomes the base, the image the container, and the guest is mounted in as a read-only image volume | | `GetEnvironment` | Retrieves environment details and status | | `SuspendEnvironment` | Suspends and checkpoints the environment to snapshot storage | | `DeleteEnvironment` | Deletes the environment permanently | diff --git a/clients/python/README.md b/clients/python/README.md index bac229e..0b2c481 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -144,6 +144,17 @@ env = await client.create("dev1", template_name="my-template", template_atespace="my-atespace") ``` +To run an arbitrary OCI image, pass it pinned by digest. The server derives +an ActorTemplate from the template above, which acts as the base: the image +becomes the container and the `ate-env-guest` is mounted into it as a +read-only image volume, so the image runs unmodified. The derived template +is created on first use and shared by every environment on that image: + +```python +env = await client.create("py1", image="docker.io/library/python@sha256:…") +print((await env.info()).template.name) # default-template-<12 hex of the digest> +``` + To get a handle to an environment that already exists (no RPC is made): ```python @@ -379,4 +390,7 @@ ATE_ENV_API_TARGET=127.0.0.1:17777 .venv/bin/pytest tests/e2e/test_full_stack.py `ATE_ENV_TEMPLATE` optionally overrides the ActorTemplate used for the test environment; `ATE_ENV_READY_TIMEOUT` (default 180s) bounds the wait -for the environment to start serving. +for the environment to start serving. `ATE_ENV_TASK_IMAGE`, a digest-pinned +image that has `sh`, enables the create-from-image test, which runs that +image unmodified with the guest injected and checks the derived template is +reused by a second environment. diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py index 199d872..0ac59a4 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py @@ -38,7 +38,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x61teenv/v1alpha/env.proto\x12\x0e\x61teenv.v1alpha\"*\n\x08Template\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x8a\x01\n\x0b\x45nvironment\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x31\n\x06status\x18\x04 \x01(\x0e\x32!.ateenv.v1alpha.EnvironmentStatus\"d\n\x18\x43reateEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\"M\n\x19\x43reateEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"5\n\x15GetEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"J\n\x16GetEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"9\n\x19SuspendEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1c\n\x1aSuspendEnvironmentResponse\"8\n\x18\x44\x65leteEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1b\n\x19\x44\x65leteEnvironmentResponse*\xbd\x02\n\x11\x45nvironmentStatus\x12\"\n\x1e\x45NVIRONMENT_STATUS_UNSPECIFIED\x10\x00\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_RESUMING\x10\x01\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_RUNNING\x10\x02\x12!\n\x1d\x45NVIRONMENT_STATUS_SUSPENDING\x10\x03\x12 \n\x1c\x45NVIRONMENT_STATUS_SUSPENDED\x10\x04\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_PAUSING\x10\x05\x12\x1d\n\x19\x45NVIRONMENT_STATUS_PAUSED\x10\x06\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_CRASHED\x10\x07\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_DELETING\x10\x08\x32\xb6\x03\n\x12\x45nvironmentService\x12h\n\x11\x43reateEnvironment\x12(.ateenv.v1alpha.CreateEnvironmentRequest\x1a).ateenv.v1alpha.CreateEnvironmentResponse\x12_\n\x0eGetEnvironment\x12%.ateenv.v1alpha.GetEnvironmentRequest\x1a&.ateenv.v1alpha.GetEnvironmentResponse\x12k\n\x12SuspendEnvironment\x12).ateenv.v1alpha.SuspendEnvironmentRequest\x1a*.ateenv.v1alpha.SuspendEnvironmentResponse\x12h\n\x11\x44\x65leteEnvironment\x12(.ateenv.v1alpha.DeleteEnvironmentRequest\x1a).ateenv.v1alpha.DeleteEnvironmentResponseBCZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alphab\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x61teenv/v1alpha/env.proto\x12\x0e\x61teenv.v1alpha\"*\n\x08Template\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x8a\x01\n\x0b\x45nvironment\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x31\n\x06status\x18\x04 \x01(\x0e\x32!.ateenv.v1alpha.EnvironmentStatus\"s\n\x18\x43reateEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\r\n\x05image\x18\x04 \x01(\t\"M\n\x19\x43reateEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"5\n\x15GetEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"J\n\x16GetEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"9\n\x19SuspendEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1c\n\x1aSuspendEnvironmentResponse\"8\n\x18\x44\x65leteEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1b\n\x19\x44\x65leteEnvironmentResponse*\xbd\x02\n\x11\x45nvironmentStatus\x12\"\n\x1e\x45NVIRONMENT_STATUS_UNSPECIFIED\x10\x00\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_RESUMING\x10\x01\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_RUNNING\x10\x02\x12!\n\x1d\x45NVIRONMENT_STATUS_SUSPENDING\x10\x03\x12 \n\x1c\x45NVIRONMENT_STATUS_SUSPENDED\x10\x04\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_PAUSING\x10\x05\x12\x1d\n\x19\x45NVIRONMENT_STATUS_PAUSED\x10\x06\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_CRASHED\x10\x07\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_DELETING\x10\x08\x32\xb6\x03\n\x12\x45nvironmentService\x12h\n\x11\x43reateEnvironment\x12(.ateenv.v1alpha.CreateEnvironmentRequest\x1a).ateenv.v1alpha.CreateEnvironmentResponse\x12_\n\x0eGetEnvironment\x12%.ateenv.v1alpha.GetEnvironmentRequest\x1a&.ateenv.v1alpha.GetEnvironmentResponse\x12k\n\x12SuspendEnvironment\x12).ateenv.v1alpha.SuspendEnvironmentRequest\x1a*.ateenv.v1alpha.SuspendEnvironmentResponse\x12h\n\x11\x44\x65leteEnvironment\x12(.ateenv.v1alpha.DeleteEnvironmentRequest\x1a).ateenv.v1alpha.DeleteEnvironmentResponseBCZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alphab\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -46,28 +46,28 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'ZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alpha' - _globals['_ENVIRONMENTSTATUS']._serialized_start=718 - _globals['_ENVIRONMENTSTATUS']._serialized_end=1035 + _globals['_ENVIRONMENTSTATUS']._serialized_start=733 + _globals['_ENVIRONMENTSTATUS']._serialized_end=1050 _globals['_TEMPLATE']._serialized_start=44 _globals['_TEMPLATE']._serialized_end=86 _globals['_ENVIRONMENT']._serialized_start=89 _globals['_ENVIRONMENT']._serialized_end=227 _globals['_CREATEENVIRONMENTREQUEST']._serialized_start=229 - _globals['_CREATEENVIRONMENTREQUEST']._serialized_end=329 - _globals['_CREATEENVIRONMENTRESPONSE']._serialized_start=331 - _globals['_CREATEENVIRONMENTRESPONSE']._serialized_end=408 - _globals['_GETENVIRONMENTREQUEST']._serialized_start=410 - _globals['_GETENVIRONMENTREQUEST']._serialized_end=463 - _globals['_GETENVIRONMENTRESPONSE']._serialized_start=465 - _globals['_GETENVIRONMENTRESPONSE']._serialized_end=539 - _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_start=541 - _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_end=598 - _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_start=600 - _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_end=628 - _globals['_DELETEENVIRONMENTREQUEST']._serialized_start=630 - _globals['_DELETEENVIRONMENTREQUEST']._serialized_end=686 - _globals['_DELETEENVIRONMENTRESPONSE']._serialized_start=688 - _globals['_DELETEENVIRONMENTRESPONSE']._serialized_end=715 - _globals['_ENVIRONMENTSERVICE']._serialized_start=1038 - _globals['_ENVIRONMENTSERVICE']._serialized_end=1476 + _globals['_CREATEENVIRONMENTREQUEST']._serialized_end=344 + _globals['_CREATEENVIRONMENTRESPONSE']._serialized_start=346 + _globals['_CREATEENVIRONMENTRESPONSE']._serialized_end=423 + _globals['_GETENVIRONMENTREQUEST']._serialized_start=425 + _globals['_GETENVIRONMENTREQUEST']._serialized_end=478 + _globals['_GETENVIRONMENTRESPONSE']._serialized_start=480 + _globals['_GETENVIRONMENTRESPONSE']._serialized_end=554 + _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_start=556 + _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_end=613 + _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_start=615 + _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_end=643 + _globals['_DELETEENVIRONMENTREQUEST']._serialized_start=645 + _globals['_DELETEENVIRONMENTREQUEST']._serialized_end=701 + _globals['_DELETEENVIRONMENTRESPONSE']._serialized_start=703 + _globals['_DELETEENVIRONMENTRESPONSE']._serialized_end=730 + _globals['_ENVIRONMENTSERVICE']._serialized_start=1053 + _globals['_ENVIRONMENTSERVICE']._serialized_end=1491 # @@protoc_insertion_point(module_scope) diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi index 08ab40e..63776c4 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi @@ -62,14 +62,16 @@ class Environment(_message.Message): def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., status: _Optional[_Union[EnvironmentStatus, str]] = ...) -> None: ... class CreateEnvironmentRequest(_message.Message): - __slots__ = ("id", "atespace", "template") + __slots__ = ("id", "atespace", "template", "image") ID_FIELD_NUMBER: _ClassVar[int] ATESPACE_FIELD_NUMBER: _ClassVar[int] TEMPLATE_FIELD_NUMBER: _ClassVar[int] + IMAGE_FIELD_NUMBER: _ClassVar[int] id: str atespace: str template: Template - def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ...) -> None: ... + image: str + def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., image: _Optional[str] = ...) -> None: ... class CreateEnvironmentResponse(_message.Message): __slots__ = ("environment",) diff --git a/clients/python/src/ate_env/client.py b/clients/python/src/ate_env/client.py index ee4c6b4..4804447 100644 --- a/clients/python/src/ate_env/client.py +++ b/clients/python/src/ate_env/client.py @@ -88,13 +88,21 @@ async def create( atespace: str = DEFAULT_ATESPACE, template_name: str | None = None, template_atespace: str | None = None, + image: str | None = None, ) -> Env: """Register and start a new environment; returns a handle to it. The server fills defaults for the template (name "default-template" in atespace "ate-env") when none is given. + + With image (a digest-pinned OCI reference, repo@sha256:...), the + environment runs that image unmodified: the server derives an + ActorTemplate from the template, which then acts as the base, with the + image as the container and the ate-env-guest mounted into it as an + image volume. The derived template is created on first use and reused + for later environments on the same image; info() reports its name. """ - req = env_pb2.CreateEnvironmentRequest(id=id, atespace=atespace) + req = env_pb2.CreateEnvironmentRequest(id=id, atespace=atespace, image=image or "") if template_name or template_atespace: req.template.name = template_name or "" req.template.atespace = template_atespace or "" diff --git a/clients/python/tests/e2e/test_full_stack.py b/clients/python/tests/e2e/test_full_stack.py index 7705bcb..cd3ab73 100644 --- a/clients/python/tests/e2e/test_full_stack.py +++ b/clients/python/tests/e2e/test_full_stack.py @@ -25,6 +25,11 @@ Unlike test_guest_daemon.py, this exercises the whole path: environment lifecycle against the Substrate control plane, and guest operations proxied by ate-env-api through the atenet router into the environment's actor. + +Set ATE_ENV_TASK_IMAGE to a digest-pinned image that has `sh` (for example +docker.io/library/python@sha256:...) to also run the create-from-image test, +which checks that an unmodified image runs with the guest injected as an +image volume and that a second environment reuses the derived template. """ from __future__ import annotations @@ -36,12 +41,16 @@ import pytest +import grpc + from ate_env import ( Client, EnvError, EnvironmentStatus, + InvalidArgumentError, NotFoundError, ProcessState, + RpcError, Signal, ) @@ -49,6 +58,8 @@ READY_TIMEOUT = float(os.environ.get("ATE_ENV_READY_TIMEOUT", "180")) # Optional ActorTemplate override; the server default is "default-template". TEMPLATE = os.environ.get("ATE_ENV_TEMPLATE") +# Optional digest-pinned task image for the create-from-image test. +TASK_IMAGE = os.environ.get("ATE_ENV_TASK_IMAGE") pytestmark = pytest.mark.skipif( not TARGET, @@ -171,3 +182,79 @@ async def test_full_lifecycle(client): return await asyncio.sleep(2) pytest.fail(f"environment {env_id} still exists after delete") + + +async def test_create_from_image_rejects_unpinned_image(client): + # Validated before anything touches the control plane. + with pytest.raises(InvalidArgumentError): + await client.create(f"pye2e-img-{uuid.uuid4().hex[:8]}", image="python:3.12-slim") + + +async def test_create_from_image_needs_a_base_template(client): + missing = f"pye2e-nobase-{uuid.uuid4().hex[:8]}" + with pytest.raises(RpcError) as excinfo: + await client.create( + f"pye2e-img-{uuid.uuid4().hex[:8]}", + template_name=missing, + image="docker.io/library/busybox@sha256:" + "0" * 64, + ) + assert excinfo.value.code == grpc.StatusCode.FAILED_PRECONDITION + assert missing in str(excinfo.value) + + +@pytest.mark.skipif( + not TASK_IMAGE, + reason="set ATE_ENV_TASK_IMAGE=repo@sha256:... (an image with sh) for the create-from-image test", +) +async def test_create_from_image(client): + base = TEMPLATE or "default-template" + want_template = f"{base}-{TASK_IMAGE.split('@sha256:', 1)[1][:12]}" + envs = [] + try: + env = await client.create( + f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, image=TASK_IMAGE + ) + envs.append(env) + + # The derived template is reported at once, before the actor serves. + info = await env.info() + assert info.template is not None + assert info.template.name == want_template + assert info.template.atespace == env.atespace + + await _wait_until_serving(env) + + # The process runs in the task image's rootfs, with the guest coming + # from the image volume rather than from the image itself. + result = await env.shell( + "test -x /ate/ko-app/ate-env-guest && echo guest:volume; " + "test -e /ko-app/ate-env-guest && echo guest:baked; " + "test -r /etc/os-release && echo rootfs:ok" + ) + assert result.exit_code == 0, result + lines = result.stdout.splitlines() + assert "guest:volume" in lines, result + assert "guest:baked" not in lines, "the task image should not carry the guest" + assert "rootfs:ok" in lines, result + + # The workspace is writable and the guest file path works unchanged. + content = os.urandom(64 * 1024 + 1) + path = f"/tmp/pye2e-img-{uuid.uuid4().hex[:8]}.bin" + assert await env.write_file(path, content) == len(content) + assert await env.read_file_bytes(path) == content + + # A second environment on the same image reuses the derived template + # instead of minting another one. + env2 = await client.create( + f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, image=TASK_IMAGE + ) + envs.append(env2) + assert (await env2.info()).template.name == want_template + await _wait_until_serving(env2) + assert (await env2.shell("echo second")).stdout == "second\n" + finally: + for e in envs: + try: + await e.delete() + except EnvError as exc: + pytest.fail(f"cleanup delete of {e.id} failed: {exc}") diff --git a/clients/python/tests/fakes.py b/clients/python/tests/fakes.py index 5bf5f96..fdd5198 100644 --- a/clients/python/tests/fakes.py +++ b/clients/python/tests/fakes.py @@ -62,6 +62,13 @@ async def CreateEnvironment(self, request, context): template.name = request.template.name if request.template.atespace: template.atespace = request.template.atespace + if request.image: + # Mirror ate-env-api: the template becomes the base of one derived + # per image, named after the digest. + if "@sha256:" not in request.image: + await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "image is not pinned by digest") + digest = request.image.split("@sha256:", 1)[1] + template.name = f"{template.name}-{digest[:12]}" environment = env_pb2.Environment( id=request.id, atespace=atespace, diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 15ed523..3879f20 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -49,6 +49,30 @@ async def test_create_omits_template_when_not_given(fake_stack): assert not fakes.environments.last_create_request.HasField("template") +async def test_create_with_image_reports_derived_template(fake_stack): + client, fakes = fake_stack + image = "docker.io/library/python@sha256:" + "0123456789abcdef" * 4 + env = await client.create("dev3", image=image) + assert fakes.environments.last_create_request.image == image + info = await env.info() + assert info.template is not None + assert info.template.name == "default-template-0123456789ab" + + +async def test_create_with_image_and_base_template(fake_stack): + client, fakes = fake_stack + image = "docker.io/library/python@sha256:" + "0123456789abcdef" * 4 + env = await client.create("dev4", template_name="py-base", image=image) + info = await env.info() + assert info.template.name == "py-base-0123456789ab" + + +async def test_create_omits_image_when_not_given(fake_stack): + client, fakes = fake_stack + await client.create("dev5") + assert fakes.environments.last_create_request.image == "" + + async def test_create_duplicate_maps_to_rpc_error_with_code(fake_stack): client, _ = fake_stack await client.create("dev1") diff --git a/cmd/ate-env/main.go b/cmd/ate-env/main.go index fff6767..23a3ade 100644 --- a/cmd/ate-env/main.go +++ b/cmd/ate-env/main.go @@ -137,6 +137,7 @@ func newCreateCommand() *cobra.Command { atespace string createTemplate string createTemplateAtespace string + image string ) cmd := &cobra.Command{ Use: "create ", @@ -154,6 +155,7 @@ func newCreateCommand() *cobra.Command { req := &ateenvv1alpha.CreateEnvironmentRequest{ Id: args[0], Atespace: atespace, + Image: image, } if createTemplate != "" || createTemplateAtespace != "" { tmplAtespace := createTemplateAtespace @@ -173,6 +175,7 @@ func newCreateCommand() *cobra.Command { cmd.Flags().StringVar(&atespace, "atespace", apiservice.DefaultAtespace, "Substrate atespace") cmd.Flags().StringVar(&createTemplate, "template", "", "ActorTemplate name (defaults to server default)") cmd.Flags().StringVar(&createTemplateAtespace, "template-atespace", "", "Substrate atespace of the ActorTemplate (defaults to environment atespace)") + cmd.Flags().StringVar(&image, "image", "", "digest-pinned OCI image to run unmodified (repo@sha256:...); the template becomes the base the guest is taken from") return cmd } diff --git a/cmd/ate-env/main_test.go b/cmd/ate-env/main_test.go index ee995ca..911005a 100644 --- a/cmd/ate-env/main_test.go +++ b/cmd/ate-env/main_test.go @@ -202,3 +202,35 @@ func TestHelpOutput(t *testing.T) { } }) } + +func TestCreateImageFlag(t *testing.T) { + args := []string{"create", "dev1", "--image", "example.com/task@sha256:abc"} + root := newRootCommand(args) + cmd, _, err := root.Find(args[:2]) + if err != nil { + t.Fatalf("root.Find failed: %v", err) + } + if cmd.Flags().Lookup("image") == nil { + t.Fatal("create has no --image flag") + } + if err := cmd.ParseFlags(args[2:]); err != nil { + t.Fatalf("parsing --image: %v", err) + } + if got, _ := cmd.Flags().GetString("image"); got != "example.com/task@sha256:abc" { + t.Errorf("--image = %q", got) + } +} + +func TestManifestTemplateTaskImageFlags(t *testing.T) { + args := []string{"manifest", "template"} + root := newRootCommand(args) + cmd, _, err := root.Find(args) + if err != nil { + t.Fatalf("root.Find failed: %v", err) + } + for _, name := range []string{"task-image", "workspace", "guest-image", "snapshots-bucket"} { + if cmd.Flags().Lookup(name) == nil { + t.Errorf("manifest template has no --%s flag", name) + } + } +} diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index a83a04a..b906013 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -64,6 +64,12 @@ type templateConfig struct { guestImage string guestCommand []string snapshotsBucket string + // taskImage, when set, is the digest-pinned image the actor runs; the + // guest is then mounted into it as an image volume instead of being the + // container image itself. + taskImage string + // workspace, when set, is passed to the guest as -workspace. + workspace string } func (c *templateConfig) resolveImages() error { @@ -73,6 +79,14 @@ func (c *templateConfig) resolveImages() error { if c.snapshotsBucket == "" { return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") } + if c.taskImage != "" { + if _, err := apiservice.ImageDigest(c.taskImage); err != nil { + return fmt.Errorf("--task-image: %w", err) + } + if _, err := apiservice.ImageDigest(c.guestImage); err != nil { + return fmt.Errorf("--guest-image must be digest-pinned to be mounted as an image volume: %w", err) + } + } return nil } @@ -135,7 +149,11 @@ It prints YAML to stdout without touching the cluster.`, if err := tCfg.resolveImages(); err != nil { return err } - return writeActorTemplate(cmd.OutOrStdout(), buildActorTemplate(tCfg)) + tmpl, err := buildTemplate(tCfg) + if err != nil { + return err + } + return writeActorTemplate(cmd.OutOrStdout(), tmpl) }, } @@ -144,6 +162,8 @@ It prints YAML to stdout without touching the cluster.`, cmd.Flags().StringVar(&tCfg.guestImage, "guest-image", "", "digest-pinned ate-env-guest image (repo@sha256:...)") cmd.Flags().StringSliceVar(&tCfg.guestCommand, "guest-command", []string{"/ko-app/ate-env-guest"}, "guest container entrypoint") cmd.Flags().StringVar(&tCfg.snapshotsBucket, "snapshots-bucket", "", "object-storage bucket (with optional prefix) for actor snapshots, e.g. gs://bucket/prefix/") + cmd.Flags().StringVar(&tCfg.taskImage, "task-image", "", "digest-pinned image to run unmodified; the guest is mounted into it as a read-only image volume at "+apiservice.GuestMountPath) + cmd.Flags().StringVar(&tCfg.workspace, "workspace", "", "workspace root inside the actor, passed to the guest as -workspace (default: the guest's default)") return cmd } @@ -305,6 +325,21 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { } } +// buildTemplate returns the ActorTemplate for cfg: the guest template from +// buildActorTemplate with the workspace flag applied and, when a task image is +// set, that image as the container with the guest mounted into it. +func buildTemplate(cfg templateConfig) (*ateapipb.ActorTemplate, error) { + tmpl := buildActorTemplate(cfg) + if cfg.workspace != "" { + c := tmpl.Containers[0] + c.Command = append(append([]string(nil), c.Command...), "-workspace", cfg.workspace) + } + if cfg.taskImage == "" { + return tmpl, nil + } + return apiservice.DeriveImageTemplate(tmpl, cfg.template, cfg.taskImage) +} + // buildAPIDeployment returns the ate-env-api Deployment, pointed at the // in-cluster Substrate endpoints. func buildAPIDeployment(cfg manifestConfig) *appsv1.Deployment { diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index 42a5426..cca2e8b 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -19,6 +19,7 @@ import ( "strings" "testing" + "github.com/agent-substrate/env/internal/apiservice" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" appsv1 "k8s.io/api/apps/v1" @@ -265,3 +266,66 @@ func TestManifestTemplateCommandExecution(t *testing.T) { t.Errorf("expected guest image and storage location in output:\n%s", out) } } + +const testHex64 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestBuildTemplateTaskImage(t *testing.T) { + cfg := testTemplateConfig() + cfg.guestImage = "example.com/guest@sha256:" + testHex64 + cfg.taskImage = "docker.io/library/python@sha256:" + testHex64 + cfg.workspace = "/workspace" + + tmpl, err := buildTemplate(cfg) + if err != nil { + t.Fatalf("buildTemplate: %v", err) + } + if tmpl.GetMetadata().GetName() != "default-template" { + t.Errorf("name = %q, want the configured template name unchanged", tmpl.GetMetadata().GetName()) + } + c := tmpl.GetContainers()[0] + if c.GetImage() != cfg.taskImage { + t.Errorf("container image = %q, want the task image", c.GetImage()) + } + if len(tmpl.GetVolumes()) != 1 || tmpl.GetVolumes()[0].GetImage().GetReference() != cfg.guestImage { + t.Errorf("volumes = %v, want the guest image volume", tmpl.GetVolumes()) + } + if len(c.GetVolumeMounts()) != 1 || c.GetVolumeMounts()[0].GetMountPath() != apiservice.GuestMountPath { + t.Errorf("mounts = %v", c.GetVolumeMounts()) + } + want := []string{"/ate/ko-app/ate-env-guest", "-workspace", "/workspace"} + if got := c.GetCommand(); len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] { + t.Errorf("command = %v, want %v", got, want) + } +} + +func TestBuildTemplateWorkspaceOnly(t *testing.T) { + cfg := testTemplateConfig() + cfg.workspace = "/w" + tmpl, err := buildTemplate(cfg) + if err != nil { + t.Fatalf("buildTemplate: %v", err) + } + c := tmpl.GetContainers()[0] + if c.GetImage() != cfg.guestImage || len(tmpl.GetVolumes()) != 0 { + t.Errorf("without --task-image the guest stays the container image and no volume is added: %v", tmpl) + } + if got := c.GetCommand(); len(got) != 3 || got[0] != "/ko-app/ate-env-guest" || got[2] != "/w" { + t.Errorf("command = %v", got) + } +} + +func TestTemplateConfigResolveImagesTaskImage(t *testing.T) { + cfg := testTemplateConfig() + cfg.taskImage = "python:3.12" + if err := cfg.resolveImages(); err == nil { + t.Error("unpinned --task-image accepted") + } + cfg.taskImage = "docker.io/library/python@sha256:" + testHex64 + if err := cfg.resolveImages(); err == nil { + t.Error("short guest digest accepted together with --task-image") + } + cfg.guestImage = "example.com/guest@sha256:" + testHex64 + if err := cfg.resolveImages(); err != nil { + t.Errorf("pinned images rejected: %v", err) + } +} diff --git a/docs/task-images/DESIGN.md b/docs/task-images/DESIGN.md new file mode 100644 index 0000000..b806acc --- /dev/null +++ b/docs/task-images/DESIGN.md @@ -0,0 +1,195 @@ +# Design: injecting the guest into unmodified task images + +Status: implemented. Companion to [README.md](README.md), which is the how-to. + +## Problem + +Every ate-env operation goes through `ate-env-guest`, a daemon inside the +actor. Until now the only way to run an image as an environment was to make the +guest the container image, which for a task image meant rebuilding it with the +guest layered on top and set as the entrypoint. For one image that is a `ko` +invocation. For an evaluation or RL fleet it is a build and a push per image, +duplicated guest bytes in the registry, a rebuild of every image for every guest +change, and a registry pipeline the harness has to own. + +The two things a container sources, its rootfs and its process, can be supplied +independently. Substrate's ActorTemplate already allows a second OCI image to be +mounted read-only into the actor. This design uses that to inject the guest. + +## Goals + +- Run any digest-pinned image as an environment with no rebuild. +- Keep the guest behavior, API and client code identical in both modes. +- Make the on-demand path idempotent and safe under concurrency, so harnesses + can call it per task without coordinating. +- Keep one place that knows the mount layout, so a second injected runtime can + reuse it. + +Non-goals: validating that an image exists or starts (the actor reports that), +per-create resource overrides, template garbage collection, and multi-container +templates. These are listed under future work. + +## Design + +### Template derivation + +`internal/apiservice/imagetemplate.go` holds `DeriveImageTemplate(base, name, +image)`. Given a base template it returns a new template that: + +- keeps the base's worker selector, snapshot configuration, sandbox + configuration, resources, container env and readiness probe; +- replaces the first container's image with the task image; +- if the base runs the guest as its image, adds a read-only image volume named + `guest` whose reference is the base's image, mounts it at `/ate`, and moves + the command's first element under `/ate` (`/ko-app/ate-env-guest` becomes + `/ate/ko-app/ate-env-guest`); +- if the base already has an image volume, leaves volumes, mounts and command + alone; +- drops server-assigned metadata and status. + +It refuses an unpinned task or guest image, a base without containers, and a +base command that is not an absolute path (a shell wrapper cannot be re-rooted +and would produce a template whose guest is silently missing). + +Both entry points use it: `ate-env manifest template --task-image` derives from +the guest-image template it would otherwise print, and `ate-env-api` derives +from a template fetched from the control plane. + +### Naming and idempotency + +A derived template is named `-`, in the base's atespace. The name is a pure function of the inputs a +caller controls, so: + +- the same image always maps to the same template, and a lookup by name is the + reuse check; +- a human can find the template for an image from its digest; +- 12 hex digits (48 bits) make accidental collisions a non-concern at any + realistic image count, while leaving room under the 63-character resource + name limit for a base name of up to 50 characters. Longer bases are rejected + with a message naming the limit. + +The guest is deliberately not part of the name. Templates are immutable in +Substrate, so a guest change is a new base template with a new name, which +yields new derived names. Encoding the guest digest would only lengthen names +without adding information. + +### Server flow + +`CreateEnvironment` with `image` set: + +1. Validate the digest and compute the derived name; invalid input is + `InvalidArgument` and touches nothing. +2. `GetActorTemplate(derived)`. If it exists and its container image is the + requested image, use it. If it exists with a different image, return + `FailedPrecondition` naming both images: someone else owns that name. +3. Otherwise `GetActorTemplate(base)`; a missing base is `FailedPrecondition` + with the atespace and the command that registers one. +4. Derive and `CreateActorTemplate`. `AlreadyExists` is success: a concurrent + create won the race and built the same template. +5. Create the actor from the derived template and report it in the response. + +The check in step 2 compares the container image only. Volumes and command are +not compared because the only way for them to differ is a template created by +other means under the derived name, which the image comparison already flags +in practice; a stricter comparison would cost a base fetch on every reuse. + +### API shape + +`image` is a field on `CreateEnvironmentRequest` rather than a new RPC, and the +existing `template` field doubles as the base. This keeps one create path, lets +clients that already pass a template start passing an image with one more +argument, and makes "which base" an explicit, per-request choice rather than +server configuration. The derived template is surfaced through the existing +`Environment.template`, so nothing new is needed to observe it. + +### Placement + +Derived templates live in the environment's atespace, next to the base. This is +also where `ate-env-api` already creates actors, so no new permissions are +involved beyond template creation, which the API server's identity needs for +this feature. + +## Failure semantics + +| Situation | Result | +|---|---| +| unpinned task image | `InvalidArgument`, nothing created | +| base missing | `FailedPrecondition`, nothing created | +| base command not re-rootable | `InvalidArgument`, nothing created | +| derived name held by another image | `FailedPrecondition`, nothing created | +| two creates race on a new image | both succeed, one template | +| image cannot be pulled or guest cannot start | create succeeds; actor never becomes ready; golden bake fails and the template reports it | +| control plane unavailable mid-flow | the underlying gRPC status is returned; a half-created template is reused next time because the name is deterministic | + +## Security + +- The task image is run with the same sandbox class and configuration as the + base, so injection changes what runs, not how it is isolated. +- The guest volume is read-only. The task image cannot alter the guest. +- Digest pinning is enforced for both images at every entry point. A tag could + be re-pointed between the golden bake and a later cold boot, which would + invalidate snapshots and make two environments on "the same image" differ. +- Callers of `CreateEnvironment` can now cause template creation. An + `ate-env-api` that is reachable by untrusted callers should already be behind + authentication; this adds template records and golden snapshots to what such + a caller can accumulate. + +## Performance + +The first environment on an image pays the task image pull on the worker it +lands on and a cold boot; the derived template's golden bake runs concurrently +on another worker. Later environments restore from the golden. In the +verification run below the first environment answered a shell command 37 s +after the create call on a small two-worker pool, most of it image pull. + +Costs that scale with the number of distinct images: one template record and +one golden snapshot per image. Neither is reclaimed automatically. + +## Alternatives considered + +- **Bake the guest into each image.** Works today and remains supported; it is + the per-image build cost this design removes. +- **An init container that copies the guest into a shared volume.** Needs a + writable volume and an extra container per actor, and the copy happens on + every cold boot. An image volume is shared by the node's cache and needs no + copy. +- **A server-side image-to-template map.** Equivalent to the derived name, but + state that can drift from the control plane. The name derivation is stateless + and recoverable from the templates themselves. +- **Including the guest digest in the derived name.** Rejected because template + immutability already forces a new base name for a new guest; see Naming. +- **A separate `CreateEnvironmentFromImage` RPC.** More surface for the same + behavior; a field keeps one path. + +## Future work + +- **Second injected runtime.** The derivation is runtime-agnostic apart from + the binary path. A runtime such as OpenSandbox's `execd` or E2B's `envd` is a + second image volume plus a command and readiness probe; the base template + describes which runtime it carries. +- **Per-create overrides** for resources and workspace, so one base can serve + images with different needs. +- **Derived-template garbage collection**, by age or by last use, with the + golden snapshot removed alongside. +- **Image validation at create time**, so a bad reference fails the call + instead of the actor. +- **Template atespace.** `CreateEnvironment` takes `template.atespace` but the + server creates the actor and resolves the template in the environment's + atespace; this predates the change and is unchanged by it. + +## Verification + +- Unit tests cover derivation from both base kinds, naming and its limits, and + every rejection. +- Server tests against the in-process fake control plane cover first create, + reuse, a named base in another atespace, and the failure table. +- `clients/python/tests/e2e/test_full_stack.py` has a create-from-image test + gated on `ATE_ENV_TASK_IMAGE`, which checks that the guest comes from the + volume and not the image, that the rootfs is the task image's, that files + round-trip, and that a second environment reuses the derived template. +- Run once end to end on a Kubernetes cluster running Substrate with a fresh + two-worker ate-env: `ate-env create --image` of an unmodified + `python:3.12-slim` produced the derived template, served `python3 --version` + from the image's own rootfs with the guest present under `/ate`, and deleted + cleanly. diff --git a/docs/task-images/README.md b/docs/task-images/README.md new file mode 100644 index 0000000..6baecb6 --- /dev/null +++ b/docs/task-images/README.md @@ -0,0 +1,137 @@ +# Running task images on ate-env + +Any OCI image can run as an ate-env environment without being rebuilt. The +`ate-env-guest` daemon that serves exec and file operations is mounted into the +image's container as a read-only image volume and started from there, so the +image's filesystem, interpreters and tools are exactly what was published. + +This page is the how-to. The reasoning behind it is in [DESIGN.md](DESIGN.md). + +## How it works + +An ActorTemplate normally names one container image, and for ate-env that image +has been the guest itself. A *task-image template* splits the two things a +container sources: + +``` +rootfs = the task image, unmodified, pinned by digest +process = /ate/ko-app/ate-env-guest, from a second image mounted read-only at /ate +``` + +Substrate materializes both at actor start. Everything else about the template +(worker selector, snapshot settings, sandbox class, resources, readiness probe) +is the same as for a guest-image template, and the guest behaves identically: +the environment's `shell`, process and file APIs, MCP, and suspend and resume +all work unchanged. + +## Prerequisites + +- A Substrate control plane whose ActorTemplate API supports image volumes + (`volumes[].image`). Every supported release does. +- Images pinned by digest (`repo@sha256:...`), both the task image and the + guest. Substrate requires this because a changed image invalidates snapshots. +- Worker nodes that can pull the task image's registry. +- A base template registered with `ate-env manifest template` (the quickstart + in the root README does this). Its guest image is the one injected. + +What the task image needs: + +- A `sh` for `Env.shell()` and for the `shell` MCP tool. Images without one + (distroless) still work for `start_process` with absolute binaries and for + file transfer. +- Port 80 free inside the sandbox: the guest listens there and the readiness + probe hits `/readyz` on it. +- Nothing else. The guest is a static binary and runs as the image's default + user; a writable workspace is wherever `--workspace` points (the guest's + default is `/`). + +## Option A: register a task-image template up front + +Good for a fixed set of images, or when you want the template to carry its own +name, resources or workspace. + +```bash +ate-env manifest template --template py312 \ + --task-image docker.io/library/python@sha256: \ + --guest-image /ate-env-guest@sha256: \ + --workspace /workspace \ + --snapshots-bucket | kubectl-ate create actor-template -f - + +ate-env create job1 --template py312 +``` + +`--task-image` makes the task image the container and the guest an image +volume; `--workspace` is passed to the guest as `-workspace` in either mode. The +printed template is plain YAML you can edit before registering it (resources, +worker selector, sandbox class). + +## Option B: create from an image on demand + +Good when the set of images is open-ended or chosen by a caller, as in RL and +evaluation harnesses. + +```bash +ate-env create py1 --image docker.io/library/python@sha256: +``` + +```python +env = await client.create("py1", image="docker.io/library/python@sha256:") +(await env.info()).template.name # "default-template-<12 hex of the digest>" +``` + +```go +client.Create(ctx, &ateenvv1alpha.CreateEnvironmentRequest{ + Id: "py1", Image: "docker.io/library/python@sha256:", +}) +``` + +What happens on the server: + +1. The request's template (default `default-template`) is the **base**. It must + exist in the environment's atespace. +2. `ate-env-api` looks for a template named `-` next to the base. If it exists and runs that image, it is reused. +3. Otherwise the server derives it from the base and creates it: same worker + selector, snapshot and sandbox settings, env and readiness; the task image as + the container; the base's guest image as a volume at `/ate`; the command + re-rooted to `/ate/ko-app/ate-env-guest`. +4. The environment is created from the derived template, which the response and + `GetEnvironment` report. + +The first environment on an image pays the image pull and a cold boot while the +template's golden snapshot bakes in the background; later ones restore from the +golden. Concurrent first creates on the same image are safe: whichever wins +creates the template, the others reuse it. + +Templates created this way are visible like any other: + +```bash +kubectl-ate get actor-template --atespace ate-env +``` + +A fresh template shows `Failed` in that listing for a few seconds while its +golden bakes, then `Ready`. Environments can be created in the meantime. + +## Errors + +| Error | Meaning | +|---|---| +| `InvalidArgument: image ... is not pinned by digest` | Pass `repo@sha256:<64 hex>`; tags are rejected before anything is created | +| `FailedPrecondition: base actor template ... not found` | Register the base in the environment's atespace first (`ate-env manifest template`) | +| `FailedPrecondition: actor template ... exists but runs ...` | The derived name is taken by a template with a different image; delete it or use another base name | +| `InvalidArgument: ... cannot re-root command ...` | The base's command is not an absolute path to the guest (a shell wrapper); fix the base template | +| Environment never serves, derived template stays `Failed` | The task image could not be pulled or does not start the guest (port 80 taken, no exec permission); check the worker pod logs | + +## Cleanup and limits + +- Derived templates are not deleted when environments are. They are small + control-plane records plus one golden snapshot in object storage each; remove + them with `kubectl-ate delete actor-template` when an image is retired. +- Bumping the guest means a new base template (templates are immutable), and a + new base name yields new derived names. Existing derived templates keep the + guest they were built with. +- Resources, sandbox class and worker selector come from the base; a task image + that needs more is served by registering its own template (Option A) or a + second base. +- The task image's existence is not checked at create time. A bad reference is + reported by the actor failing to start, not by `CreateEnvironment`. diff --git a/internal/apiservice/imagetemplate.go b/internal/apiservice/imagetemplate.go new file mode 100644 index 0000000..706b8de --- /dev/null +++ b/internal/apiservice/imagetemplate.go @@ -0,0 +1,164 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiservice + +import ( + "errors" + "fmt" + "path" + "regexp" + "strings" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" +) + +// GuestMountPath is where the ate-env-guest image is mounted inside an actor +// whose container image is a task image rather than the guest itself. The +// guest binary is then at GuestMountPath + "/ko-app/ate-env-guest". +const GuestMountPath = "/ate" + +// GuestVolumeName is the name of the image volume carrying the guest. +const GuestVolumeName = "guest" + +// defaultGuestBinary is the guest's path inside its own image (ko layout). +const defaultGuestBinary = "/ko-app/ate-env-guest" + +// imageTemplateDigestLen is how many hex digits of the digest go into a +// derived template's name: enough to make collisions a non-concern and short +// enough to keep the name a valid k8s short name next to any base name. +const imageTemplateDigestLen = 12 + +var digestRef = regexp.MustCompile(`@sha256:([0-9a-f]{64})$`) + +// shortName is the shape substrate accepts for a resource name (k8s short +// name): lowercase alphanumerics and '-', at most 63 characters. +var shortName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +const maxShortNameLen = 63 + +// ImageDigest returns the hex digest of a digest-pinned image reference, or an +// error when the reference is not pinned. Substrate requires pinned images in +// ActorTemplates because changing the image invalidates snapshots. +func ImageDigest(image string) (string, error) { + m := digestRef.FindStringSubmatch(image) + if m == nil { + return "", fmt.Errorf("image %q is not pinned by digest (want repo@sha256:<64 hex>)", image) + } + return m[1], nil +} + +// ImageTemplateName is the name of the ActorTemplate derived from base for +// image: "-". +func ImageTemplateName(base, image string) (string, error) { + digest, err := ImageDigest(image) + if err != nil { + return "", err + } + name := base + "-" + digest[:imageTemplateDigestLen] + if len(name) > maxShortNameLen { + return "", fmt.Errorf("derived template name %q is %d characters, the limit is %d; use a shorter base template name", + name, len(name), maxShortNameLen) + } + if !shortName.MatchString(name) { + return "", fmt.Errorf("derived template name %q is not a valid resource name (lowercase alphanumerics and '-')", name) + } + return name, nil +} + +// DeriveImageTemplate returns a copy of base, named name, whose container runs +// image unmodified with the ate-env-guest mounted into it as a read-only image +// volume at GuestMountPath. +// +// base can be either kind of template: one whose container image is the +// guest itself (the shape `ate-env manifest template` writes), in which case +// that image becomes the guest volume and the command is re-rooted under the +// mount; or one that already mounts the guest as an image volume, in which +// case only the container image changes. Everything else (worker selector, +// snapshots, sandbox config, resources, env, readiness) carries over. +func DeriveImageTemplate(base *ateapipb.ActorTemplate, name, image string) (*ateapipb.ActorTemplate, error) { + if base == nil { + return nil, errors.New("base template is required") + } + if name == "" { + return nil, errors.New("derived template name is required") + } + if _, err := ImageDigest(image); err != nil { + return nil, err + } + if len(base.GetContainers()) == 0 { + return nil, fmt.Errorf("base template %q has no containers", base.GetMetadata().GetName()) + } + + tmpl := proto.Clone(base).(*ateapipb.ActorTemplate) + tmpl.Metadata = &ateapipb.ResourceMetadata{ + Atespace: base.GetMetadata().GetAtespace(), + Name: name, + } + tmpl.Status = nil + + c := tmpl.Containers[0] + if guestVolume(tmpl) == nil { + // The base runs the guest as its container image: that image becomes + // the guest volume, and the command moves under the mount. + guestImage := c.GetImage() + if _, err := ImageDigest(guestImage); err != nil { + return nil, fmt.Errorf("base template %q guest image: %w", base.GetMetadata().GetName(), err) + } + tmpl.Volumes = append(tmpl.Volumes, &ateapipb.Volume{ + Name: GuestVolumeName, + Image: &ateapipb.ImageVolumeSource{Reference: guestImage}, + }) + c.VolumeMounts = append(c.VolumeMounts, &ateapipb.VolumeMount{ + Name: GuestVolumeName, + MountPath: GuestMountPath, + }) + command, err := rerootCommand(c.GetCommand()) + if err != nil { + return nil, fmt.Errorf("base template %q: %w", base.GetMetadata().GetName(), err) + } + c.Command = command + } + c.Image = image + return tmpl, nil +} + +// guestVolume returns the template's image volume, or nil when it has none. +func guestVolume(tmpl *ateapipb.ActorTemplate) *ateapipb.Volume { + for _, v := range tmpl.GetVolumes() { + if v.GetImage() != nil { + return v + } + } + return nil +} + +// rerootCommand moves a guest command that pointed into the guest image's +// root under GuestMountPath. An empty command means the guest's default path. +// A command that does not start with an absolute path cannot be re-rooted +// (a shell wrapper, for instance), so it is an error rather than a template +// whose guest is silently not found. +func rerootCommand(command []string) ([]string, error) { + if len(command) == 0 { + return []string{path.Join(GuestMountPath, defaultGuestBinary)}, nil + } + if !strings.HasPrefix(command[0], "/") { + return nil, fmt.Errorf("cannot re-root command %q under %s: it must start with an absolute path to the guest binary", + strings.Join(command, " "), GuestMountPath) + } + out := append([]string(nil), command...) + out[0] = path.Join(GuestMountPath, out[0]) + return out, nil +} diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go new file mode 100644 index 0000000..9190eb4 --- /dev/null +++ b/internal/apiservice/imagetemplate_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiservice + +import ( + "strings" + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" +) + +const ( + testGuestImage = "example.com/ate-env-guest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testTaskImage = "docker.io/library/python@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +) + +func modeABase() *ateapipb.ActorTemplate { + return &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ate-env", Name: "default-template", Uid: "u1", Version: 3}, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"workload": "default-env"}}, + Containers: []*ateapipb.Container{{ + Name: "guest", + Image: testGuestImage, + Command: []string{"/ko-app/ate-env-guest", "-workspace", "/workspace"}, + Env: []*ateapipb.EnvVar{{Name: "PORT", Value: "80"}}, + Readyz: &ateapipb.ContainerReadyz{HttpGet: &ateapipb.HTTPGetAction{Path: "/readyz", Port: 80}}, + }}, + SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://b/p/"}, + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, + Status: &ateapipb.ActorTemplateStatus{}, + } +} + +func TestImageDigestAndTemplateName(t *testing.T) { + name, err := ImageTemplateName("default-template", testTaskImage) + if err != nil { + t.Fatal(err) + } + if name != "default-template-0123456789ab" { + t.Errorf("name = %q", name) + } + for _, bad := range []string{"python:3.12", "python@sha256:abc", "python@sha512:" + strings.Repeat("a", 128), ""} { + if _, err := ImageDigest(bad); err == nil { + t.Errorf("ImageDigest(%q) accepted an unpinned reference", bad) + } + } + // The derived name must stay a valid resource name. + if _, err := ImageTemplateName(strings.Repeat("b", 51), testTaskImage); err == nil { + t.Error("a 64-character derived name was accepted") + } + if name, err := ImageTemplateName(strings.Repeat("b", 50), testTaskImage); err != nil || len(name) != 63 { + t.Errorf("a 63-character derived name should be accepted: %q, %v", name, err) + } + if _, err := ImageTemplateName("Default", testTaskImage); err == nil { + t.Error("an uppercase base name was accepted") + } +} + +func TestDeriveImageTemplateFromGuestImageBase(t *testing.T) { + base := modeABase() + before := proto.Clone(base).(*ateapipb.ActorTemplate) + + got, err := DeriveImageTemplate(base, "default-template-0123456789ab", testTaskImage) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(base, before) { + t.Error("DeriveImageTemplate mutated the base template") + } + + md := got.GetMetadata() + if md.GetName() != "default-template-0123456789ab" || md.GetAtespace() != "ate-env" { + t.Errorf("metadata = %v", md) + } + if md.GetUid() != "" || md.GetVersion() != 0 || got.GetStatus() != nil { + t.Errorf("server-assigned fields were copied: uid=%q version=%d status=%v", md.GetUid(), md.GetVersion(), got.GetStatus()) + } + + c := got.GetContainers()[0] + if c.GetImage() != testTaskImage { + t.Errorf("container image = %q, want the task image", c.GetImage()) + } + wantCmd := []string{"/ate/ko-app/ate-env-guest", "-workspace", "/workspace"} + if strings.Join(c.GetCommand(), " ") != strings.Join(wantCmd, " ") { + t.Errorf("command = %v, want %v", c.GetCommand(), wantCmd) + } + if len(got.GetVolumes()) != 1 || got.GetVolumes()[0].GetName() != GuestVolumeName || + got.GetVolumes()[0].GetImage().GetReference() != testGuestImage { + t.Errorf("volumes = %v, want one image volume %q with the guest image", got.GetVolumes(), GuestVolumeName) + } + if len(c.GetVolumeMounts()) != 1 || c.GetVolumeMounts()[0].GetName() != GuestVolumeName || + c.GetVolumeMounts()[0].GetMountPath() != GuestMountPath { + t.Errorf("volume mounts = %v, want %s at %s", c.GetVolumeMounts(), GuestVolumeName, GuestMountPath) + } + // Everything else carries over. + if !proto.Equal(got.GetWorkerSelector(), base.GetWorkerSelector()) || + !proto.Equal(got.GetSnapshotsConfig(), base.GetSnapshotsConfig()) || + !proto.Equal(got.GetSandboxConfig(), base.GetSandboxConfig()) || + !proto.Equal(c.GetReadyz(), base.GetContainers()[0].GetReadyz()) || + len(c.GetEnv()) != 1 { + t.Error("worker selector, snapshots, sandbox config, readyz or env did not carry over") + } +} + +func TestDeriveImageTemplateFromInjectedBase(t *testing.T) { + // A base that already mounts the guest: only the container image changes. + base, err := DeriveImageTemplate(modeABase(), "py-base", "example.com/base@sha256:"+strings.Repeat("b", 64)) + if err != nil { + t.Fatal(err) + } + got, err := DeriveImageTemplate(base, "py-base-0123456789ab", testTaskImage) + if err != nil { + t.Fatal(err) + } + c := got.GetContainers()[0] + if c.GetImage() != testTaskImage { + t.Errorf("image = %q", c.GetImage()) + } + if c.GetCommand()[0] != "/ate/ko-app/ate-env-guest" { + t.Errorf("command re-rooted twice: %v", c.GetCommand()) + } + if len(got.GetVolumes()) != 1 || len(c.GetVolumeMounts()) != 1 { + t.Errorf("guest volume duplicated: volumes=%d mounts=%d", len(got.GetVolumes()), len(c.GetVolumeMounts())) + } +} + +func TestDeriveImageTemplateDefaultsCommand(t *testing.T) { + base := modeABase() + base.Containers[0].Command = nil + got, err := DeriveImageTemplate(base, "x", testTaskImage) + if err != nil { + t.Fatal(err) + } + if cmd := got.GetContainers()[0].GetCommand(); len(cmd) != 1 || cmd[0] != "/ate/ko-app/ate-env-guest" { + t.Errorf("command = %v", cmd) + } +} + +func TestDeriveImageTemplateRejects(t *testing.T) { + if _, err := DeriveImageTemplate(modeABase(), "x", "python:3.12"); err == nil { + t.Error("unpinned task image accepted") + } + unpinnedGuest := modeABase() + unpinnedGuest.Containers[0].Image = "example.com/ate-env-guest:latest" + if _, err := DeriveImageTemplate(unpinnedGuest, "x", testTaskImage); err == nil { + t.Error("unpinned guest image accepted as an image volume") + } + if _, err := DeriveImageTemplate(&ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Name: "empty"}}, "x", testTaskImage); err == nil { + t.Error("base without containers accepted") + } + if _, err := DeriveImageTemplate(modeABase(), "", testTaskImage); err == nil { + t.Error("empty name accepted") + } + wrapped := modeABase() + wrapped.Containers[0].Command = []string{"sh", "-c", "exec /ko-app/ate-env-guest"} + if _, err := DeriveImageTemplate(wrapped, "x", testTaskImage); err == nil || !strings.Contains(err.Error(), "re-root") { + t.Errorf("a shell-wrapped base command must be rejected, got %v", err) + } +} diff --git a/internal/apiservice/server.go b/internal/apiservice/server.go index e134e15..6c735af 100644 --- a/internal/apiservice/server.go +++ b/internal/apiservice/server.go @@ -96,6 +96,14 @@ func (s *Server) CreateEnvironment(ctx context.Context, req *ateenvv1alpha.Creat atespace = DefaultAtespace } + if image := req.GetImage(); image != "" { + name, err := s.ensureImageTemplate(ctx, atespace, templateName, image) + if err != nil { + return nil, err + } + templateName = name + } + opts := ate.CreateOptions{ ID: req.GetId(), Template: templateName, @@ -118,6 +126,51 @@ func (s *Server) CreateEnvironment(ctx context.Context, req *ateenvv1alpha.Creat }, nil } +// ensureImageTemplate returns the name of the ActorTemplate that runs image +// on top of base, creating it from base on first use. The derived template +// lives in the environment's atespace, next to base. +func (s *Server) ensureImageTemplate(ctx context.Context, atespace, base, image string) (string, error) { + name, err := ImageTemplateName(base, image) + if err != nil { + return "", status.Error(codes.InvalidArgument, err.Error()) + } + existing, err := s.client.GetActorTemplate(ctx, atespace, name) + switch { + case err == nil: + if got := firstContainerImage(existing); got != image { + return "", status.Errorf(codes.FailedPrecondition, + "actor template %q exists but runs %q, not %q; delete it or use another base template", name, got, image) + } + return name, nil + case !errors.Is(err, ate.ErrNotFound): + return "", toGRPCError(err) + } + baseTmpl, err := s.client.GetActorTemplate(ctx, atespace, base) + if err != nil { + if errors.Is(err, ate.ErrNotFound) { + return "", status.Errorf(codes.FailedPrecondition, + "base actor template %q not found in atespace %q; register it first (ate-env manifest template)", base, atespace) + } + return "", toGRPCError(err) + } + derived, err := DeriveImageTemplate(baseTmpl, name, image) + if err != nil { + return "", status.Error(codes.InvalidArgument, err.Error()) + } + // A concurrent create may have won the race; it built the same template. + if _, err := s.client.CreateActorTemplate(ctx, derived); err != nil && !errors.Is(err, ate.ErrAlreadyExists) { + return "", toGRPCError(err) + } + return name, nil +} + +func firstContainerImage(t *ateapipb.ActorTemplate) string { + if len(t.GetContainers()) == 0 { + return "" + } + return t.GetContainers()[0].GetImage() +} + // GetEnvironment retrieves the status and configuration of an existing environment. func (s *Server) GetEnvironment(ctx context.Context, req *ateenvv1alpha.GetEnvironmentRequest) (*ateenvv1alpha.GetEnvironmentResponse, error) { if req.GetId() == "" { diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go index 28a9c2f..6dd2752 100644 --- a/internal/apiservice/server_test.go +++ b/internal/apiservice/server_test.go @@ -528,3 +528,131 @@ func TestActorStatusToEnvStatus(t *testing.T) { } } } + +const ( + testGuestImage = "example.com/ate-env-guest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testTaskImage = "docker.io/library/python@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +) + +// seedGuestTemplate registers the template `ate-env manifest template` +// writes: the guest is the container image. +func seedGuestTemplate(control *fakecontrol.Server, atespace, name string) { + control.AddTemplate(&ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"workload": "default-env"}}, + Containers: []*ateapipb.Container{{ + Name: "guest", + Image: testGuestImage, + Command: []string{"/ko-app/ate-env-guest"}, + Env: []*ateapipb.EnvVar{{Name: "PORT", Value: "80"}}, + Readyz: &ateapipb.ContainerReadyz{HttpGet: &ateapipb.HTTPGetAction{Path: "/readyz", Port: 80}}, + }}, + SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://bucket/ate-env/"}, + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, + }) +} + +func TestCreateFromImageDerivesTemplate(t *testing.T) { + envClient, control := newTestEnv(t) + seedGuestTemplate(control, "ate-env", "default-template") + ctx := context.Background() + + resp, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img1", Image: testTaskImage}) + if err != nil { + t.Fatal(err) + } + const want = "default-template-0123456789ab" + if got := resp.GetEnvironment().GetTemplate().GetName(); got != want { + t.Errorf("response template = %q, want %q", got, want) + } + + tmpl := control.Template("ate-env", want) + if tmpl == nil { + t.Fatalf("derived template %q was not created", want) + } + c := tmpl.GetContainers()[0] + if c.GetImage() != testTaskImage { + t.Errorf("derived container image = %q", c.GetImage()) + } + if len(tmpl.GetVolumes()) != 1 || tmpl.GetVolumes()[0].GetImage().GetReference() != testGuestImage { + t.Errorf("derived volumes = %v, want the guest image volume", tmpl.GetVolumes()) + } + if len(c.GetVolumeMounts()) != 1 || c.GetVolumeMounts()[0].GetMountPath() != apiservice.GuestMountPath { + t.Errorf("derived mounts = %v", c.GetVolumeMounts()) + } + if len(c.GetCommand()) != 1 || c.GetCommand()[0] != "/ate/ko-app/ate-env-guest" { + t.Errorf("derived command = %v", c.GetCommand()) + } + if tmpl.GetSnapshotsConfig().GetStorageLocation() != "gs://bucket/ate-env/" || tmpl.GetWorkerSelector() == nil { + t.Error("snapshots config or worker selector did not carry over") + } + + // The actor references the derived template. + got, err := envClient.GetEnvironment(ctx, &ateenvv1alpha.GetEnvironmentRequest{Id: "img1"}) + if err != nil { + t.Fatal(err) + } + if got.GetEnvironment().GetTemplate().GetName() != want { + t.Errorf("actor template = %q, want %q", got.GetEnvironment().GetTemplate().GetName(), want) + } + + // A second environment on the same image reuses the template. + if _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img2", Image: testTaskImage}); err != nil { + t.Fatal(err) + } + if n := control.TemplateCount(); n != 2 { + t.Errorf("template count = %d, want 2 (base + one derived)", n) + } +} + +func TestCreateFromImageUsesNamedBaseAndAtespace(t *testing.T) { + envClient, control := newTestEnv(t) + seedGuestTemplate(control, "team-a", "py-base") + + resp, err := envClient.CreateEnvironment(context.Background(), &ateenvv1alpha.CreateEnvironmentRequest{ + Id: "img1", + Atespace: "team-a", + Template: &ateenvv1alpha.Template{Name: "py-base"}, + Image: testTaskImage, + }) + if err != nil { + t.Fatal(err) + } + if got := resp.GetEnvironment().GetTemplate().GetName(); got != "py-base-0123456789ab" { + t.Errorf("template = %q", got) + } + if control.Template("team-a", "py-base-0123456789ab") == nil { + t.Error("derived template not created in the environment's atespace") + } +} + +func TestCreateFromImageRejects(t *testing.T) { + envClient, control := newTestEnv(t) + seedGuestTemplate(control, "ate-env", "default-template") + ctx := context.Background() + + cases := []struct { + name string + req *ateenvv1alpha.CreateEnvironmentRequest + code codes.Code + }{ + {"unpinned image", &ateenvv1alpha.CreateEnvironmentRequest{Id: "a", Image: "python:3.12"}, codes.InvalidArgument}, + {"missing base", &ateenvv1alpha.CreateEnvironmentRequest{Id: "b", Template: &ateenvv1alpha.Template{Name: "nope"}, Image: testTaskImage}, codes.FailedPrecondition}, + } + for _, tc := range cases { + _, err := envClient.CreateEnvironment(ctx, tc.req) + if status.Code(err) != tc.code { + t.Errorf("%s: code = %v (%v), want %v", tc.name, status.Code(err), err, tc.code) + } + } + + // A derived name already taken by a template running something else. + control.AddTemplate(&ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ate-env", Name: "default-template-0123456789ab"}, + Containers: []*ateapipb.Container{{Name: "guest", Image: "example.com/other@sha256:" + testGuestImage[len(testGuestImage)-64:]}}, + }) + _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "c", Image: testTaskImage}) + if status.Code(err) != codes.FailedPrecondition { + t.Errorf("conflicting derived template: code = %v (%v), want FailedPrecondition", status.Code(err), err) + } +} diff --git a/internal/ate/client.go b/internal/ate/client.go index a3bf79c..5109d84 100644 --- a/internal/ate/client.go +++ b/internal/ate/client.go @@ -68,6 +68,9 @@ const DefaultAtespace = "ate-env" // ErrNotFound is returned when an env, file, or directory does not exist. var ErrNotFound = errors.New("not found") +// ErrAlreadyExists is returned when a resource with the same name exists. +var ErrAlreadyExists = errors.New("already exists") + // Options configures a Client. type Options struct { // ControlAddr is the ateapi gRPC endpoint, e.g. "localhost:8080" @@ -309,6 +312,39 @@ func (c *Client) Delete(ctx context.Context, atespace, id string) error { // ref returns the ObjectRef identifying the actor backing actor id in atespace. // If atespace is empty, DefaultAtespace is used. +// GetActorTemplate reads an ActorTemplate. A missing template is ErrNotFound. +func (c *Client) GetActorTemplate(ctx context.Context, atespace, name string) (*ateapipb.ActorTemplate, error) { + if name == "" { + return nil, errors.New("ate: template name is required") + } + tmpl, err := c.control.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: c.ref(atespace, name)}) + if err != nil { + return nil, fmt.Errorf("ate: getting template %q: %w", name, wrapGRPCError(err)) + } + return tmpl, nil +} + +// CreateActorTemplate creates an ActorTemplate in its metadata's atespace. A +// name collision is ErrAlreadyExists, so callers racing to create the same +// derived template can treat it as success. +func (c *Client) CreateActorTemplate(ctx context.Context, tmpl *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) { + name := tmpl.GetMetadata().GetName() + if name == "" { + return nil, errors.New("ate: template metadata.name is required") + } + if tmpl.GetMetadata().GetAtespace() == "" { + tmpl.Metadata.Atespace = DefaultAtespace + } + created, err := c.control.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: tmpl}) + if err != nil { + if status.Code(err) == codes.AlreadyExists { + return nil, fmt.Errorf("ate: creating template %q: %w: %s", name, ErrAlreadyExists, status.Convert(err).Message()) + } + return nil, fmt.Errorf("ate: creating template %q: %w", name, wrapGRPCError(err)) + } + return created, nil +} + func (c *Client) ref(atespace, id string) *ateapipb.ObjectRef { if atespace == "" { atespace = DefaultAtespace diff --git a/internal/internaltest/fakecontrol/fakecontrol.go b/internal/internaltest/fakecontrol/fakecontrol.go index c382822..8baac22 100644 --- a/internal/internaltest/fakecontrol/fakecontrol.go +++ b/internal/internaltest/fakecontrol/fakecontrol.go @@ -47,6 +47,7 @@ type Server struct { mu sync.Mutex actors map[string]*ateapipb.Actor atespaces map[string]*ateapipb.Atespace + templates map[string]*ateapipb.ActorTemplate } // New returns an empty fake control server. @@ -54,9 +55,76 @@ func New() *Server { return &Server{ actors: make(map[string]*ateapipb.Actor), atespaces: make(map[string]*ateapipb.Atespace), + templates: make(map[string]*ateapipb.ActorTemplate), } } +// AddTemplate seeds an ActorTemplate, as an operator would have registered it. +func (s *Server) AddTemplate(tmpl *ateapipb.ActorTemplate) { + s.mu.Lock() + defer s.mu.Unlock() + s.templates[key(tmpl.GetMetadata().GetAtespace(), tmpl.GetMetadata().GetName())] = proto.Clone(tmpl).(*ateapipb.ActorTemplate) +} + +// Template returns a copy of a stored ActorTemplate, or nil. +func (s *Server) Template(atespace, name string) *ateapipb.ActorTemplate { + s.mu.Lock() + defer s.mu.Unlock() + t, ok := s.templates[key(atespace, name)] + if !ok { + return nil + } + return proto.Clone(t).(*ateapipb.ActorTemplate) +} + +// TemplateCount is how many ActorTemplates are stored. +func (s *Server) TemplateCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.templates) +} + +func (s *Server) GetActorTemplate(ctx context.Context, req *ateapipb.GetActorTemplateRequest) (*ateapipb.ActorTemplate, error) { + s.mu.Lock() + defer s.mu.Unlock() + ref := req.GetActorTemplate() + t, ok := s.templates[key(ref.GetAtespace(), ref.GetName())] + if !ok { + return nil, status.Errorf(codes.NotFound, "actor template %q not found", ref.GetName()) + } + return proto.Clone(t).(*ateapipb.ActorTemplate), nil +} + +func (s *Server) CreateActorTemplate(ctx context.Context, req *ateapipb.CreateActorTemplateRequest) (*ateapipb.ActorTemplate, error) { + s.mu.Lock() + defer s.mu.Unlock() + t := req.GetActorTemplate() + md := t.GetMetadata() + if md.GetName() == "" { + return nil, status.Error(codes.InvalidArgument, "metadata.name is required") + } + k := key(md.GetAtespace(), md.GetName()) + if _, exists := s.templates[k]; exists { + return nil, status.Errorf(codes.AlreadyExists, "actor template %q already exists", md.GetName()) + } + // Mirror the server's volume_mounts validation so a bad derivation fails here too. + declared := map[string]bool{} + for _, v := range t.GetVolumes() { + declared[v.GetName()] = true + } + for _, c := range t.GetContainers() { + for _, m := range c.GetVolumeMounts() { + if !declared[m.GetName()] { + return nil, status.Errorf(codes.InvalidArgument, "container %q mounts undeclared volume %q", c.GetName(), m.GetName()) + } + } + } + stored := proto.Clone(t).(*ateapipb.ActorTemplate) + stored.Metadata.Uid = "tmpl-" + md.GetName() + stored.Metadata.Version = 1 + s.templates[k] = stored + return proto.Clone(stored).(*ateapipb.ActorTemplate), nil +} // Serve starts the fake on a random localhost port and returns its // address and a shutdown function. Like the real ateapi, it serves TLS @@ -189,7 +257,6 @@ func (s *Server) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorReq return &ateapipb.SuspendActorResponse{Actor: clone(a)}, nil } - // suspend checkpoints a. The caller holds s.mu. func (s *Server) suspend(a *ateapipb.Actor) { a.Status = &ateapipb.ActorStatus{ diff --git a/proto/ateenv/v1alpha/env.pb.go b/proto/ateenv/v1alpha/env.pb.go index a242a1d..4007dc0 100644 --- a/proto/ateenv/v1alpha/env.pb.go +++ b/proto/ateenv/v1alpha/env.pb.go @@ -252,7 +252,17 @@ type CreateEnvironmentRequest struct { // Substrate atespace for the environment (defaults to "ate-env"). Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` // ActorTemplate configuration to instantiate (defaults to name "default-template" in atespace "ate-env"). - Template *Template `protobuf:"bytes,3,opt,name=template,proto3" json:"template,omitempty"` + Template *Template `protobuf:"bytes,3,opt,name=template,proto3" json:"template,omitempty"` + // OCI image to run, pinned by digest (repo@sha256:...). Optional. When set, + // the environment is created from an ActorTemplate derived from `template`, + // which acts as the base: its worker selector, snapshot, sandbox and + // resource settings are kept, its container image is replaced by this + // image, and the ate-env-guest from the base is mounted into it as a + // read-only image volume, so the task image runs unmodified. The derived + // template is named "-" in the + // base's atespace; it is created on first use and reused afterwards, and + // the response reports it. + Image string `protobuf:"bytes,4,opt,name=image,proto3" json:"image,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -308,6 +318,13 @@ func (x *CreateEnvironmentRequest) GetTemplate() *Template { return nil } +func (x *CreateEnvironmentRequest) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + // Response returned after creating an environment. type CreateEnvironmentResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -651,11 +668,12 @@ const file_proto_ateenv_v1alpha_env_proto_rawDesc = "" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x124\n" + "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x129\n" + - "\x06status\x18\x04 \x01(\x0e2!.ateenv.v1alpha.EnvironmentStatusR\x06status\"|\n" + + "\x06status\x18\x04 \x01(\x0e2!.ateenv.v1alpha.EnvironmentStatusR\x06status\"\x92\x01\n" + "\x18CreateEnvironmentRequest\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x124\n" + - "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\"Z\n" + + "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x12\x14\n" + + "\x05image\x18\x04 \x01(\tR\x05image\"Z\n" + "\x19CreateEnvironmentResponse\x12=\n" + "\venvironment\x18\x01 \x01(\v2\x1b.ateenv.v1alpha.EnvironmentR\venvironment\"C\n" + "\x15GetEnvironmentRequest\x12\x0e\n" + diff --git a/proto/ateenv/v1alpha/env.proto b/proto/ateenv/v1alpha/env.proto index 64d77ef..b2ce9b0 100644 --- a/proto/ateenv/v1alpha/env.proto +++ b/proto/ateenv/v1alpha/env.proto @@ -105,6 +105,16 @@ message CreateEnvironmentRequest { string atespace = 2; // ActorTemplate configuration to instantiate (defaults to name "default-template" in atespace "ate-env"). Template template = 3; + // OCI image to run, pinned by digest (repo@sha256:...). Optional. When set, + // the environment is created from an ActorTemplate derived from `template`, + // which acts as the base: its worker selector, snapshot, sandbox and + // resource settings are kept, its container image is replaced by this + // image, and the ate-env-guest from the base is mounted into it as a + // read-only image volume, so the task image runs unmodified. The derived + // template is named "-" in the + // base's atespace; it is created on first use and reused afterwards, and + // the response reports it. + string image = 4; } // Response returned after creating an environment. From 32f83794c3ed07e2afd97d1f2ed5e11eb2c0c30a Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:46:46 -0700 Subject: [PATCH 2/4] Inject a second runtime as a layer: guest sidecars and template layers A task-image template carries the guest as a read-only image volume; an agent framework's own in-sandbox daemon, such as OpenSandbox's execd, is one more volume plus a process that has to start next to the guest. The guest is the container's only process, so it is the one that starts it. ate-env-guest gains -sidecar (repeatable: an absolute binary path and its arguments, run without a shell, output logged with the binary's name as prefix, restarted with a backoff that doubles from one second to thirty and resets after a minute of stable running) and -sidecar-readyz (repeatable: URLs that must answer 2xx once before /readyz reports ready, so Substrate's wakeup probe admits traffic only when every runtime is serving). Bad -sidecar values fail at start rather than loop. ate-env manifest template gains --layer name=image@sha256:...=/mount, --sidecar and --sidecar-readyz, which become image volumes, mounts and guest flags on the template. Template derivation recognizes the guest volume by name rather than as "an image volume", so a base with layers still gets the guest added or kept correctly and environments created on demand with --image inherit the second runtime. Only a command that starts with the guest binary is re-rooted under the mount. docs/task-images/RUNTIMES.md describes the model, the flags, who can reach the second runtime from where (processes inside the environment now; outside callers still only reach port 80 through the router, which is the next piece), suspend and restart behavior, and the limits. examples/runtime-layers is a runnable demo with a static busybox as the stand-in runtime: httpd on 44772 as the sidecar under both a python and a debian task image, the second one created from an image on the layered base. The first attempt with the dynamically linked busybox failed on the debian image with a glibc version error, which is why the docs make the self-contained-layer rule and its failure signature explicit. --- README.md | 3 +- cmd/ate-env-guest/main.go | 27 +++ cmd/ate-env-guest/sidecar.go | 191 ++++++++++++++++++++++ cmd/ate-env-guest/sidecar_test.go | 142 ++++++++++++++++ cmd/ate-env/manifest.go | 97 ++++++++++- cmd/ate-env/manifest_test.go | 58 +++++++ docs/task-images/DESIGN.md | 36 +++- docs/task-images/README.md | 7 + docs/task-images/RUNTIMES.md | 125 ++++++++++++++ examples/runtime-layers/README.md | 60 +++++++ examples/runtime-layers/demo.sh | 107 ++++++++++++ internal/apiservice/imagetemplate.go | 24 ++- internal/apiservice/imagetemplate_test.go | 49 +++++- 13 files changed, 905 insertions(+), 21 deletions(-) create mode 100644 cmd/ate-env-guest/sidecar.go create mode 100644 cmd/ate-env-guest/sidecar_test.go create mode 100644 docs/task-images/RUNTIMES.md create mode 100644 examples/runtime-layers/README.md create mode 100755 examples/runtime-layers/demo.sh diff --git a/README.md b/README.md index bc12088..0ae107c 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,8 @@ ate-env manifest template --template py312 \ ``` See [docs/task-images/README.md](docs/task-images/README.md) for the full guide, including -creating environments from an image on demand, and [docs/task-images/DESIGN.md](docs/task-images/DESIGN.md) +creating environments from an image on demand, [docs/task-images/RUNTIMES.md](docs/task-images/RUNTIMES.md) +for injecting a second runtime as a layer, and [docs/task-images/DESIGN.md](docs/task-images/DESIGN.md) for the design. Then create and use an environment: diff --git a/cmd/ate-env-guest/main.go b/cmd/ate-env-guest/main.go index d6f952f..ac50196 100644 --- a/cmd/ate-env-guest/main.go +++ b/cmd/ate-env-guest/main.go @@ -15,6 +15,8 @@ // Command ate-env-guest is the daemon that runs inside a Substrate actor // and exposes command execution and filesystem access over gRPC // (ProcessService and FileSystemService) with HTTP readiness probe support. +// It can also start and supervise extra runtimes next to itself (-sidecar), +// folding their readiness into /readyz (-sidecar-readyz). package main import ( @@ -27,6 +29,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "github.com/agent-substrate/env/guest" @@ -36,8 +39,23 @@ func main() { listen := flag.String("listen", ":80", "address to serve the guest API on") logDir := flag.String("log-dir", "", "directory for process logs (defaults to /var/log/ate-jobs or temporary dir)") workspace := flag.String("workspace", "/", "workspace root directory") + var sidecars, sidecarReadyz multiFlag + flag.Var(&sidecars, "sidecar", "extra runtime to start next to the guest and keep running: an absolute binary path followed by its arguments, whitespace-separated (repeatable)") + flag.Var(&sidecarReadyz, "sidecar-readyz", "URL that must answer 2xx before /readyz reports ready, e.g. http://127.0.0.1:44772/ready (repeatable)") flag.Parse() + // Reject bad sidecar values before listening, so a misconfigured template + // fails at actor start instead of looping on a restarting sidecar. + var sidecarArgv [][]string + for _, s := range sidecars { + argv, err := parseSidecar(s) + if err != nil { + log.Fatalf("invalid -sidecar: %v", err) + } + sidecarArgv = append(sidecarArgv, argv) + } + gate := newReadyGate(sidecarReadyz) + addr := *listen if addr == "" { addr = ":80" @@ -59,6 +77,10 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + if ok, why := gate.Ready(r.Context()); !ok { + http.Error(w, why, http.StatusServiceUnavailable) + return + } io.WriteString(w, "ok\n") }) mux.Handle("/", grpcServer) @@ -90,6 +112,11 @@ func main() { _ = srv.Shutdown(context.Background()) }() + for _, argv := range sidecarArgv { + log.Printf("starting sidecar %s", strings.Join(argv, " ")) + go runSidecar(ctx, argv, log.Printf) + } + log.Printf("ate-env-guest listening on %s (gRPC services=%s, workspace=%s, logdir=%s)", lis.Addr(), guest.FormatEnabledServices(cfg), cfg.Workspace, cfg.LogDir) if err := srv.Serve(lis); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/cmd/ate-env-guest/sidecar.go b/cmd/ate-env-guest/sidecar.go new file mode 100644 index 0000000..f7d9d0e --- /dev/null +++ b/cmd/ate-env-guest/sidecar.go @@ -0,0 +1,191 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +// Sidecars are extra runtimes started next to the guest, from the same +// container: typically a second data-plane daemon that was mounted into the +// actor as an image volume (see docs/task-images/RUNTIMES.md). The guest is +// the container's only process by contract, so it is the one that starts +// them, restarts them if they exit, and folds their readiness into /readyz. + +// multiFlag collects the values of a repeatable flag. +type multiFlag []string + +func (m *multiFlag) String() string { return strings.Join(*m, "; ") } + +func (m *multiFlag) Set(v string) error { + if strings.TrimSpace(v) == "" { + return errors.New("value must not be empty") + } + *m = append(*m, v) + return nil +} + +// parseSidecar turns a -sidecar value into argv. Values are split on +// whitespace and no quoting is interpreted: sidecars run without a shell, so +// a runtime that needs one ships a wrapper script in its layer. The binary +// must be an absolute path, which is what a mounted layer provides and what +// keeps the guest independent of the task image's PATH. +func parseSidecar(v string) ([]string, error) { + argv := strings.Fields(v) + if len(argv) == 0 { + return nil, errors.New("sidecar command is empty") + } + if !filepath.IsAbs(argv[0]) { + return nil, fmt.Errorf("sidecar %q: the binary must be an absolute path (sidecars run without a shell or PATH lookup)", argv[0]) + } + return argv, nil +} + +const ( + sidecarMinBackoff = time.Second + sidecarMaxBackoff = 30 * time.Second + // sidecarStableAfter is how long a sidecar must have run for its next + // restart to start from the minimum backoff again. + sidecarStableAfter = time.Minute +) + +// runSidecar runs argv until ctx is done, restarting it whenever it exits. +// The backoff doubles from one second to thirty while the sidecar keeps +// failing quickly and resets once it has run for a while. Output goes to +// logf, one line at a time, prefixed with the binary's name. +func runSidecar(ctx context.Context, argv []string, logf func(string, ...any)) { + name := filepath.Base(argv[0]) + backoff := sidecarMinBackoff + for { + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = os.Environ() + out := &lineLogger{prefix: name, logf: logf} + cmd.Stdout = out + cmd.Stderr = out + start := time.Now() + err := cmd.Run() + out.flush() + if ctx.Err() != nil { + return + } + ran := time.Since(start) + logf("sidecar %s exited after %s: %v; restarting in %s", name, ran.Round(time.Millisecond), err, backoff) + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + if ran >= sidecarStableAfter { + backoff = sidecarMinBackoff + } else { + backoff = min(backoff*2, sidecarMaxBackoff) + } + } +} + +// lineLogger writes complete lines to logf with a prefix; a partial trailing +// line is kept until the next write or flush. +type lineLogger struct { + prefix string + logf func(string, ...any) + mu sync.Mutex + buf bytes.Buffer +} + +func (l *lineLogger) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.buf.Write(p) + for { + line, err := l.buf.ReadString('\n') + if err != nil { + // No newline yet: put the partial line back. + l.buf.Reset() + l.buf.WriteString(line) + break + } + l.logf("[%s] %s", l.prefix, strings.TrimRight(line, "\r\n")) + } + return len(p), nil +} + +func (l *lineLogger) flush() { + l.mu.Lock() + defer l.mu.Unlock() + if l.buf.Len() > 0 { + l.logf("[%s] %s", l.prefix, l.buf.String()) + l.buf.Reset() + } +} + +// readyGate makes /readyz wait for the sidecars' own readiness endpoints. +// Each URL is polled until it answers 2xx once; after that it is not asked +// again. Readiness is a start-up gate, so Substrate's wakeup probe admits +// traffic only when every runtime in the actor is serving, not a liveness +// check on the sidecars. +type readyGate struct { + urls []string + client *http.Client + + mu sync.Mutex + seen map[string]bool +} + +func newReadyGate(urls []string) *readyGate { + return &readyGate{ + urls: urls, + client: &http.Client{Timeout: 2 * time.Second}, + seen: make(map[string]bool, len(urls)), + } +} + +// Ready reports whether every URL has answered 2xx, and if not, which one is +// still pending and why. +func (g *readyGate) Ready(ctx context.Context) (bool, string) { + for _, u := range g.urls { + g.mu.Lock() + ok := g.seen[u] + g.mu.Unlock() + if ok { + continue + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return false, fmt.Sprintf("sidecar readiness %s: %v", u, err) + } + resp, err := g.client.Do(req) + if err != nil { + return false, fmt.Sprintf("sidecar readiness %s: %v", u, err) + } + resp.Body.Close() + if resp.StatusCode/100 != 2 { + return false, fmt.Sprintf("sidecar readiness %s: HTTP %d", u, resp.StatusCode) + } + g.mu.Lock() + g.seen[u] = true + g.mu.Unlock() + } + return true, "" +} diff --git a/cmd/ate-env-guest/sidecar_test.go b/cmd/ate-env-guest/sidecar_test.go new file mode 100644 index 0000000..3480d76 --- /dev/null +++ b/cmd/ate-env-guest/sidecar_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os/exec" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestParseSidecar(t *testing.T) { + argv, err := parseSidecar(" /opt/osb/execd --port 44772 ") + if err != nil || strings.Join(argv, " ") != "/opt/osb/execd --port 44772" { + t.Errorf("parseSidecar = %v, %v", argv, err) + } + for _, bad := range []string{"", " ", "execd --port 1", "sh -c /opt/osb/execd"} { + if _, err := parseSidecar(bad); err == nil { + t.Errorf("parseSidecar(%q) accepted a non-absolute or empty command", bad) + } + } + var m multiFlag + if err := m.Set(""); err == nil { + t.Error("empty flag value accepted") + } + _ = m.Set("/a") + _ = m.Set("/b x") + if len(m) != 2 || m.String() != "/a; /b x" { + t.Errorf("multiFlag = %v", m) + } +} + +func TestReadyGateWaitsOnceForEachURL(t *testing.T) { + var state atomic.Int32 // 0: 503, 1: 200 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if state.Load() == 0 { + http.Error(w, "booting", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + closed := httptest.NewServer(http.NotFoundHandler()) + closedURL := closed.URL + closed.Close() + + g := newReadyGate([]string{srv.URL + "/ready"}) + ctx := context.Background() + if ok, why := g.Ready(ctx); ok || !strings.Contains(why, "HTTP 503") { + t.Errorf("not ready expected, got ok=%v why=%q", ok, why) + } + state.Store(1) + if ok, why := g.Ready(ctx); !ok { + t.Errorf("ready expected, got %q", why) + } + // Once seen, the URL is not re-checked: readiness is a start-up gate. + state.Store(0) + if ok, _ := g.Ready(ctx); !ok { + t.Error("readiness flipped back after the sidecar had answered once") + } + + unreachable := newReadyGate([]string{closedURL + "/ready"}) + if ok, why := unreachable.Ready(ctx); ok || why == "" { + t.Errorf("an unreachable sidecar must report not ready with a reason, got ok=%v why=%q", ok, why) + } + if ok, _ := newReadyGate(nil).Ready(ctx); !ok { + t.Error("no sidecars means ready") + } +} + +func TestRunSidecarRestartsAndStopsWithContext(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh on this host") + } + var mu sync.Mutex + var lines []string + logf := func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + lines = append(lines, fmt.Sprintf(format, args...)) + } + + // A sidecar that prints and exits non-zero is restarted with backoff. + ctx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond) + defer cancel() + done := make(chan struct{}) + go func() { + runSidecar(ctx, []string{sh, "-c", "echo hello from sidecar; exit 3"}, logf) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("runSidecar did not return after its context ended") + } + mu.Lock() + joined := strings.Join(lines, "\n") + mu.Unlock() + if !strings.Contains(joined, "[sh] hello from sidecar") { + t.Errorf("sidecar output was not logged with its prefix:\n%s", joined) + } + if n := strings.Count(joined, "restarting in"); n < 2 { + t.Errorf("expected at least two restarts within the window (1s then 2s backoff), got %d:\n%s", n, joined) + } + if !strings.Contains(joined, "restarting in 1s") || !strings.Contains(joined, "restarting in 2s") { + t.Errorf("backoff did not double:\n%s", joined) + } + + // A long-running sidecar is killed promptly when the guest shuts down. + ctx2, cancel2 := context.WithCancel(context.Background()) + done2 := make(chan struct{}) + go func() { + runSidecar(ctx2, []string{sh, "-c", "sleep 60"}, logf) + close(done2) + }() + time.Sleep(200 * time.Millisecond) + cancel2() + select { + case <-done2: + case <-time.After(3 * time.Second): + t.Fatal("sidecar was not stopped when the context was cancelled") + } +} diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index b906013..08f64ea 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "strings" "github.com/agent-substrate/env/internal/apiservice" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -70,6 +71,58 @@ type templateConfig struct { taskImage string // workspace, when set, is passed to the guest as -workspace. workspace string + // layers are extra runtimes mounted as read-only image volumes, each + // "name=image@sha256:...=/mount/path". + layers []string + // sidecars are command lines the guest starts and supervises next to + // itself (absolute binary path first), one per -sidecar flag. + sidecars []string + // sidecarReadyz are URLs the guest's /readyz waits for. + sidecarReadyz []string +} + +// runtimeLayer is a parsed --layer value. +type runtimeLayer struct { + name, image, mountPath string +} + +// parseLayer parses "name=image@sha256:...=/mount/path". The image reference +// carries no '=' and the mount path is absolute, so splitting on the first two +// '=' is unambiguous. +func parseLayer(v string) (runtimeLayer, error) { + parts := strings.SplitN(v, "=", 3) + if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return runtimeLayer{}, fmt.Errorf("--layer %q: want name=image@sha256:...=/mount/path", v) + } + l := runtimeLayer{name: parts[0], image: parts[1], mountPath: parts[2]} + if l.name == apiservice.GuestVolumeName { + return runtimeLayer{}, fmt.Errorf("--layer %q: the name %q is reserved for the guest", v, apiservice.GuestVolumeName) + } + if _, err := apiservice.ImageDigest(l.image); err != nil { + return runtimeLayer{}, fmt.Errorf("--layer %q: %w", l.name, err) + } + if !strings.HasPrefix(l.mountPath, "/") || l.mountPath == apiservice.GuestMountPath { + return runtimeLayer{}, fmt.Errorf("--layer %q: mount path must be absolute and not %s", l.name, apiservice.GuestMountPath) + } + return l, nil +} + +// parseLayers parses and cross-checks every --layer value. +func parseLayers(values []string) ([]runtimeLayer, error) { + var layers []runtimeLayer + names, mounts := map[string]bool{}, map[string]bool{} + for _, v := range values { + l, err := parseLayer(v) + if err != nil { + return nil, err + } + if names[l.name] || mounts[l.mountPath] { + return nil, fmt.Errorf("--layer %q: duplicate name or mount path", v) + } + names[l.name], mounts[l.mountPath] = true, true + layers = append(layers, l) + } + return layers, nil } func (c *templateConfig) resolveImages() error { @@ -87,6 +140,20 @@ func (c *templateConfig) resolveImages() error { return fmt.Errorf("--guest-image must be digest-pinned to be mounted as an image volume: %w", err) } } + if _, err := parseLayers(c.layers); err != nil { + return err + } + for _, s := range c.sidecars { + argv := strings.Fields(s) + if len(argv) == 0 || !strings.HasPrefix(argv[0], "/") { + return fmt.Errorf("--sidecar %q: want an absolute binary path followed by its arguments", s) + } + } + for _, u := range c.sidecarReadyz { + if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") { + return fmt.Errorf("--sidecar-readyz %q: want an http(s) URL", u) + } + } return nil } @@ -164,6 +231,9 @@ It prints YAML to stdout without touching the cluster.`, cmd.Flags().StringVar(&tCfg.snapshotsBucket, "snapshots-bucket", "", "object-storage bucket (with optional prefix) for actor snapshots, e.g. gs://bucket/prefix/") cmd.Flags().StringVar(&tCfg.taskImage, "task-image", "", "digest-pinned image to run unmodified; the guest is mounted into it as a read-only image volume at "+apiservice.GuestMountPath) cmd.Flags().StringVar(&tCfg.workspace, "workspace", "", "workspace root inside the actor, passed to the guest as -workspace (default: the guest's default)") + cmd.Flags().StringArrayVar(&tCfg.layers, "layer", nil, "extra runtime mounted as a read-only image volume, as name=image@sha256:...=/mount/path (repeatable)") + cmd.Flags().StringArrayVar(&tCfg.sidecars, "sidecar", nil, "command the guest starts and keeps running next to itself, an absolute binary path plus arguments, e.g. '/opt/opensandbox/execd --port 44772' (repeatable)") + cmd.Flags().StringArrayVar(&tCfg.sidecarReadyz, "sidecar-readyz", nil, "URL the actor's readiness waits for, e.g. http://127.0.0.1:44772/ready (repeatable)") return cmd } @@ -330,10 +400,33 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { // set, that image as the container with the guest mounted into it. func buildTemplate(cfg templateConfig) (*ateapipb.ActorTemplate, error) { tmpl := buildActorTemplate(cfg) + c := tmpl.Containers[0] + command := append([]string(nil), c.Command...) if cfg.workspace != "" { - c := tmpl.Containers[0] - c.Command = append(append([]string(nil), c.Command...), "-workspace", cfg.workspace) + command = append(command, "-workspace", cfg.workspace) + } + // Extra runtimes: each layer is an image volume, each sidecar a flag the + // guest acts on. They are added before any task-image derivation so the + // derived template inherits them, and so do environments created from + // it with an image of their own. + layers, err := parseLayers(cfg.layers) + if err != nil { + return nil, err + } + for _, l := range layers { + tmpl.Volumes = append(tmpl.Volumes, &ateapipb.Volume{ + Name: l.name, + Image: &ateapipb.ImageVolumeSource{Reference: l.image}, + }) + c.VolumeMounts = append(c.VolumeMounts, &ateapipb.VolumeMount{Name: l.name, MountPath: l.mountPath}) + } + for _, s := range cfg.sidecars { + command = append(command, "-sidecar", s) + } + for _, u := range cfg.sidecarReadyz { + command = append(command, "-sidecar-readyz", u) } + c.Command = command if cfg.taskImage == "" { return tmpl, nil } diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index cca2e8b..b0f69f0 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -329,3 +329,61 @@ func TestTemplateConfigResolveImagesTaskImage(t *testing.T) { t.Errorf("pinned images rejected: %v", err) } } + +func TestBuildTemplateWithRuntimeLayer(t *testing.T) { + cfg := testTemplateConfig() + cfg.guestImage = "example.com/guest@sha256:" + testHex64 + cfg.taskImage = "docker.io/library/python@sha256:" + testHex64 + cfg.layers = []string{"execd=example.com/execd@sha256:" + testHex64 + "=/opt/opensandbox"} + cfg.sidecars = []string{"/opt/opensandbox/execd --port 44772"} + cfg.sidecarReadyz = []string{"http://127.0.0.1:44772/ready"} + + tmpl, err := buildTemplate(cfg) + if err != nil { + t.Fatalf("buildTemplate: %v", err) + } + vols := tmpl.GetVolumes() + if len(vols) != 2 || vols[0].GetName() != "execd" || vols[0].GetImage().GetReference() != "example.com/execd@sha256:"+testHex64 || vols[1].GetName() != apiservice.GuestVolumeName { + t.Errorf("volumes = %v, want the execd layer then the guest volume", vols) + } + c := tmpl.GetContainers()[0] + if len(c.GetVolumeMounts()) != 2 || c.GetVolumeMounts()[0].GetMountPath() != "/opt/opensandbox" || c.GetVolumeMounts()[1].GetMountPath() != apiservice.GuestMountPath { + t.Errorf("mounts = %v", c.GetVolumeMounts()) + } + want := "/ate/ko-app/ate-env-guest -sidecar /opt/opensandbox/execd --port 44772 -sidecar-readyz http://127.0.0.1:44772/ready" + if got := strings.Join(c.GetCommand(), " "); got != want { + t.Errorf("command = %q, want %q", got, want) + } + if c.GetImage() != cfg.taskImage { + t.Errorf("image = %q", c.GetImage()) + } +} + +func TestParseLayerRejects(t *testing.T) { + cases := map[string]string{ + "missing parts": "execd=example.com/execd@sha256:" + testHex64, + "unpinned image": "execd=example.com/execd:latest=/opt/execd", + "relative mount": "execd=example.com/execd@sha256:" + testHex64 + "=opt/execd", + "guest mount path": "execd=example.com/execd@sha256:" + testHex64 + "=/ate", + "reserved name": "guest=example.com/execd@sha256:" + testHex64 + "=/opt/execd", + } + for name, v := range cases { + if _, err := parseLayer(v); err == nil { + t.Errorf("%s: %q accepted", name, v) + } + } + dup := "a=example.com/x@sha256:" + testHex64 + "=/opt/a" + if _, err := parseLayers([]string{dup, dup}); err == nil { + t.Error("duplicate layer accepted") + } + cfg := testTemplateConfig() + cfg.sidecars = []string{"execd --port 1"} + if err := cfg.resolveImages(); err == nil { + t.Error("relative sidecar binary accepted") + } + cfg = testTemplateConfig() + cfg.sidecarReadyz = []string{"127.0.0.1:44772/ready"} + if err := cfg.resolveImages(); err == nil { + t.Error("sidecar-readyz without a scheme accepted") + } +} diff --git a/docs/task-images/DESIGN.md b/docs/task-images/DESIGN.md index b806acc..24165e4 100644 --- a/docs/task-images/DESIGN.md +++ b/docs/task-images/DESIGN.md @@ -110,6 +110,28 @@ also where `ate-env-api` already creates actors, so no new permissions are involved beyond template creation, which the API server's identity needs for this feature. +### Runtime layers + +A second runtime is expressed with the same primitives rather than a new +template concept: an extra image volume for its binary, and two guest flags. +`-sidecar` makes the guest start and supervise a process (restart with +backoff, output into the guest log), and `-sidecar-readyz` makes the guest's +`/readyz`, which Substrate uses as the wakeup probe, wait for the runtime's own +health endpoint once. The guest is the right supervisor because it is already +the container's only process and the readiness authority. + +Derivation treats layers as part of what carries over. The guest volume is +recognized by its name, not by being an image volume, so a base with a layer +still gets the guest added or kept correctly, and `create --image` on a layered +base yields environments with the second runtime present. Layers are +documented in [RUNTIMES.md](RUNTIMES.md); the verified example uses busybox as +a stand-in runtime. + +What this does not solve is reaching a second runtime from outside the actor. +The router forwards to port 80 unless a CONNECT authority names another port, +and `ate-env-api` proxies only the guest. Exposing a runtime's port is the next +piece, and it belongs with the OpenSandbox backend that needs it. + ## Failure semantics | Situation | Result | @@ -164,10 +186,10 @@ one golden snapshot per image. Neither is reclaimed automatically. ## Future work -- **Second injected runtime.** The derivation is runtime-agnostic apart from - the binary path. A runtime such as OpenSandbox's `execd` or E2B's `envd` is a - second image volume plus a command and readiness probe; the base template - describes which runtime it carries. +- **Reaching a second runtime from outside.** Layers and sidecars put a + runtime such as OpenSandbox's `execd` inside the actor and gate readiness on + it; an endpoint lookup on `ate-env-api` that names the runtime's port + through the router is what an external SDK still needs. - **Per-create overrides** for resources and workspace, so one base can serve images with different needs. - **Derived-template garbage collection**, by age or by last use, with the @@ -180,8 +202,10 @@ one golden snapshot per image. Neither is reclaimed automatically. ## Verification -- Unit tests cover derivation from both base kinds, naming and its limits, and - every rejection. +- Unit tests cover derivation from both base kinds, naming and its limits, + every rejection, and that runtime layers and sidecar flags survive + derivation. The guest's sidecar supervisor and readiness gate have their own + tests (restart with backoff, stop on shutdown, gate once per URL). - Server tests against the in-process fake control plane cover first create, reuse, a named base in another atespace, and the failure table. - `clients/python/tests/e2e/test_full_stack.py` has a create-from-image test diff --git a/docs/task-images/README.md b/docs/task-images/README.md index 6baecb6..3af73ac 100644 --- a/docs/task-images/README.md +++ b/docs/task-images/README.md @@ -112,6 +112,13 @@ kubectl-ate get actor-template --atespace ate-env A fresh template shows `Failed` in that listing for a few seconds while its golden bakes, then `Ready`. Environments can be created in the meantime. +## A second runtime + +The same mechanism carries more runtimes. `--layer` mounts another image, +`--sidecar` has the guest start a process from it, and `--sidecar-readyz` +folds its readiness into the actor's. See [RUNTIMES.md](RUNTIMES.md) and the +runnable example in [`examples/runtime-layers`](../../examples/runtime-layers). + ## Errors | Error | Meaning | diff --git a/docs/task-images/RUNTIMES.md b/docs/task-images/RUNTIMES.md new file mode 100644 index 0000000..25ab84e --- /dev/null +++ b/docs/task-images/RUNTIMES.md @@ -0,0 +1,125 @@ +# Injecting a second runtime as a layer + +[README.md](README.md) explains how the ate-env guest is injected into an +unmodified task image. The same mechanism carries any number of further +runtimes: a second daemon that an agent framework expects inside its sandbox, +such as OpenSandbox's `execd`, is one more read-only image volume plus a process +the guest starts next to itself. This page explains the model, the flags, and +what reaches the second runtime from where. + +## Model + +A runtime layer is three things: + +| part | where it is expressed | what it does | +|---|---|---| +| the binary | an image volume on the ActorTemplate (`--layer name=image@sha256:...=/mount`) | mounts the runtime's image read-only at a path of your choice, next to the guest at `/ate` | +| the process | a guest flag (`--sidecar '/mount/binary args...'`) | the guest starts it after it is listening, logs its output with a prefix, and restarts it with backoff if it exits | +| its readiness | a guest flag (`--sidecar-readyz http://127.0.0.1:PORT/health`) | the guest's `/readyz`, which is the actor's wakeup probe, answers 503 until every such URL has answered 2xx once | + +The guest is the container's only process by contract, which is why it is the +one that supervises the others. Sidecars run without a shell or `PATH` lookup, +so the binary is an absolute path into its mount, and the task image needs +nothing for it. They inherit the container's user, environment and filesystem. + +Template derivation keeps all of it. A base with layers yields derived +templates with the same layers and the same guest flags, so environments +created on demand with `image=` inherit the second runtime without any caller +knowing it exists. + +## Registering a layered template + +```bash +ate-env manifest template --template osb-base \ + --task-image docker.io/library/python@sha256: \ + --guest-image /ate-env-guest@sha256: \ + --layer execd=/opensandbox/execd@sha256:=/opt/opensandbox \ + --sidecar '/opt/opensandbox/execd' \ + --sidecar-readyz http://127.0.0.1:44772/ready \ + --snapshots-bucket | kubectl-ate create actor-template -f - +``` + +The resulting template has two image volumes (`execd` at `/opt/opensandbox`, +`guest` at `/ate`), the task image as the container, and the command + +``` +/ate/ko-app/ate-env-guest -sidecar "/opt/opensandbox/execd" -sidecar-readyz http://127.0.0.1:44772/ready +``` + +Everything else is as in a plain task-image template. Environments are created +the usual way, from this template by name or from any other image with `--image` +while naming it as the base: + +```bash +ate-env create py1 --template osb-base +ate-env create node1 --template osb-base --image docker.io/library/node@sha256: +``` + +Both run the guest and `execd`; the second one from a derived template named +`osb-base-<12 hex of the node digest>`. + +### OpenSandbox `execd` specifics + +`execd` is a static binary that OpenSandbox injects into sandboxes without +modifying the base image, which is exactly the shape this page describes. Its +image carries the binary at `/execd`; mounting the image at `/opt/opensandbox` +puts it at `/opt/opensandbox/execd`, OpenSandbox's own default install path. It +serves HTTP on port 44772 by default, with `/ping` and `/ready` open and every +other endpoint behind the `X-EXECD-ACCESS-TOKEN` header when a token is +configured. Check the OpenSandbox documentation for the current flags and +environment variables for the port, token and workspace; pass them in the +`--sidecar` value, or wrap them in a script shipped in the layer if quoting is +needed. Its optional Jupyter code-interpreter mode starts a Jupyter server +inside the sandbox; for fleets of small environments use the plain command and +file endpoints. + +A verified stand-in for the mechanics, which needs nothing from any registry +but Docker Hub, is in [`examples/runtime-layers`](../../examples/runtime-layers): +the static `busybox:1.37-musl` mounted as the layer and its `httpd` as the +sidecar, run under two different task images. + +## Who can reach the second runtime + +- **Processes inside the environment** reach it at `127.0.0.1:` right + away. An agent framework whose tool calls run inside the sandbox is served. +- **The guest's own APIs** are unaffected: exec, files and MCP keep going + through `ate-env-api` on port 80. +- **Clients outside the environment** currently reach only port 80. The atenet + router forwards to the actor's port 80 unless the request is a CONNECT whose + authority names another port, and `ate-env-api` proxies only the guest. An + SDK that speaks HTTP to the second runtime therefore needs one of two things, + neither of which is in this branch: an endpoint lookup on `ate-env-api` that + returns a router address naming the runtime's port, or a reverse proxy. This + is the next step for the OpenSandbox integration, whose lifecycle server + resolves sandbox endpoints exactly this way. + +## Suspend, resume and restarts + +A sidecar is part of the actor, so a suspend checkpoints its memory with the +guest's and a resume brings it back without a restart, open sockets included. If +a runtime does not survive that (it polls a clock it can no longer trust, say) +and exits, the guest restarts it: after one second, doubling up to thirty while +it keeps failing, and from one second again once it has run for a minute. +Readiness is gated once, at start; a sidecar that dies later is restarted but +does not take the actor out of service. + +## Limits + +- **A layer must be self-contained.** The task image's libraries are whatever + that image ships, so a runtime binary that is dynamically linked works on + some images and not on others. Ship static binaries (the guest and `execd` + are) or put the runtime's libraries in the layer and point it at them. The + failure is easy to recognize: the guest log shows the sidecar exiting at + once with a loader error such as ``version `GLIBC_2.38' not found`` and + restarting with growing backoff, readiness never passes, and the actor + fails after the wakeup probe timeout. +- One container per actor: sidecars share the task image's user, environment + variables and filesystem, and have no resource limits of their own. +- `--sidecar` values are split on whitespace with no quoting. Arguments that + need quotes or shell features go in a wrapper script inside the layer. +- Port 80 belongs to the guest. A runtime that insists on it needs a wrapper + that moves it elsewhere. +- Sidecar output goes to the guest's log, line by line, prefixed with the + binary's name. There is no separate log stream yet. +- Layers are read-only. A runtime that writes next to its binary needs a + writable working directory passed as an argument. diff --git a/examples/runtime-layers/README.md b/examples/runtime-layers/README.md new file mode 100644 index 0000000..7687433 --- /dev/null +++ b/examples/runtime-layers/README.md @@ -0,0 +1,60 @@ +# Example: a second runtime as a layer + +Runs an unmodified task image with two injected runtimes: the ate-env guest at +`/ate`, and a static busybox at `/opt/web` standing in for a real runtime +daemon such as OpenSandbox's `execd`. The guest starts `busybox httpd` on port 44772 as a +sidecar and holds the actor's readiness until it answers. The mechanics are the +ones [docs/task-images/RUNTIMES.md](../../docs/task-images/RUNTIMES.md) +describes; only the binary differs. + +`demo.sh` registers the template, creates an environment, fetches a file from +the second runtime from inside the environment, optionally creates another +environment from a different image on the same base to show the layer is +inherited, and deletes what it created. + +```bash +kubectl -n ate-env port-forward svc/ate-env-api 7777:7777 & + +GUEST_IMAGE=/ate-env-guest@sha256: \ +SNAPSHOTS_BUCKET= \ +TASK_IMAGE=docker.io/library/python@sha256: \ +LAYER_IMAGE=docker.io/library/busybox@sha256: \ +OTHER_IMAGE=docker.io/library/node@sha256: \ +examples/runtime-layers/demo.sh +``` + +Digests for public images come from `crane digest --platform linux/amd64 +python:3.12-slim` or `docker manifest inspect`. The guest image must be built +from a revision of this repo that has the `-sidecar` flags. + +Use the `busybox:1.37-musl` tag, which is statically linked. The default +`busybox:1.37` is built against glibc and only runs on task images whose glibc +is at least as new as its own: it works inside `python:3.12-slim` and fails +inside `debian:bookworm-slim` with ``GLIBC_2.38' not found``, which is the +textbook case of why a runtime layer has to be self-contained. + +What the template looks like, as `ate-env manifest template` prints it: + +```yaml +containers: +- command: + - /ate/ko-app/ate-env-guest + - -sidecar + - /opt/web/bin/busybox httpd -f -p 44772 -h / + - -sidecar-readyz + - http://127.0.0.1:44772/etc/hostname + image: docker.io/library/python@sha256: + name: guest + readyz: {httpGet: {path: /readyz, port: 80}} + volumeMounts: + - {name: web, mountPath: /opt/web} + - {name: guest, mountPath: /ate} +volumes: +- {name: web, image: {reference: docker.io/library/busybox@sha256:}} +- {name: guest, image: {reference: /ate-env-guest@sha256:}} +``` + +To swap in a real runtime, change the three values: the layer image, the +sidecar command, and the readiness URL. For `execd` that is its image mounted +at `/opt/opensandbox`, `/opt/opensandbox/execd` with its flags, and +`http://127.0.0.1:44772/ready`. diff --git a/examples/runtime-layers/demo.sh b/examples/runtime-layers/demo.sh new file mode 100755 index 0000000..a080475 --- /dev/null +++ b/examples/runtime-layers/demo.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A second runtime injected as a layer, end to end. +# +# busybox stands in for a real runtime daemon (OpenSandbox execd, say): its +# image is mounted read-only at /opt/web, the guest starts `busybox httpd` on +# port 44772 as a sidecar and waits for it before reporting the actor ready. +# The task image is unmodified. The script registers the template, creates an +# environment from it, asks the second runtime for a file from inside the +# environment, creates a second environment from a different image on the +# same base to show the layer is inherited, and deletes both. +# +# Required: +# GUEST_IMAGE digest-pinned ate-env-guest image (built from this repo, +# at or after the -sidecar flags) +# SNAPSHOTS_BUCKET object-storage URL for actor snapshots +# TASK_IMAGE digest-pinned task image, e.g. docker.io/library/python@sha256:... +# LAYER_IMAGE digest-pinned STATIC busybox image (the 1.37-musl tag), +# e.g. docker.io/library/busybox@sha256:... +# Optional: +# OTHER_IMAGE a second digest-pinned image for the inheritance check +# TEMPLATE base template name (default layered-demo) +# ATESPACE atespace (default ate-env) +# SUBSTRATE_ENV_API ate-env-api address (default 127.0.0.1:7777; port-forward it) +# ATE_ENV, KUBECTL_ATE binaries (default: on PATH) +set -euo pipefail + +: "${GUEST_IMAGE:?set GUEST_IMAGE to the digest-pinned ate-env-guest image}" +: "${SNAPSHOTS_BUCKET:?set SNAPSHOTS_BUCKET to the snapshot object-storage URL}" +: "${TASK_IMAGE:?set TASK_IMAGE to a digest-pinned task image}" +: "${LAYER_IMAGE:?set LAYER_IMAGE to a digest-pinned busybox image}" +TEMPLATE=${TEMPLATE:-layered-demo} +ATESPACE=${ATESPACE:-ate-env} +export SUBSTRATE_ENV_API=${SUBSTRATE_ENV_API:-127.0.0.1:7777} +ATE_ENV=${ATE_ENV:-ate-env} +KUBECTL_ATE=${KUBECTL_ATE:-kubectl-ate} + +PORT=44772 +SIDECAR="/opt/web/bin/busybox httpd -f -p ${PORT} -h /" +READYZ="http://127.0.0.1:${PORT}/etc/hostname" + +step() { printf '\n== %s\n' "$*"; } + +step "register ${ATESPACE}/${TEMPLATE}: ${TASK_IMAGE} + guest + busybox layer" +"${ATE_ENV}" manifest template --template "${TEMPLATE}" --atespace "${ATESPACE}" \ + --task-image "${TASK_IMAGE}" --guest-image "${GUEST_IMAGE}" \ + --snapshots-bucket "${SNAPSHOTS_BUCKET}" \ + --layer "web=${LAYER_IMAGE}=/opt/web" \ + --sidecar "${SIDECAR}" \ + --sidecar-readyz "${READYZ}" | "${KUBECTL_ATE}" create actor-template -f - >/dev/null +"${KUBECTL_ATE}" get actor-template --atespace "${ATESPACE}" "${TEMPLATE}" + +ids=() +cleanup() { + for id in "${ids[@]:-}"; do + [ -n "${id}" ] && "${ATE_ENV}" delete "${id}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +# wait_serving : the guest answers only once the sidecar's readiness URL +# has answered, so the first successful shell means both runtimes are up. +wait_serving() { + local id=$1 i + for i in $(seq 1 60); do + if "${ATE_ENV}" "${id}" shell 'true' >/dev/null 2>&1; then return 0; fi + sleep 3 + done + echo "environment ${id} did not start serving" >&2 + return 1 +} + +ID="layered-$(date +%s)" +ids+=("${ID}") +step "create ${ID} from ${TEMPLATE}" +t0=$(date +%s) +"${ATE_ENV}" create "${ID}" --atespace "${ATESPACE}" --template "${TEMPLATE}" +wait_serving "${ID}" +echo "serving after $(( $(date +%s) - t0 ))s" + +step "ask the second runtime from inside the environment" +"${ATE_ENV}" "${ID}" shell "/opt/web/bin/busybox wget -qO- http://127.0.0.1:${PORT}/etc/os-release | head -2; echo guest: \$(ls /ate/ko-app); echo layer: \$(ls /opt/web/bin | head -1)" + +if [ -n "${OTHER_IMAGE:-}" ]; then + ID2="layered2-$(date +%s)" + ids+=("${ID2}") + step "create ${ID2} from ${OTHER_IMAGE} with ${TEMPLATE} as the base (layer inherited)" + "${ATE_ENV}" create "${ID2}" --atespace "${ATESPACE}" --template "${TEMPLATE}" --image "${OTHER_IMAGE}" + "${KUBECTL_ATE}" get actor-template --atespace "${ATESPACE}" | grep "${TEMPLATE}-" || true + wait_serving "${ID2}" + "${ATE_ENV}" "${ID2}" shell "/opt/web/bin/busybox wget -qO- http://127.0.0.1:${PORT}/etc/os-release | head -1" +fi + +step "done; deleting" diff --git a/internal/apiservice/imagetemplate.go b/internal/apiservice/imagetemplate.go index 706b8de..f986ed4 100644 --- a/internal/apiservice/imagetemplate.go +++ b/internal/apiservice/imagetemplate.go @@ -36,6 +36,10 @@ const GuestVolumeName = "guest" // defaultGuestBinary is the guest's path inside its own image (ko layout). const defaultGuestBinary = "/ko-app/ate-env-guest" +// guestBinaryName is what a re-rootable command must start with: only the +// guest's own path moves under the mount, never a shell or another binary. +const guestBinaryName = "ate-env-guest" + // imageTemplateDigestLen is how many hex digits of the digest go into a // derived template's name: enough to make collisions a non-concern and short // enough to keep the name a valid k8s short name next to any base name. @@ -85,9 +89,11 @@ func ImageTemplateName(base, image string) (string, error) { // base can be either kind of template: one whose container image is the // guest itself (the shape `ate-env manifest template` writes), in which case // that image becomes the guest volume and the command is re-rooted under the -// mount; or one that already mounts the guest as an image volume, in which -// case only the container image changes. Everything else (worker selector, -// snapshots, sandbox config, resources, env, readiness) carries over. +// mount; or one that already mounts the guest as an image volume (named +// GuestVolumeName), in which case only the container image changes. +// Everything else carries over: worker selector, snapshots, sandbox config, +// resources, env, readiness, and any other volumes, such as extra runtime +// layers, together with the command-line flags that start them. func DeriveImageTemplate(base *ateapipb.ActorTemplate, name, image string) (*ateapipb.ActorTemplate, error) { if base == nil { return nil, errors.New("base template is required") @@ -135,10 +141,12 @@ func DeriveImageTemplate(base *ateapipb.ActorTemplate, name, image string) (*ate return tmpl, nil } -// guestVolume returns the template's image volume, or nil when it has none. +// guestVolume returns the template's guest image volume, or nil when the +// guest is the container image. Other image volumes (extra runtime layers) +// do not count: they can be present in either kind of base. func guestVolume(tmpl *ateapipb.ActorTemplate) *ateapipb.Volume { for _, v := range tmpl.GetVolumes() { - if v.GetImage() != nil { + if v.GetName() == GuestVolumeName && v.GetImage() != nil { return v } } @@ -154,9 +162,9 @@ func rerootCommand(command []string) ([]string, error) { if len(command) == 0 { return []string{path.Join(GuestMountPath, defaultGuestBinary)}, nil } - if !strings.HasPrefix(command[0], "/") { - return nil, fmt.Errorf("cannot re-root command %q under %s: it must start with an absolute path to the guest binary", - strings.Join(command, " "), GuestMountPath) + if !strings.HasPrefix(command[0], "/") || path.Base(command[0]) != guestBinaryName { + return nil, fmt.Errorf("cannot re-root command %q under %s: it must start with the absolute path of the guest binary (.../%s)", + strings.Join(command, " "), GuestMountPath, guestBinaryName) } out := append([]string(nil), command...) out[0] = path.Join(GuestMountPath, out[0]) diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go index 9190eb4..06621cc 100644 --- a/internal/apiservice/imagetemplate_test.go +++ b/internal/apiservice/imagetemplate_test.go @@ -164,9 +164,50 @@ func TestDeriveImageTemplateRejects(t *testing.T) { if _, err := DeriveImageTemplate(modeABase(), "", testTaskImage); err == nil { t.Error("empty name accepted") } - wrapped := modeABase() - wrapped.Containers[0].Command = []string{"sh", "-c", "exec /ko-app/ate-env-guest"} - if _, err := DeriveImageTemplate(wrapped, "x", testTaskImage); err == nil || !strings.Contains(err.Error(), "re-root") { - t.Errorf("a shell-wrapped base command must be rejected, got %v", err) + for _, cmd := range [][]string{ + {"sh", "-c", "exec /ko-app/ate-env-guest"}, + {"/bin/sh", "-c", "exec /ko-app/ate-env-guest"}, + {"/opt/other/daemon"}, + } { + wrapped := modeABase() + wrapped.Containers[0].Command = cmd + if _, err := DeriveImageTemplate(wrapped, "x", testTaskImage); err == nil || !strings.Contains(err.Error(), "re-root") { + t.Errorf("command %v must be rejected (only the guest binary is re-rooted), got %v", cmd, err) + } + } +} + +func TestDeriveImageTemplateKeepsRuntimeLayers(t *testing.T) { + // A guest-image base that also carries a second runtime as an image + // volume and starts it as a guest sidecar: the layer and the flags + // survive derivation, and the guest volume is still added. + layerImage := "example.com/execd@sha256:" + strings.Repeat("c", 64) + base := modeABase() + base.Volumes = []*ateapipb.Volume{{Name: "execd", Image: &ateapipb.ImageVolumeSource{Reference: layerImage}}} + base.Containers[0].VolumeMounts = []*ateapipb.VolumeMount{{Name: "execd", MountPath: "/opt/opensandbox"}} + base.Containers[0].Command = []string{"/ko-app/ate-env-guest", "-sidecar", "/opt/opensandbox/execd --port 44772", "-sidecar-readyz", "http://127.0.0.1:44772/ready"} + + got, err := DeriveImageTemplate(base, "x", testTaskImage) + if err != nil { + t.Fatal(err) + } + if len(got.GetVolumes()) != 2 || got.GetVolumes()[0].GetName() != "execd" || got.GetVolumes()[1].GetName() != GuestVolumeName { + t.Errorf("volumes = %v, want the layer followed by the guest volume", got.GetVolumes()) + } + c := got.GetContainers()[0] + if len(c.GetVolumeMounts()) != 2 || c.GetVolumeMounts()[1].GetMountPath() != GuestMountPath { + t.Errorf("mounts = %v", c.GetVolumeMounts()) + } + want := "/ate/ko-app/ate-env-guest -sidecar /opt/opensandbox/execd --port 44772 -sidecar-readyz http://127.0.0.1:44772/ready" + if got := strings.Join(c.GetCommand(), " "); got != want { + t.Errorf("command = %q, want %q", got, want) + } + // Deriving again from the result (an already-injected base) changes only the image. + again, err := DeriveImageTemplate(got, "y", "example.com/other@sha256:"+strings.Repeat("d", 64)) + if err != nil { + t.Fatal(err) + } + if len(again.GetVolumes()) != 2 || strings.Join(again.GetContainers()[0].GetCommand(), " ") != want { + t.Errorf("second derivation altered layers or command: %v", again) } } From 92b27f9031190694e12ab61af82ae88f0d33990d Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:23:34 -0700 Subject: [PATCH 3/4] Address review: document CreateEnvironment defaults and prioritize derived-template GC Spell out on CreateEnvironment how id, template and atespace default, that one atespace resolves the template and places the actor, how an image request derives and reuses its template, and which inputs yield InvalidArgument versus FailedPrecondition. Move derived-template garbage collection to the top of the design's future work and record why it is not tied to environment deletion: the template is shared by every environment on the image and holds the golden snapshot. --- docs/task-images/DESIGN.md | 9 +++++++-- internal/apiservice/server.go | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/task-images/DESIGN.md b/docs/task-images/DESIGN.md index 24165e4..b706cb6 100644 --- a/docs/task-images/DESIGN.md +++ b/docs/task-images/DESIGN.md @@ -186,14 +186,19 @@ one golden snapshot per image. Neither is reclaimed automatically. ## Future work +- **Derived-template garbage collection** is the first follow-up. A derived + template is shared by every environment on its image and carries the + golden snapshot, so deleting it when one environment is deleted would send + the next create on that image back through the cold path; the count grows + with distinct images, not with environments. The plan is a last-used + annotation updated on each create, collection by age or by last use, and + the golden snapshot removed alongside. - **Reaching a second runtime from outside.** Layers and sidecars put a runtime such as OpenSandbox's `execd` inside the actor and gate readiness on it; an endpoint lookup on `ate-env-api` that names the runtime's port through the router is what an external SDK still needs. - **Per-create overrides** for resources and workspace, so one base can serve images with different needs. -- **Derived-template garbage collection**, by age or by last use, with the - golden snapshot removed alongside. - **Image validation at create time**, so a bad reference fails the call instead of the actor. - **Template atespace.** `CreateEnvironment` takes `template.atespace` but the diff --git a/internal/apiservice/server.go b/internal/apiservice/server.go index 6c735af..6ddae6c 100644 --- a/internal/apiservice/server.go +++ b/internal/apiservice/server.go @@ -79,6 +79,30 @@ func (s *Server) Close() {} // ============================================================================ // CreateEnvironment registers and starts a new environment. +// +// Defaults: id is required. The template name defaults to DefaultTemplate. +// The atespace is the request's atespace, else template.atespace, else +// DefaultAtespace, and that one atespace is used both to resolve the template +// and to create the actor; a template.atespace that differs from a non-empty +// request atespace is ignored. +// +// When image is set, the template acts as the base and the environment is +// created from a template derived from it, named +// "-" in the same atespace. +// The derived template keeps the base's settings, runs image as the +// container image and mounts the base's guest as a read-only image volume, +// so the image runs unmodified (see ensureImageTemplate and +// DeriveImageTemplate). It is created on first use and reused by later +// creates on the same image, and the response reports it in +// Environment.template. When image is empty the template is used as is. +// +// Errors: InvalidArgument for a missing id, an image not pinned by digest, a +// derived name that is too long or not a valid resource name, or a base that +// cannot be derived from (no containers, unpinned guest image, or a command +// that cannot be re-rooted under the guest volume). FailedPrecondition when +// the base template does not exist in the atespace, or when the derived name +// is already held by a template running a different image. Other control +// plane failures are mapped by toGRPCError. func (s *Server) CreateEnvironment(ctx context.Context, req *ateenvv1alpha.CreateEnvironmentRequest) (*ateenvv1alpha.CreateEnvironmentResponse, error) { if req.GetId() == "" { return nil, status.Error(codes.InvalidArgument, "id is required") From e9fdc59ae5441ccb263ff42d378635c91726a41a Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:34:57 -0700 Subject: [PATCH 4/4] Rename CreateEnvironmentRequest.image to task_image Review asked for a less ambiguous name now that runtime layers are also images on the template. The field is new in this branch, so the rename is free: proto field task_image (same number), Go GetTaskImage, CLI `ate-env create --task-image` to match `manifest template --task-image`, Python Client.create(task_image=), and the docs and examples. Generated Go and Python stubs regenerated; the Go file keeps main's descriptor path. --- README.md | 4 +- clients/python/README.md | 2 +- .../ate_env/_gen/ateenv/v1alpha/env_pb2.py | 40 +++++++++---------- .../ate_env/_gen/ateenv/v1alpha/env_pb2.pyi | 8 ++-- clients/python/src/ate_env/client.py | 6 +-- clients/python/tests/e2e/test_full_stack.py | 8 ++-- clients/python/tests/fakes.py | 6 +-- clients/python/tests/test_client.py | 8 ++-- cmd/ate-env/main.go | 10 ++--- cmd/ate-env/main_test.go | 12 +++--- docs/task-images/DESIGN.md | 8 ++-- docs/task-images/README.md | 4 +- docs/task-images/RUNTIMES.md | 6 +-- examples/runtime-layers/demo.sh | 2 +- internal/apiservice/server.go | 12 +++--- internal/apiservice/server_test.go | 18 ++++----- proto/ateenv/v1alpha/env.pb.go | 34 ++++++++-------- proto/ateenv/v1alpha/env.proto | 21 +++++----- 18 files changed, 106 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 0ae107c..2a7543b 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ ate-env create dev1 # Or run any digest-pinned image unmodified. ate-env-api derives a template # from default-template on first use (guest mounted in as an image volume) # and reuses it for later environments on the same image. -ate-env create py1 --image docker.io/library/python@sha256: +ate-env create py1 --task-image docker.io/library/python@sha256: # Execute a shell command inside the environment. ate-env dev1 shell 'echo hello > /note.txt' @@ -177,7 +177,7 @@ Manages the lifecycle of isolated execution environments (defined in [`proto/ate | RPC | Description | | --- | ----------- | -| `CreateEnvironment` | Creates and starts a new environment actor from an ActorTemplate, or from a digest-pinned `image` on top of one: the template becomes the base, the image the container, and the guest is mounted in as a read-only image volume | +| `CreateEnvironment` | Creates and starts a new environment actor from an ActorTemplate, or from a digest-pinned `task_image` on top of one: the template becomes the base, the image the container, and the guest is mounted in as a read-only image volume | | `GetEnvironment` | Retrieves environment details and status | | `SuspendEnvironment` | Suspends and checkpoints the environment to snapshot storage | | `DeleteEnvironment` | Deletes the environment permanently | diff --git a/clients/python/README.md b/clients/python/README.md index 0b2c481..8953c8d 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -151,7 +151,7 @@ read-only image volume, so the image runs unmodified. The derived template is created on first use and shared by every environment on that image: ```python -env = await client.create("py1", image="docker.io/library/python@sha256:…") +env = await client.create("py1", task_image="docker.io/library/python@sha256:…") print((await env.info()).template.name) # default-template-<12 hex of the digest> ``` diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py index 0ac59a4..ea3a5ca 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py @@ -38,7 +38,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x61teenv/v1alpha/env.proto\x12\x0e\x61teenv.v1alpha\"*\n\x08Template\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x8a\x01\n\x0b\x45nvironment\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x31\n\x06status\x18\x04 \x01(\x0e\x32!.ateenv.v1alpha.EnvironmentStatus\"s\n\x18\x43reateEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\r\n\x05image\x18\x04 \x01(\t\"M\n\x19\x43reateEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"5\n\x15GetEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"J\n\x16GetEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"9\n\x19SuspendEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1c\n\x1aSuspendEnvironmentResponse\"8\n\x18\x44\x65leteEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1b\n\x19\x44\x65leteEnvironmentResponse*\xbd\x02\n\x11\x45nvironmentStatus\x12\"\n\x1e\x45NVIRONMENT_STATUS_UNSPECIFIED\x10\x00\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_RESUMING\x10\x01\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_RUNNING\x10\x02\x12!\n\x1d\x45NVIRONMENT_STATUS_SUSPENDING\x10\x03\x12 \n\x1c\x45NVIRONMENT_STATUS_SUSPENDED\x10\x04\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_PAUSING\x10\x05\x12\x1d\n\x19\x45NVIRONMENT_STATUS_PAUSED\x10\x06\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_CRASHED\x10\x07\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_DELETING\x10\x08\x32\xb6\x03\n\x12\x45nvironmentService\x12h\n\x11\x43reateEnvironment\x12(.ateenv.v1alpha.CreateEnvironmentRequest\x1a).ateenv.v1alpha.CreateEnvironmentResponse\x12_\n\x0eGetEnvironment\x12%.ateenv.v1alpha.GetEnvironmentRequest\x1a&.ateenv.v1alpha.GetEnvironmentResponse\x12k\n\x12SuspendEnvironment\x12).ateenv.v1alpha.SuspendEnvironmentRequest\x1a*.ateenv.v1alpha.SuspendEnvironmentResponse\x12h\n\x11\x44\x65leteEnvironment\x12(.ateenv.v1alpha.DeleteEnvironmentRequest\x1a).ateenv.v1alpha.DeleteEnvironmentResponseBCZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alphab\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x61teenv/v1alpha/env.proto\x12\x0e\x61teenv.v1alpha\"*\n\x08Template\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x8a\x01\n\x0b\x45nvironment\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x31\n\x06status\x18\x04 \x01(\x0e\x32!.ateenv.v1alpha.EnvironmentStatus\"x\n\x18\x43reateEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x12\n\ntask_image\x18\x04 \x01(\t\"M\n\x19\x43reateEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"5\n\x15GetEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"J\n\x16GetEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"9\n\x19SuspendEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1c\n\x1aSuspendEnvironmentResponse\"8\n\x18\x44\x65leteEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1b\n\x19\x44\x65leteEnvironmentResponse*\xbd\x02\n\x11\x45nvironmentStatus\x12\"\n\x1e\x45NVIRONMENT_STATUS_UNSPECIFIED\x10\x00\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_RESUMING\x10\x01\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_RUNNING\x10\x02\x12!\n\x1d\x45NVIRONMENT_STATUS_SUSPENDING\x10\x03\x12 \n\x1c\x45NVIRONMENT_STATUS_SUSPENDED\x10\x04\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_PAUSING\x10\x05\x12\x1d\n\x19\x45NVIRONMENT_STATUS_PAUSED\x10\x06\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_CRASHED\x10\x07\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_DELETING\x10\x08\x32\xb6\x03\n\x12\x45nvironmentService\x12h\n\x11\x43reateEnvironment\x12(.ateenv.v1alpha.CreateEnvironmentRequest\x1a).ateenv.v1alpha.CreateEnvironmentResponse\x12_\n\x0eGetEnvironment\x12%.ateenv.v1alpha.GetEnvironmentRequest\x1a&.ateenv.v1alpha.GetEnvironmentResponse\x12k\n\x12SuspendEnvironment\x12).ateenv.v1alpha.SuspendEnvironmentRequest\x1a*.ateenv.v1alpha.SuspendEnvironmentResponse\x12h\n\x11\x44\x65leteEnvironment\x12(.ateenv.v1alpha.DeleteEnvironmentRequest\x1a).ateenv.v1alpha.DeleteEnvironmentResponseBCZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alphab\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -46,28 +46,28 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'ZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alpha' - _globals['_ENVIRONMENTSTATUS']._serialized_start=733 - _globals['_ENVIRONMENTSTATUS']._serialized_end=1050 + _globals['_ENVIRONMENTSTATUS']._serialized_start=738 + _globals['_ENVIRONMENTSTATUS']._serialized_end=1055 _globals['_TEMPLATE']._serialized_start=44 _globals['_TEMPLATE']._serialized_end=86 _globals['_ENVIRONMENT']._serialized_start=89 _globals['_ENVIRONMENT']._serialized_end=227 _globals['_CREATEENVIRONMENTREQUEST']._serialized_start=229 - _globals['_CREATEENVIRONMENTREQUEST']._serialized_end=344 - _globals['_CREATEENVIRONMENTRESPONSE']._serialized_start=346 - _globals['_CREATEENVIRONMENTRESPONSE']._serialized_end=423 - _globals['_GETENVIRONMENTREQUEST']._serialized_start=425 - _globals['_GETENVIRONMENTREQUEST']._serialized_end=478 - _globals['_GETENVIRONMENTRESPONSE']._serialized_start=480 - _globals['_GETENVIRONMENTRESPONSE']._serialized_end=554 - _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_start=556 - _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_end=613 - _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_start=615 - _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_end=643 - _globals['_DELETEENVIRONMENTREQUEST']._serialized_start=645 - _globals['_DELETEENVIRONMENTREQUEST']._serialized_end=701 - _globals['_DELETEENVIRONMENTRESPONSE']._serialized_start=703 - _globals['_DELETEENVIRONMENTRESPONSE']._serialized_end=730 - _globals['_ENVIRONMENTSERVICE']._serialized_start=1053 - _globals['_ENVIRONMENTSERVICE']._serialized_end=1491 + _globals['_CREATEENVIRONMENTREQUEST']._serialized_end=349 + _globals['_CREATEENVIRONMENTRESPONSE']._serialized_start=351 + _globals['_CREATEENVIRONMENTRESPONSE']._serialized_end=428 + _globals['_GETENVIRONMENTREQUEST']._serialized_start=430 + _globals['_GETENVIRONMENTREQUEST']._serialized_end=483 + _globals['_GETENVIRONMENTRESPONSE']._serialized_start=485 + _globals['_GETENVIRONMENTRESPONSE']._serialized_end=559 + _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_start=561 + _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_end=618 + _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_start=620 + _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_end=648 + _globals['_DELETEENVIRONMENTREQUEST']._serialized_start=650 + _globals['_DELETEENVIRONMENTREQUEST']._serialized_end=706 + _globals['_DELETEENVIRONMENTRESPONSE']._serialized_start=708 + _globals['_DELETEENVIRONMENTRESPONSE']._serialized_end=735 + _globals['_ENVIRONMENTSERVICE']._serialized_start=1058 + _globals['_ENVIRONMENTSERVICE']._serialized_end=1496 # @@protoc_insertion_point(module_scope) diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi index 63776c4..0d44c61 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi @@ -62,16 +62,16 @@ class Environment(_message.Message): def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., status: _Optional[_Union[EnvironmentStatus, str]] = ...) -> None: ... class CreateEnvironmentRequest(_message.Message): - __slots__ = ("id", "atespace", "template", "image") + __slots__ = ("id", "atespace", "template", "task_image") ID_FIELD_NUMBER: _ClassVar[int] ATESPACE_FIELD_NUMBER: _ClassVar[int] TEMPLATE_FIELD_NUMBER: _ClassVar[int] - IMAGE_FIELD_NUMBER: _ClassVar[int] + TASK_IMAGE_FIELD_NUMBER: _ClassVar[int] id: str atespace: str template: Template - image: str - def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., image: _Optional[str] = ...) -> None: ... + task_image: str + def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., task_image: _Optional[str] = ...) -> None: ... class CreateEnvironmentResponse(_message.Message): __slots__ = ("environment",) diff --git a/clients/python/src/ate_env/client.py b/clients/python/src/ate_env/client.py index 4804447..f5cd7f7 100644 --- a/clients/python/src/ate_env/client.py +++ b/clients/python/src/ate_env/client.py @@ -88,21 +88,21 @@ async def create( atespace: str = DEFAULT_ATESPACE, template_name: str | None = None, template_atespace: str | None = None, - image: str | None = None, + task_image: str | None = None, ) -> Env: """Register and start a new environment; returns a handle to it. The server fills defaults for the template (name "default-template" in atespace "ate-env") when none is given. - With image (a digest-pinned OCI reference, repo@sha256:...), the + With task_image (a digest-pinned OCI reference, repo@sha256:...), the environment runs that image unmodified: the server derives an ActorTemplate from the template, which then acts as the base, with the image as the container and the ate-env-guest mounted into it as an image volume. The derived template is created on first use and reused for later environments on the same image; info() reports its name. """ - req = env_pb2.CreateEnvironmentRequest(id=id, atespace=atespace, image=image or "") + req = env_pb2.CreateEnvironmentRequest(id=id, atespace=atespace, task_image=task_image or "") if template_name or template_atespace: req.template.name = template_name or "" req.template.atespace = template_atespace or "" diff --git a/clients/python/tests/e2e/test_full_stack.py b/clients/python/tests/e2e/test_full_stack.py index cd3ab73..ebbbe0e 100644 --- a/clients/python/tests/e2e/test_full_stack.py +++ b/clients/python/tests/e2e/test_full_stack.py @@ -187,7 +187,7 @@ async def test_full_lifecycle(client): async def test_create_from_image_rejects_unpinned_image(client): # Validated before anything touches the control plane. with pytest.raises(InvalidArgumentError): - await client.create(f"pye2e-img-{uuid.uuid4().hex[:8]}", image="python:3.12-slim") + await client.create(f"pye2e-img-{uuid.uuid4().hex[:8]}", task_image="python:3.12-slim") async def test_create_from_image_needs_a_base_template(client): @@ -196,7 +196,7 @@ async def test_create_from_image_needs_a_base_template(client): await client.create( f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=missing, - image="docker.io/library/busybox@sha256:" + "0" * 64, + task_image="docker.io/library/busybox@sha256:" + "0" * 64, ) assert excinfo.value.code == grpc.StatusCode.FAILED_PRECONDITION assert missing in str(excinfo.value) @@ -212,7 +212,7 @@ async def test_create_from_image(client): envs = [] try: env = await client.create( - f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, image=TASK_IMAGE + f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, task_image=TASK_IMAGE ) envs.append(env) @@ -246,7 +246,7 @@ async def test_create_from_image(client): # A second environment on the same image reuses the derived template # instead of minting another one. env2 = await client.create( - f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, image=TASK_IMAGE + f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, task_image=TASK_IMAGE ) envs.append(env2) assert (await env2.info()).template.name == want_template diff --git a/clients/python/tests/fakes.py b/clients/python/tests/fakes.py index fdd5198..158e27c 100644 --- a/clients/python/tests/fakes.py +++ b/clients/python/tests/fakes.py @@ -62,12 +62,12 @@ async def CreateEnvironment(self, request, context): template.name = request.template.name if request.template.atespace: template.atespace = request.template.atespace - if request.image: + if request.task_image: # Mirror ate-env-api: the template becomes the base of one derived # per image, named after the digest. - if "@sha256:" not in request.image: + if "@sha256:" not in request.task_image: await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "image is not pinned by digest") - digest = request.image.split("@sha256:", 1)[1] + digest = request.task_image.split("@sha256:", 1)[1] template.name = f"{template.name}-{digest[:12]}" environment = env_pb2.Environment( id=request.id, diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 3879f20..5aaa420 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -52,8 +52,8 @@ async def test_create_omits_template_when_not_given(fake_stack): async def test_create_with_image_reports_derived_template(fake_stack): client, fakes = fake_stack image = "docker.io/library/python@sha256:" + "0123456789abcdef" * 4 - env = await client.create("dev3", image=image) - assert fakes.environments.last_create_request.image == image + env = await client.create("dev3", task_image=image) + assert fakes.environments.last_create_request.task_image == image info = await env.info() assert info.template is not None assert info.template.name == "default-template-0123456789ab" @@ -62,7 +62,7 @@ async def test_create_with_image_reports_derived_template(fake_stack): async def test_create_with_image_and_base_template(fake_stack): client, fakes = fake_stack image = "docker.io/library/python@sha256:" + "0123456789abcdef" * 4 - env = await client.create("dev4", template_name="py-base", image=image) + env = await client.create("dev4", template_name="py-base", task_image=image) info = await env.info() assert info.template.name == "py-base-0123456789ab" @@ -70,7 +70,7 @@ async def test_create_with_image_and_base_template(fake_stack): async def test_create_omits_image_when_not_given(fake_stack): client, fakes = fake_stack await client.create("dev5") - assert fakes.environments.last_create_request.image == "" + assert fakes.environments.last_create_request.task_image == "" async def test_create_duplicate_maps_to_rpc_error_with_code(fake_stack): diff --git a/cmd/ate-env/main.go b/cmd/ate-env/main.go index 23a3ade..b700a9d 100644 --- a/cmd/ate-env/main.go +++ b/cmd/ate-env/main.go @@ -137,7 +137,7 @@ func newCreateCommand() *cobra.Command { atespace string createTemplate string createTemplateAtespace string - image string + taskImage string ) cmd := &cobra.Command{ Use: "create ", @@ -153,9 +153,9 @@ func newCreateCommand() *cobra.Command { defer client.Close() req := &ateenvv1alpha.CreateEnvironmentRequest{ - Id: args[0], - Atespace: atespace, - Image: image, + Id: args[0], + Atespace: atespace, + TaskImage: taskImage, } if createTemplate != "" || createTemplateAtespace != "" { tmplAtespace := createTemplateAtespace @@ -175,7 +175,7 @@ func newCreateCommand() *cobra.Command { cmd.Flags().StringVar(&atespace, "atespace", apiservice.DefaultAtespace, "Substrate atespace") cmd.Flags().StringVar(&createTemplate, "template", "", "ActorTemplate name (defaults to server default)") cmd.Flags().StringVar(&createTemplateAtespace, "template-atespace", "", "Substrate atespace of the ActorTemplate (defaults to environment atespace)") - cmd.Flags().StringVar(&image, "image", "", "digest-pinned OCI image to run unmodified (repo@sha256:...); the template becomes the base the guest is taken from") + cmd.Flags().StringVar(&taskImage, "task-image", "", "digest-pinned task image to run unmodified (repo@sha256:...); the template becomes the base the guest is taken from") return cmd } diff --git a/cmd/ate-env/main_test.go b/cmd/ate-env/main_test.go index 911005a..d80c840 100644 --- a/cmd/ate-env/main_test.go +++ b/cmd/ate-env/main_test.go @@ -204,20 +204,20 @@ func TestHelpOutput(t *testing.T) { } func TestCreateImageFlag(t *testing.T) { - args := []string{"create", "dev1", "--image", "example.com/task@sha256:abc"} + args := []string{"create", "dev1", "--task-image", "example.com/task@sha256:abc"} root := newRootCommand(args) cmd, _, err := root.Find(args[:2]) if err != nil { t.Fatalf("root.Find failed: %v", err) } - if cmd.Flags().Lookup("image") == nil { - t.Fatal("create has no --image flag") + if cmd.Flags().Lookup("task-image") == nil { + t.Fatal("create has no --task-image flag") } if err := cmd.ParseFlags(args[2:]); err != nil { - t.Fatalf("parsing --image: %v", err) + t.Fatalf("parsing --task-image: %v", err) } - if got, _ := cmd.Flags().GetString("image"); got != "example.com/task@sha256:abc" { - t.Errorf("--image = %q", got) + if got, _ := cmd.Flags().GetString("task-image"); got != "example.com/task@sha256:abc" { + t.Errorf("--task-image = %q", got) } } diff --git a/docs/task-images/DESIGN.md b/docs/task-images/DESIGN.md index b706cb6..53c3fe0 100644 --- a/docs/task-images/DESIGN.md +++ b/docs/task-images/DESIGN.md @@ -76,7 +76,7 @@ without adding information. ### Server flow -`CreateEnvironment` with `image` set: +`CreateEnvironment` with `task_image` set: 1. Validate the digest and compute the derived name; invalid input is `InvalidArgument` and touches nothing. @@ -96,7 +96,7 @@ in practice; a stricter comparison would cost a base fetch on every reuse. ### API shape -`image` is a field on `CreateEnvironmentRequest` rather than a new RPC, and the +`task_image` is a field on `CreateEnvironmentRequest` rather than a new RPC, and the existing `template` field doubles as the base. This keeps one create path, lets clients that already pass a template start passing an image with one more argument, and makes "which base" an explicit, per-request choice rather than @@ -122,7 +122,7 @@ the container's only process and the readiness authority. Derivation treats layers as part of what carries over. The guest volume is recognized by its name, not by being an image volume, so a base with a layer -still gets the guest added or kept correctly, and `create --image` on a layered +still gets the guest added or kept correctly, and `create --task-image` on a layered base yields environments with the second runtime present. Layers are documented in [RUNTIMES.md](RUNTIMES.md); the verified example uses busybox as a stand-in runtime. @@ -218,7 +218,7 @@ one golden snapshot per image. Neither is reclaimed automatically. volume and not the image, that the rootfs is the task image's, that files round-trip, and that a second environment reuses the derived template. - Run once end to end on a Kubernetes cluster running Substrate with a fresh - two-worker ate-env: `ate-env create --image` of an unmodified + two-worker ate-env: `ate-env create --task-image` of an unmodified `python:3.12-slim` produced the derived template, served `python3 --version` from the image's own rootfs with the guest present under `/ate`, and deleted cleanly. diff --git a/docs/task-images/README.md b/docs/task-images/README.md index 3af73ac..8e6d14d 100644 --- a/docs/task-images/README.md +++ b/docs/task-images/README.md @@ -71,11 +71,11 @@ Good when the set of images is open-ended or chosen by a caller, as in RL and evaluation harnesses. ```bash -ate-env create py1 --image docker.io/library/python@sha256: +ate-env create py1 --task-image docker.io/library/python@sha256: ``` ```python -env = await client.create("py1", image="docker.io/library/python@sha256:") +env = await client.create("py1", task_image="docker.io/library/python@sha256:") (await env.info()).template.name # "default-template-<12 hex of the digest>" ``` diff --git a/docs/task-images/RUNTIMES.md b/docs/task-images/RUNTIMES.md index 25ab84e..dbd0f24 100644 --- a/docs/task-images/RUNTIMES.md +++ b/docs/task-images/RUNTIMES.md @@ -24,7 +24,7 @@ nothing for it. They inherit the container's user, environment and filesystem. Template derivation keeps all of it. A base with layers yields derived templates with the same layers and the same guest flags, so environments -created on demand with `image=` inherit the second runtime without any caller +created on demand with `task_image=` inherit the second runtime without any caller knowing it exists. ## Registering a layered template @@ -47,12 +47,12 @@ The resulting template has two image volumes (`execd` at `/opt/opensandbox`, ``` Everything else is as in a plain task-image template. Environments are created -the usual way, from this template by name or from any other image with `--image` +the usual way, from this template by name or from any other image with `--task-image` while naming it as the base: ```bash ate-env create py1 --template osb-base -ate-env create node1 --template osb-base --image docker.io/library/node@sha256: +ate-env create node1 --template osb-base --task-image docker.io/library/node@sha256: ``` Both run the guest and `execd`; the second one from a derived template named diff --git a/examples/runtime-layers/demo.sh b/examples/runtime-layers/demo.sh index a080475..27f47d2 100755 --- a/examples/runtime-layers/demo.sh +++ b/examples/runtime-layers/demo.sh @@ -98,7 +98,7 @@ if [ -n "${OTHER_IMAGE:-}" ]; then ID2="layered2-$(date +%s)" ids+=("${ID2}") step "create ${ID2} from ${OTHER_IMAGE} with ${TEMPLATE} as the base (layer inherited)" - "${ATE_ENV}" create "${ID2}" --atespace "${ATESPACE}" --template "${TEMPLATE}" --image "${OTHER_IMAGE}" + "${ATE_ENV}" create "${ID2}" --atespace "${ATESPACE}" --template "${TEMPLATE}" --task-image "${OTHER_IMAGE}" "${KUBECTL_ATE}" get actor-template --atespace "${ATESPACE}" | grep "${TEMPLATE}-" || true wait_serving "${ID2}" "${ATE_ENV}" "${ID2}" shell "/opt/web/bin/busybox wget -qO- http://127.0.0.1:${PORT}/etc/os-release | head -1" diff --git a/internal/apiservice/server.go b/internal/apiservice/server.go index 6ddae6c..d2a7471 100644 --- a/internal/apiservice/server.go +++ b/internal/apiservice/server.go @@ -86,17 +86,17 @@ func (s *Server) Close() {} // and to create the actor; a template.atespace that differs from a non-empty // request atespace is ignored. // -// When image is set, the template acts as the base and the environment is +// When task_image is set, the template acts as the base and the environment is // created from a template derived from it, named // "-" in the same atespace. -// The derived template keeps the base's settings, runs image as the -// container image and mounts the base's guest as a read-only image volume, +// The derived template keeps the base's settings, runs the task image as +// the container image and mounts the base's guest as a read-only image volume, // so the image runs unmodified (see ensureImageTemplate and // DeriveImageTemplate). It is created on first use and reused by later // creates on the same image, and the response reports it in -// Environment.template. When image is empty the template is used as is. +// Environment.template. When task_image is empty the template is used as is. // -// Errors: InvalidArgument for a missing id, an image not pinned by digest, a +// Errors: InvalidArgument for a missing id, a task image not pinned by digest, a // derived name that is too long or not a valid resource name, or a base that // cannot be derived from (no containers, unpinned guest image, or a command // that cannot be re-rooted under the guest volume). FailedPrecondition when @@ -120,7 +120,7 @@ func (s *Server) CreateEnvironment(ctx context.Context, req *ateenvv1alpha.Creat atespace = DefaultAtespace } - if image := req.GetImage(); image != "" { + if image := req.GetTaskImage(); image != "" { name, err := s.ensureImageTemplate(ctx, atespace, templateName, image) if err != nil { return nil, err diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go index 6dd2752..671511f 100644 --- a/internal/apiservice/server_test.go +++ b/internal/apiservice/server_test.go @@ -557,7 +557,7 @@ func TestCreateFromImageDerivesTemplate(t *testing.T) { seedGuestTemplate(control, "ate-env", "default-template") ctx := context.Background() - resp, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img1", Image: testTaskImage}) + resp, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img1", TaskImage: testTaskImage}) if err != nil { t.Fatal(err) } @@ -597,7 +597,7 @@ func TestCreateFromImageDerivesTemplate(t *testing.T) { } // A second environment on the same image reuses the template. - if _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img2", Image: testTaskImage}); err != nil { + if _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img2", TaskImage: testTaskImage}); err != nil { t.Fatal(err) } if n := control.TemplateCount(); n != 2 { @@ -610,10 +610,10 @@ func TestCreateFromImageUsesNamedBaseAndAtespace(t *testing.T) { seedGuestTemplate(control, "team-a", "py-base") resp, err := envClient.CreateEnvironment(context.Background(), &ateenvv1alpha.CreateEnvironmentRequest{ - Id: "img1", - Atespace: "team-a", - Template: &ateenvv1alpha.Template{Name: "py-base"}, - Image: testTaskImage, + Id: "img1", + Atespace: "team-a", + Template: &ateenvv1alpha.Template{Name: "py-base"}, + TaskImage: testTaskImage, }) if err != nil { t.Fatal(err) @@ -636,8 +636,8 @@ func TestCreateFromImageRejects(t *testing.T) { req *ateenvv1alpha.CreateEnvironmentRequest code codes.Code }{ - {"unpinned image", &ateenvv1alpha.CreateEnvironmentRequest{Id: "a", Image: "python:3.12"}, codes.InvalidArgument}, - {"missing base", &ateenvv1alpha.CreateEnvironmentRequest{Id: "b", Template: &ateenvv1alpha.Template{Name: "nope"}, Image: testTaskImage}, codes.FailedPrecondition}, + {"unpinned image", &ateenvv1alpha.CreateEnvironmentRequest{Id: "a", TaskImage: "python:3.12"}, codes.InvalidArgument}, + {"missing base", &ateenvv1alpha.CreateEnvironmentRequest{Id: "b", Template: &ateenvv1alpha.Template{Name: "nope"}, TaskImage: testTaskImage}, codes.FailedPrecondition}, } for _, tc := range cases { _, err := envClient.CreateEnvironment(ctx, tc.req) @@ -651,7 +651,7 @@ func TestCreateFromImageRejects(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: "ate-env", Name: "default-template-0123456789ab"}, Containers: []*ateapipb.Container{{Name: "guest", Image: "example.com/other@sha256:" + testGuestImage[len(testGuestImage)-64:]}}, }) - _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "c", Image: testTaskImage}) + _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "c", TaskImage: testTaskImage}) if status.Code(err) != codes.FailedPrecondition { t.Errorf("conflicting derived template: code = %v (%v), want FailedPrecondition", status.Code(err), err) } diff --git a/proto/ateenv/v1alpha/env.pb.go b/proto/ateenv/v1alpha/env.pb.go index 4007dc0..ca31c98 100644 --- a/proto/ateenv/v1alpha/env.pb.go +++ b/proto/ateenv/v1alpha/env.pb.go @@ -20,7 +20,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v5.28.2 +// protoc v6.33.5 // source: proto/ateenv/v1alpha/env.proto package ateenvv1alpha @@ -253,16 +253,17 @@ type CreateEnvironmentRequest struct { Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` // ActorTemplate configuration to instantiate (defaults to name "default-template" in atespace "ate-env"). Template *Template `protobuf:"bytes,3,opt,name=template,proto3" json:"template,omitempty"` - // OCI image to run, pinned by digest (repo@sha256:...). Optional. When set, - // the environment is created from an ActorTemplate derived from `template`, - // which acts as the base: its worker selector, snapshot, sandbox and - // resource settings are kept, its container image is replaced by this - // image, and the ate-env-guest from the base is mounted into it as a - // read-only image volume, so the task image runs unmodified. The derived - // template is named "-" in the - // base's atespace; it is created on first use and reused afterwards, and - // the response reports it. - Image string `protobuf:"bytes,4,opt,name=image,proto3" json:"image,omitempty"` + // Task image to run, pinned by digest (repo@sha256:...). Optional. When + // set, the environment is created from an ActorTemplate derived from + // `template`, which acts as the base: its worker selector, snapshot, + // sandbox and resource settings are kept, its container image is replaced + // by the task image, and the ate-env-guest from the base is mounted into + // it as a read-only image volume (a runtime layer, not part of the task + // image), so the task image runs unmodified. The derived template is named + // "-" in the base's atespace; it + // is created on first use and reused afterwards, and the response reports + // it. + TaskImage string `protobuf:"bytes,4,opt,name=task_image,json=taskImage,proto3" json:"task_image,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -318,9 +319,9 @@ func (x *CreateEnvironmentRequest) GetTemplate() *Template { return nil } -func (x *CreateEnvironmentRequest) GetImage() string { +func (x *CreateEnvironmentRequest) GetTaskImage() string { if x != nil { - return x.Image + return x.TaskImage } return "" } @@ -668,12 +669,13 @@ const file_proto_ateenv_v1alpha_env_proto_rawDesc = "" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x124\n" + "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x129\n" + - "\x06status\x18\x04 \x01(\x0e2!.ateenv.v1alpha.EnvironmentStatusR\x06status\"\x92\x01\n" + + "\x06status\x18\x04 \x01(\x0e2!.ateenv.v1alpha.EnvironmentStatusR\x06status\"\x9b\x01\n" + "\x18CreateEnvironmentRequest\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x124\n" + - "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x12\x14\n" + - "\x05image\x18\x04 \x01(\tR\x05image\"Z\n" + + "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x12\x1d\n" + + "\n" + + "task_image\x18\x04 \x01(\tR\ttaskImage\"Z\n" + "\x19CreateEnvironmentResponse\x12=\n" + "\venvironment\x18\x01 \x01(\v2\x1b.ateenv.v1alpha.EnvironmentR\venvironment\"C\n" + "\x15GetEnvironmentRequest\x12\x0e\n" + diff --git a/proto/ateenv/v1alpha/env.proto b/proto/ateenv/v1alpha/env.proto index b2ce9b0..a2d0927 100644 --- a/proto/ateenv/v1alpha/env.proto +++ b/proto/ateenv/v1alpha/env.proto @@ -105,16 +105,17 @@ message CreateEnvironmentRequest { string atespace = 2; // ActorTemplate configuration to instantiate (defaults to name "default-template" in atespace "ate-env"). Template template = 3; - // OCI image to run, pinned by digest (repo@sha256:...). Optional. When set, - // the environment is created from an ActorTemplate derived from `template`, - // which acts as the base: its worker selector, snapshot, sandbox and - // resource settings are kept, its container image is replaced by this - // image, and the ate-env-guest from the base is mounted into it as a - // read-only image volume, so the task image runs unmodified. The derived - // template is named "-" in the - // base's atespace; it is created on first use and reused afterwards, and - // the response reports it. - string image = 4; + // Task image to run, pinned by digest (repo@sha256:...). Optional. When + // set, the environment is created from an ActorTemplate derived from + // `template`, which acts as the base: its worker selector, snapshot, + // sandbox and resource settings are kept, its container image is replaced + // by the task image, and the ate-env-guest from the base is mounted into + // it as a read-only image volume (a runtime layer, not part of the task + // image), so the task image runs unmodified. The derived template is named + // "-" in the base's atespace; it + // is created on first use and reused afterwards, and the response reports + // it. + string task_image = 4; } // Response returned after creating an environment.