From f7098d8c0b803c1755f68be53189909d1ac73753 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Thu, 29 Oct 2026 08:49:58 +0000 Subject: [PATCH] e2e: cover controller-managed volume hibernation Complete the volume series with the external detach, hibernate, wake and reattach flow. Exercise the Python example through SDK requests and keep reservation, quiescence and access gating with the controller. --- docs/index.md | 2 + docs/live-volumes.md | 3 +- docs/sdk-python.md | 3 + docs/volume-controller.md | 80 +++++++++++ e2e/volume_catalog_test.go | 83 +++++++++++ sdk/python/examples/volume_controller.py | 63 +++++++++ sdk/python/pyproject.toml | 5 +- sdk/python/tests/test_volume_controller.py | 155 +++++++++++++++++++++ 8 files changed, 392 insertions(+), 2 deletions(-) create mode 100644 docs/volume-controller.md create mode 100644 sdk/python/examples/volume_controller.py create mode 100644 sdk/python/tests/test_volume_controller.py diff --git a/docs/index.md b/docs/index.md index 0c0e15e55..9ecbc8876 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,8 @@ vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. online pool reconfigure, metrics, the usage journal - [Replace a dataset in a running sandbox](live-volumes.md) — swapping a read-only catalog volume on a running claim with the volume API +- [Hibernate with an external volume controller](volume-controller.md) — detach, + hibernate, wake, and reattach while retaining the reservation and access gate - [Go SDK](sdk.md) — connecting (single node and clusters), every option, the full sandbox surface, error handling - [Python SDK](sdk-python.md) — the guest and data-plane surface for the diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 1d42f0038..3d959a48b 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -7,7 +7,8 @@ Register both read-only catalog entries on the owning node and grant the sandbox's tenant access to them. Put `v1` and `v2` in each image's `/version` file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. -A sandbox token alone cannot manage volumes. +A sandbox token alone cannot manage volumes. For a complete sleep and wake sequence, +see [the external controller example](volume-controller.md). Entries [registered at runtime](sandboxd-api.md#put-v1volumesname) work the same way. The API also supports diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 375dd0d64..bdcd6ed81 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -137,6 +137,9 @@ entirely yours: sandboxd writes and clears no dirty marker for an attach-only `rw` claim, because it cannot verify your unmount, so releasing without unmounting cleanly leaves the image as a crash would — see [sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +For a root client managing a tenant claim, see the +[external volume controller example](volume-controller.md). It keeps the +controller reservation and access gate across detach, hibernate, and wake. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud diff --git a/docs/volume-controller.md b/docs/volume-controller.md new file mode 100644 index 000000000..65dad95d5 --- /dev/null +++ b/docs/volume-controller.md @@ -0,0 +1,80 @@ +# Hibernate a sandbox with an external volume controller + +Remove a data disk before hibernation and restore it after wake, keeping the same +claim and catalog registration. The controller owns reservation and access policy. + +## Prepare the claim and controller + +Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. +Keep its TTL valid across sleep. Set `replicate_seconds: 0` and +`archive_after_seconds: 0` on its pool, and `on_expire: "destroy"` on the claim. +Keep that pool configured to avoid the node-wide archive fallback. An archived +or archiving claim refuses renew, so renewal cannot replace this configuration. +Keep replication disabled even while detached, when the claim holds no volume. +Changing replication requires a daemon restart. When migrating replicated claims +or changing failover, follow [disabling archive failover](deploy.md#disabling-archive-failover) +for retained replica or archive intent and recovery risks. + +Use Cloud Hypervisor, Cocoon v0.7.2 or newer, and the current Python SDK with +[per-entry attach-only support](sdk-python.md#changing-live-volumes). +Prepare and format the backing outside sandboxd. Follow the +[registration and backing requirements](sandboxd-api.md#put-v1volumesname), +including stable paths outside daemon cleanup directories. Register on the +claim's node with `writable: true`, the configured root token, and an explicit +[tenant access list](sandboxd-api.md#volume-access-lists). + +Hold a durable external reservation throughout use, detached sleep, and recovery. +Prevent other root or tenant callers from attaching the backing while detached. +A tenant access list is not a reservation. On owner change, keep access closed +and fence the old node before attaching elsewhere. Node-local admission is not +a distributed single-writer lock. + +## Supply the application hooks + +Copy the [tested example](https://github.com/cocoonstack/sandbox/blob/main/sdk/python/examples/volume_controller.py) and implement +its `VolumeHooks` contracts. Gate and drain API and preview traffic on every node, +including guest API access and direct ports. Issued preview URLs can wake the +claim. Stopping new URL issuance does not close access. +Stop writers, flush state, and ordinarily unmount before detach. Never use lazy +or forced unmount. Keep the reservation and access gate through every step. + +Use one attach-only writable volume. The controller owns filesystem consistency +and crash recovery. Attach-only RW writes no `.dirty` marker. An automatically +mounted RW volume that the controller unmounts externally retains its marker, +because sandboxd cannot confirm the clean unmount. See the +[dirty-marker rules](sandboxd-api.md#patch-v1sandboxesidvolumes). + +Bind a root client to the existing tenant claim. The root token authorizes +volume changes. The sandbox token authorizes guest commands and hibernation. + +```python +from cocoonsandbox import Client + +root = Client(owner_addr, api_token=root_node_token) +operator = root.attach(owner_addr, sandbox_id, sandbox_token) +``` + +Call `resume_volume` for initial attachment to an empty running claim or after +sleep. It wakes silkd, attaches, mounts, and verifies before opening access. +Call `hibernate_volume` after use. Do not use a context manager that releases +the claim when the sleep function returns. Neither function releases the reservation. + +## Recover with access closed + +For attach or detach failures, follow [partial failure and recovery](sandboxd-api.md#partial-failure-and-recovery). +Require a successful remove response before hibernation. An empty GET view does +not acknowledge durable retirement. After a failed, timed-out, or lost response, +retry the same remove target until acknowledged. Any remaining desired, applied, +or per-volume state, or ongoing convergence, prevents sleep. + +On any failure or cancellation, retain the reservation and keep access closed. +Inspect the confirmed claim state and resume from that stage. Make guest cleanup +safe to retry after unmount. Do not blindly repeat writes or formatting. +`exec("true")` proves silkd readiness, not disk attachment or mount readiness. +Attach, mount, and data checks must succeed before business traffic resumes. +Recover an uncertain gate operation externally, including a partial reopen. + +After a cycle, verify the same claim, backing, serial, filesystem, and payload. +The backing is external to the memory snapshot. The mocked HTTP tests check +SDK compatibility and failure ordering. They do not prove guest unmount, disk +integrity, external gates, or fencing. Qualify those on the real deployment. diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go index 3858f3e5c..22667f10c 100644 --- a/e2e/volume_catalog_test.go +++ b/e2e/volume_catalog_test.go @@ -2,6 +2,7 @@ package e2e import ( "encoding/json/v2" + "errors" "net/http" "slices" "testing" @@ -72,3 +73,85 @@ func TestRuntimeVolumeRegistrationThroughHTTP(t *testing.T) { t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) } } + +func TestRuntimeVolumeDetachHibernateWakeThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "external bytes"), "writable": true, "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/home", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + claim, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer claim.Close() + other, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer other.Close() + operator := st.client.Attach(st.addr, claim.ID, "") + otherOperator := st.client.Attach(st.addr, other.ID, "") + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if _, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatal(err) + } + if err = claim.Hibernate(t.Context()); err == nil { + t.Fatal("captured an attached disk") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("attached capture refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("root admitted a second writer before detach") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("second writer refusal: %v", err) + } + view, err := operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("detach acknowledgment: %+v %v", view, err) + } + if err = claim.Hibernate(t.Context()); err != nil { + t.Fatal(err) + } + if !hibernated(t, st, claim.ID) { + t.Fatal("detach followed by hibernate did not sleep the claim") + } + registered, listErr := st.client.Volumes(t.Context()) + if listErr != nil || len(registered) != 1 || registered[0].Name != "home" || !registered[0].Writable { + t.Fatalf("registration lost across sleep: %+v %v", registered, listErr) + } + if _, err = other.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatalf("detached volume stayed reserved during sleep: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatal(err) + } + beforeWake := st.eng.volumeOpsLog() + if out, execErr := claim.Exec(t.Context(), "echo", "awake"); execErr != nil || out != "awake\n" { + t.Fatalf("implicit wake: %q %v", out, execErr) + } + if hibernated(t, st, claim.ID) { + t.Fatal("guest request did not wake the original claim") + } + if ops := st.eng.volumeOpsLog(); !slices.Equal(ops, beforeWake) { + t.Fatalf("wake unexpectedly operated on external disks: %v -> %v", beforeWake, ops) + } + view, err = claim.GetVolumes(t.Context()) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("wake unexpectedly restored volume references: %+v %v", view, err) + } + view, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}) + if err != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) || view.Converging { + t.Fatalf("controller reattachment: %+v %v", view, err) + } + if status, body := rawJSON(t, st, http.MethodDelete, "/v1/volumes/home", ""); status != http.StatusConflict { + t.Fatalf("unregister attached restored disk: %d %s", status, body) + } +} diff --git a/sdk/python/examples/volume_controller.py b/sdk/python/examples/volume_controller.py new file mode 100644 index 000000000..9a5462c99 --- /dev/null +++ b/sdk/python/examples/volume_controller.py @@ -0,0 +1,63 @@ +from collections.abc import Callable +from dataclasses import dataclass + +from cocoonsandbox import Sandbox, Volume + + +@dataclass(frozen=True) +class VolumeHooks: + """Application operations required by the controller functions. + + assert_reserved verifies exclusive backing ownership and the owning node. + Hold a durable external reservation throughout use, detached sleep, failures, + and controller restarts. Renew leases separately and serialize operations. + Fence the old node before attaching at a new placement. This assertion is + neither a lock nor lease renewal. Tenant ACLs are not a reservation. + + close_access blocks and drains business requests before returning, including + API, guest API, direct ports, and preview listeners on every serving node. + Issued preview URLs can wake the claim and have no per-URL revocation. + Stopping URL issuance is insufficient. Maintenance calls may reach silkd. + + quiesce_and_unmount stops and reaps writers, flushes application and filesystem + state, and ordinarily unmounts. Raise on failure, including a busy mount. + Never use lazy or forced unmount. Account for raw-device users, partitions, + and other mount namespaces. A retry must tolerate an already unmounted disk. + + mount_and_verify finds the device by catalog serial, mounts its existing + filesystem, and checks its identity and data. Restart the application behind + the closed gate if needed. Raise before admitting traffic if any check fails. + open_access admits traffic only after verification succeeds. + + Keep access closed on exceptions, cancellation, or uncertain results. These + functions cannot prove an external gate closed or undo a partial reopen. + Recover failed gate operations externally while retaining the reservation. + """ + + assert_reserved: Callable[[], None] + close_access: Callable[[], None] + quiesce_and_unmount: Callable[[], None] + mount_and_verify: Callable[[], None] + open_access: Callable[[], None] + + +def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + hooks.quiesce_and_unmount() + view = sb.patch_volumes(remove=[name]) + if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: + raise RuntimeError("volume references remain; keep access closed") + sb.hibernate() + + +def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + sb.exec("true") + volume: Volume = {"name": name, "mode": "rw", "attach_only": True} + view = sb.patch_volumes(add=[volume]) + if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: + raise RuntimeError("volume attachment is not settled; keep access closed") + hooks.mount_and_verify() + hooks.open_access() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 5a898e7d6..0b87f3cd0 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,9 +13,12 @@ readme = "README.md" [tool.setuptools.packages.find] include = ["cocoonsandbox*"] +[tool.pytest.ini_options] +pythonpath = ["."] + [tool.mypy] strict = true -files = ["cocoonsandbox"] +files = ["cocoonsandbox", "examples"] [tool.ruff] line-length = 120 diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py new file mode 100644 index 000000000..e74360526 --- /dev/null +++ b/sdk/python/tests/test_volume_controller.py @@ -0,0 +1,155 @@ +from asyncio import CancelledError +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from conftest import FakeNode + +from cocoonsandbox import Client, VolumeError +from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume + +PATH = "/v1/sandboxes/sb_1" +EMPTY = {"volumes_want": [], "volumes": [], "states": [], "converging": False} +ATTACHED = { + "volumes_want": [{"name": "home", "mode": "rw", "attach_only": True}], + "volumes": [{"name": "home", "mode": "rw", "attach_only": True}], + "states": [{"name": "home", "state": "attached"}], + "converging": False, +} + +REMOVE = ("patch", {"remove": ["home"]}) +ADD = ("patch", {"add": [{"name": "home", "mode": "rw", "attach_only": True}]}) +REMOVE_STEPS = ["close", "reserved", "unmount", REMOVE] +ADD_STEPS = ["close", "reserved", "wake", ADD] + + +@pytest.fixture +def controller(spawn_node, monkeypatch): + events = [] + replies = [] + fail = set() + gate = {"open": True} + + def hook(name): + def run(): + events.append(name) + if name == "close": + gate["open"] = False + if name in fail: + raise RuntimeError(name) + if name == "open": + gate["open"] = True + + return run + + def patch(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer root-token" + assert not gate["open"] + events.append(("patch", body)) + if replies: + return replies.pop(0) + return 200, EMPTY if "remove" in body else ATTACHED + + def hibernate(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer sandbox-token" + assert not gate["open"] + events.append("hibernate") + return 204, {} + + addr = spawn_node({("PATCH", PATH + "/volumes"): patch, ("POST", PATH + "/hibernate"): hibernate}) + client = Client(addr, api_token="root-token") + sb = client.attach(addr, "sb_1", "sandbox-token") + + def exec_true(*argv): + assert argv == ("true",) + assert not gate["open"] + events.append("wake") + return "" + + monkeypatch.setattr(sb, "exec", exec_true) + hooks = VolumeHooks(hook("reserved"), hook("close"), hook("unmount"), hook("verify"), hook("open")) + return SimpleNamespace(sb=sb, hooks=hooks, events=events, replies=replies, fail=fail, gate=gate) + + +def test_controller_cycle_opens_after_verification(controller): + hibernate_volume(controller.sb, "home", controller.hooks) + assert not controller.gate["open"] + resume_volume(controller.sb, "home", controller.hooks) + assert controller.events == REMOVE_STEPS + ["hibernate"] + ADD_STEPS + ["verify", "open"] + assert controller.gate["open"] + assert controller.sb.volumes == [{"name": "home", "mode": "rw", "attach_only": True}] + + +@pytest.mark.parametrize( + ("failures", "responses", "expected"), + [ + (("reserved",), [], ["close", "reserved"]), + (("unmount",), [], ["close", "reserved", "unmount"]), + ((), [(200, {**EMPTY, "volumes_want": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "volumes": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "states": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "converging": True})], REMOVE_STEPS), + ], + ids=["reservation", "unmount", "desired-remains", "applied-remains", "state-remains", "converging"], +) +def test_controller_refuses_sleep(controller, failures, responses, expected): + controller.fail.update(failures) + controller.replies.extend(responses) + with pytest.raises(RuntimeError): + hibernate_volume(controller.sb, "home", controller.hooks) + assert controller.events == expected + assert not controller.gate["open"] + + +def test_controller_failed_remove_needs_successful_retry_even_with_empty_view(controller): + controller.replies.append((503, {**EMPTY, "error": "claim write failed"})) + with pytest.raises(VolumeError) as caught: + hibernate_volume(controller.sb, "home", controller.hooks) + assert caught.value.status == 503 + assert controller.sb.volumes == controller.sb.volumes_want == [] + assert not controller.gate["open"] + assert "hibernate" not in controller.events + + hibernate_volume(controller.sb, "home", controller.hooks) + assert [event for event in controller.events if isinstance(event, tuple)] == [REMOVE] * 2 + assert controller.events[-1] == "hibernate" + assert not controller.gate["open"] + + +@pytest.mark.parametrize( + ("reply", "failures", "error", "expected"), + [ + ((400, {"error": "backing missing"}), (), VolumeError, ADD_STEPS), + ((200, {**ATTACHED, "converging": True}), (), RuntimeError, ADD_STEPS), + ((200, {**ATTACHED, "volumes": []}), (), RuntimeError, ADD_STEPS), + ((200, ATTACHED), ("verify",), RuntimeError, ADD_STEPS + ["verify"]), + ], + ids=["attach-refused", "converging", "applied-missing", "verification"], +) +def test_controller_restore_failure_keeps_access_closed(controller, reply, failures, error, expected): + controller.replies.append(reply) + controller.fail.update(failures) + with pytest.raises(error): + resume_volume(controller.sb, "home", controller.hooks) + assert controller.events == expected + assert not controller.gate["open"] + + +@pytest.mark.parametrize( + ("operation", "hook_name", "event", "expected"), + [ + (hibernate_volume, "quiesce_and_unmount", "unmount", ["close", "reserved", "unmount"]), + (resume_volume, "mount_and_verify", "verify", ADD_STEPS + ["verify"]), + ], + ids=["quiesce", "verify"], +) +def test_controller_cancellation_keeps_access_closed(controller, operation, hook_name, event, expected): + def cancel(): + controller.events.append(event) + raise CancelledError() + + hooks = replace(controller.hooks, **{hook_name: cancel}) + with pytest.raises(CancelledError): + operation(controller.sb, "home", hooks) + assert controller.events == expected + assert not controller.gate["open"]