diff --git a/docs/index.md b/docs/index.md index 0c0e15e5..9ecbc887 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,8 @@ vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. online pool reconfigure, metrics, the usage journal - [Replace a dataset in a running sandbox](live-volumes.md) — swapping a read-only catalog volume on a running claim with the volume API +- [Hibernate with an external volume controller](volume-controller.md) — detach, + hibernate, wake, and reattach while retaining the reservation and access gate - [Go SDK](sdk.md) — connecting (single node and clusters), every option, the full sandbox surface, error handling - [Python SDK](sdk-python.md) — the guest and data-plane surface for the diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 1d42f003..3d959a48 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -7,7 +7,8 @@ Register both read-only catalog entries on the owning node and grant the sandbox's tenant access to them. Put `v1` and `v2` in each image's `/version` file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. -A sandbox token alone cannot manage volumes. +A sandbox token alone cannot manage volumes. For a complete sleep and wake sequence, +see [the external controller example](volume-controller.md). Entries [registered at runtime](sandboxd-api.md#put-v1volumesname) work the same way. The API also supports diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 375dd0d6..bdcd6ed8 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -137,6 +137,9 @@ entirely yours: sandboxd writes and clears no dirty marker for an attach-only `rw` claim, because it cannot verify your unmount, so releasing without unmounting cleanly leaves the image as a crash would — see [sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +For a root client managing a tenant claim, see the +[external volume controller example](volume-controller.md). It keeps the +controller reservation and access gate across detach, hibernate, and wake. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud diff --git a/docs/volume-controller.md b/docs/volume-controller.md new file mode 100644 index 00000000..65dad95d --- /dev/null +++ b/docs/volume-controller.md @@ -0,0 +1,80 @@ +# Hibernate a sandbox with an external volume controller + +Remove a data disk before hibernation and restore it after wake, keeping the same +claim and catalog registration. The controller owns reservation and access policy. + +## Prepare the claim and controller + +Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. +Keep its TTL valid across sleep. Set `replicate_seconds: 0` and +`archive_after_seconds: 0` on its pool, and `on_expire: "destroy"` on the claim. +Keep that pool configured to avoid the node-wide archive fallback. An archived +or archiving claim refuses renew, so renewal cannot replace this configuration. +Keep replication disabled even while detached, when the claim holds no volume. +Changing replication requires a daemon restart. When migrating replicated claims +or changing failover, follow [disabling archive failover](deploy.md#disabling-archive-failover) +for retained replica or archive intent and recovery risks. + +Use Cloud Hypervisor, Cocoon v0.7.2 or newer, and the current Python SDK with +[per-entry attach-only support](sdk-python.md#changing-live-volumes). +Prepare and format the backing outside sandboxd. Follow the +[registration and backing requirements](sandboxd-api.md#put-v1volumesname), +including stable paths outside daemon cleanup directories. Register on the +claim's node with `writable: true`, the configured root token, and an explicit +[tenant access list](sandboxd-api.md#volume-access-lists). + +Hold a durable external reservation throughout use, detached sleep, and recovery. +Prevent other root or tenant callers from attaching the backing while detached. +A tenant access list is not a reservation. On owner change, keep access closed +and fence the old node before attaching elsewhere. Node-local admission is not +a distributed single-writer lock. + +## Supply the application hooks + +Copy the [tested example](https://github.com/cocoonstack/sandbox/blob/main/sdk/python/examples/volume_controller.py) and implement +its `VolumeHooks` contracts. Gate and drain API and preview traffic on every node, +including guest API access and direct ports. Issued preview URLs can wake the +claim. Stopping new URL issuance does not close access. +Stop writers, flush state, and ordinarily unmount before detach. Never use lazy +or forced unmount. Keep the reservation and access gate through every step. + +Use one attach-only writable volume. The controller owns filesystem consistency +and crash recovery. Attach-only RW writes no `.dirty` marker. An automatically +mounted RW volume that the controller unmounts externally retains its marker, +because sandboxd cannot confirm the clean unmount. See the +[dirty-marker rules](sandboxd-api.md#patch-v1sandboxesidvolumes). + +Bind a root client to the existing tenant claim. The root token authorizes +volume changes. The sandbox token authorizes guest commands and hibernation. + +```python +from cocoonsandbox import Client + +root = Client(owner_addr, api_token=root_node_token) +operator = root.attach(owner_addr, sandbox_id, sandbox_token) +``` + +Call `resume_volume` for initial attachment to an empty running claim or after +sleep. It wakes silkd, attaches, mounts, and verifies before opening access. +Call `hibernate_volume` after use. Do not use a context manager that releases +the claim when the sleep function returns. Neither function releases the reservation. + +## Recover with access closed + +For attach or detach failures, follow [partial failure and recovery](sandboxd-api.md#partial-failure-and-recovery). +Require a successful remove response before hibernation. An empty GET view does +not acknowledge durable retirement. After a failed, timed-out, or lost response, +retry the same remove target until acknowledged. Any remaining desired, applied, +or per-volume state, or ongoing convergence, prevents sleep. + +On any failure or cancellation, retain the reservation and keep access closed. +Inspect the confirmed claim state and resume from that stage. Make guest cleanup +safe to retry after unmount. Do not blindly repeat writes or formatting. +`exec("true")` proves silkd readiness, not disk attachment or mount readiness. +Attach, mount, and data checks must succeed before business traffic resumes. +Recover an uncertain gate operation externally, including a partial reopen. + +After a cycle, verify the same claim, backing, serial, filesystem, and payload. +The backing is external to the memory snapshot. The mocked HTTP tests check +SDK compatibility and failure ordering. They do not prove guest unmount, disk +integrity, external gates, or fencing. Qualify those on the real deployment. diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go index 3858f3e5..22667f10 100644 --- a/e2e/volume_catalog_test.go +++ b/e2e/volume_catalog_test.go @@ -2,6 +2,7 @@ package e2e import ( "encoding/json/v2" + "errors" "net/http" "slices" "testing" @@ -72,3 +73,85 @@ func TestRuntimeVolumeRegistrationThroughHTTP(t *testing.T) { t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) } } + +func TestRuntimeVolumeDetachHibernateWakeThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "external bytes"), "writable": true, "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/home", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + claim, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer claim.Close() + other, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer other.Close() + operator := st.client.Attach(st.addr, claim.ID, "") + otherOperator := st.client.Attach(st.addr, other.ID, "") + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if _, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatal(err) + } + if err = claim.Hibernate(t.Context()); err == nil { + t.Fatal("captured an attached disk") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("attached capture refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("root admitted a second writer before detach") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("second writer refusal: %v", err) + } + view, err := operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("detach acknowledgment: %+v %v", view, err) + } + if err = claim.Hibernate(t.Context()); err != nil { + t.Fatal(err) + } + if !hibernated(t, st, claim.ID) { + t.Fatal("detach followed by hibernate did not sleep the claim") + } + registered, listErr := st.client.Volumes(t.Context()) + if listErr != nil || len(registered) != 1 || registered[0].Name != "home" || !registered[0].Writable { + t.Fatalf("registration lost across sleep: %+v %v", registered, listErr) + } + if _, err = other.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatalf("detached volume stayed reserved during sleep: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatal(err) + } + beforeWake := st.eng.volumeOpsLog() + if out, execErr := claim.Exec(t.Context(), "echo", "awake"); execErr != nil || out != "awake\n" { + t.Fatalf("implicit wake: %q %v", out, execErr) + } + if hibernated(t, st, claim.ID) { + t.Fatal("guest request did not wake the original claim") + } + if ops := st.eng.volumeOpsLog(); !slices.Equal(ops, beforeWake) { + t.Fatalf("wake unexpectedly operated on external disks: %v -> %v", beforeWake, ops) + } + view, err = claim.GetVolumes(t.Context()) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("wake unexpectedly restored volume references: %+v %v", view, err) + } + view, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}) + if err != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) || view.Converging { + t.Fatalf("controller reattachment: %+v %v", view, err) + } + if status, body := rawJSON(t, st, http.MethodDelete, "/v1/volumes/home", ""); status != http.StatusConflict { + t.Fatalf("unregister attached restored disk: %d %s", status, body) + } +} diff --git a/sdk/python/examples/volume_controller.py b/sdk/python/examples/volume_controller.py new file mode 100644 index 00000000..9a5462c9 --- /dev/null +++ b/sdk/python/examples/volume_controller.py @@ -0,0 +1,63 @@ +from collections.abc import Callable +from dataclasses import dataclass + +from cocoonsandbox import Sandbox, Volume + + +@dataclass(frozen=True) +class VolumeHooks: + """Application operations required by the controller functions. + + assert_reserved verifies exclusive backing ownership and the owning node. + Hold a durable external reservation throughout use, detached sleep, failures, + and controller restarts. Renew leases separately and serialize operations. + Fence the old node before attaching at a new placement. This assertion is + neither a lock nor lease renewal. Tenant ACLs are not a reservation. + + close_access blocks and drains business requests before returning, including + API, guest API, direct ports, and preview listeners on every serving node. + Issued preview URLs can wake the claim and have no per-URL revocation. + Stopping URL issuance is insufficient. Maintenance calls may reach silkd. + + quiesce_and_unmount stops and reaps writers, flushes application and filesystem + state, and ordinarily unmounts. Raise on failure, including a busy mount. + Never use lazy or forced unmount. Account for raw-device users, partitions, + and other mount namespaces. A retry must tolerate an already unmounted disk. + + mount_and_verify finds the device by catalog serial, mounts its existing + filesystem, and checks its identity and data. Restart the application behind + the closed gate if needed. Raise before admitting traffic if any check fails. + open_access admits traffic only after verification succeeds. + + Keep access closed on exceptions, cancellation, or uncertain results. These + functions cannot prove an external gate closed or undo a partial reopen. + Recover failed gate operations externally while retaining the reservation. + """ + + assert_reserved: Callable[[], None] + close_access: Callable[[], None] + quiesce_and_unmount: Callable[[], None] + mount_and_verify: Callable[[], None] + open_access: Callable[[], None] + + +def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + hooks.quiesce_and_unmount() + view = sb.patch_volumes(remove=[name]) + if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: + raise RuntimeError("volume references remain; keep access closed") + sb.hibernate() + + +def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + sb.exec("true") + volume: Volume = {"name": name, "mode": "rw", "attach_only": True} + view = sb.patch_volumes(add=[volume]) + if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: + raise RuntimeError("volume attachment is not settled; keep access closed") + hooks.mount_and_verify() + hooks.open_access() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 5a898e7d..0b87f3cd 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,9 +13,12 @@ readme = "README.md" [tool.setuptools.packages.find] include = ["cocoonsandbox*"] +[tool.pytest.ini_options] +pythonpath = ["."] + [tool.mypy] strict = true -files = ["cocoonsandbox"] +files = ["cocoonsandbox", "examples"] [tool.ruff] line-length = 120 diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py new file mode 100644 index 00000000..e7436052 --- /dev/null +++ b/sdk/python/tests/test_volume_controller.py @@ -0,0 +1,155 @@ +from asyncio import CancelledError +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from conftest import FakeNode + +from cocoonsandbox import Client, VolumeError +from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume + +PATH = "/v1/sandboxes/sb_1" +EMPTY = {"volumes_want": [], "volumes": [], "states": [], "converging": False} +ATTACHED = { + "volumes_want": [{"name": "home", "mode": "rw", "attach_only": True}], + "volumes": [{"name": "home", "mode": "rw", "attach_only": True}], + "states": [{"name": "home", "state": "attached"}], + "converging": False, +} + +REMOVE = ("patch", {"remove": ["home"]}) +ADD = ("patch", {"add": [{"name": "home", "mode": "rw", "attach_only": True}]}) +REMOVE_STEPS = ["close", "reserved", "unmount", REMOVE] +ADD_STEPS = ["close", "reserved", "wake", ADD] + + +@pytest.fixture +def controller(spawn_node, monkeypatch): + events = [] + replies = [] + fail = set() + gate = {"open": True} + + def hook(name): + def run(): + events.append(name) + if name == "close": + gate["open"] = False + if name in fail: + raise RuntimeError(name) + if name == "open": + gate["open"] = True + + return run + + def patch(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer root-token" + assert not gate["open"] + events.append(("patch", body)) + if replies: + return replies.pop(0) + return 200, EMPTY if "remove" in body else ATTACHED + + def hibernate(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer sandbox-token" + assert not gate["open"] + events.append("hibernate") + return 204, {} + + addr = spawn_node({("PATCH", PATH + "/volumes"): patch, ("POST", PATH + "/hibernate"): hibernate}) + client = Client(addr, api_token="root-token") + sb = client.attach(addr, "sb_1", "sandbox-token") + + def exec_true(*argv): + assert argv == ("true",) + assert not gate["open"] + events.append("wake") + return "" + + monkeypatch.setattr(sb, "exec", exec_true) + hooks = VolumeHooks(hook("reserved"), hook("close"), hook("unmount"), hook("verify"), hook("open")) + return SimpleNamespace(sb=sb, hooks=hooks, events=events, replies=replies, fail=fail, gate=gate) + + +def test_controller_cycle_opens_after_verification(controller): + hibernate_volume(controller.sb, "home", controller.hooks) + assert not controller.gate["open"] + resume_volume(controller.sb, "home", controller.hooks) + assert controller.events == REMOVE_STEPS + ["hibernate"] + ADD_STEPS + ["verify", "open"] + assert controller.gate["open"] + assert controller.sb.volumes == [{"name": "home", "mode": "rw", "attach_only": True}] + + +@pytest.mark.parametrize( + ("failures", "responses", "expected"), + [ + (("reserved",), [], ["close", "reserved"]), + (("unmount",), [], ["close", "reserved", "unmount"]), + ((), [(200, {**EMPTY, "volumes_want": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "volumes": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "states": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "converging": True})], REMOVE_STEPS), + ], + ids=["reservation", "unmount", "desired-remains", "applied-remains", "state-remains", "converging"], +) +def test_controller_refuses_sleep(controller, failures, responses, expected): + controller.fail.update(failures) + controller.replies.extend(responses) + with pytest.raises(RuntimeError): + hibernate_volume(controller.sb, "home", controller.hooks) + assert controller.events == expected + assert not controller.gate["open"] + + +def test_controller_failed_remove_needs_successful_retry_even_with_empty_view(controller): + controller.replies.append((503, {**EMPTY, "error": "claim write failed"})) + with pytest.raises(VolumeError) as caught: + hibernate_volume(controller.sb, "home", controller.hooks) + assert caught.value.status == 503 + assert controller.sb.volumes == controller.sb.volumes_want == [] + assert not controller.gate["open"] + assert "hibernate" not in controller.events + + hibernate_volume(controller.sb, "home", controller.hooks) + assert [event for event in controller.events if isinstance(event, tuple)] == [REMOVE] * 2 + assert controller.events[-1] == "hibernate" + assert not controller.gate["open"] + + +@pytest.mark.parametrize( + ("reply", "failures", "error", "expected"), + [ + ((400, {"error": "backing missing"}), (), VolumeError, ADD_STEPS), + ((200, {**ATTACHED, "converging": True}), (), RuntimeError, ADD_STEPS), + ((200, {**ATTACHED, "volumes": []}), (), RuntimeError, ADD_STEPS), + ((200, ATTACHED), ("verify",), RuntimeError, ADD_STEPS + ["verify"]), + ], + ids=["attach-refused", "converging", "applied-missing", "verification"], +) +def test_controller_restore_failure_keeps_access_closed(controller, reply, failures, error, expected): + controller.replies.append(reply) + controller.fail.update(failures) + with pytest.raises(error): + resume_volume(controller.sb, "home", controller.hooks) + assert controller.events == expected + assert not controller.gate["open"] + + +@pytest.mark.parametrize( + ("operation", "hook_name", "event", "expected"), + [ + (hibernate_volume, "quiesce_and_unmount", "unmount", ["close", "reserved", "unmount"]), + (resume_volume, "mount_and_verify", "verify", ADD_STEPS + ["verify"]), + ], + ids=["quiesce", "verify"], +) +def test_controller_cancellation_keeps_access_closed(controller, operation, hook_name, event, expected): + def cancel(): + controller.events.append(event) + raise CancelledError() + + hooks = replace(controller.hooks, **{hook_name: cancel}) + with pytest.raises(CancelledError): + operation(controller.sb, "home", hooks) + assert controller.events == expected + assert not controller.gate["open"]