From 7be736a2413e96474db639ce90e37cd54a4ffd1e Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 29 Oct 2026 21:40:32 +0800 Subject: [PATCH 1/5] utils: one SyncDir owns the directory fsync pool kept its own syncDir next to the identical tail of ReplaceFileSync; both now call utils.SyncDir, which keeps the open/sync stage errors the catalog ordering test pins. Drops a comment that restated volumeQuiesceMax's use. --- sandboxd/pool/archive.go | 3 ++- sandboxd/pool/volume.go | 16 ++++------------ sandboxd/utils/utils.go | 9 +++++++-- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 14a7c7ce..e551bdfe 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -14,6 +14,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" ) const archiveDeleteDir = "archive-deletes" @@ -315,7 +316,7 @@ func (m *Manager) markArchiveCks(ckIDs []string) (failed map[string]error) { if len(written) == 0 { return failed } - if err := syncDir(dir); err != nil { + if err := utils.SyncDir(dir); err != nil { for _, id := range written { failed[id] = err } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index b5df9ecc..aee84b92 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -15,13 +15,13 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" ) const ( // sidecar, not data_dir: the marker travels with the image and survives a data_dir wipe volumeDirtySuffix = ".dirty" - // caps the per-mount budget below, so teardown can never hang for long. - volumeQuiesceMax = 10 * time.Second + volumeQuiesceMax = 10 * time.Second ) // volumeHolders is one name's live admission state: a writer excludes every other claim. @@ -372,14 +372,14 @@ func markVolumeDirty(path string) error { if err := f.Close(); err != nil { return err } - return syncDir(filepath.Dir(path)) + return utils.SyncDir(filepath.Dir(path)) } func clearVolumeDirty(path string) error { if err := os.Remove(volumeDirtyPath(path)); err != nil && !errors.Is(err, os.ErrNotExist) { return err } - _ = syncDir(filepath.Dir(path)) // a lost removal re-converges on the next clean release + _ = utils.SyncDir(filepath.Dir(path)) // a lost removal re-converges on the next clean release return nil } @@ -392,11 +392,3 @@ func volumeDirty(path string) bool { func volumeDirtyPath(path string) string { return path + volumeDirtySuffix } - -func syncDir(dir string) error { - d, err := os.Open(dir) //nolint:gosec // parent of an operator-configured image path - if err != nil { - return err - } - return errors.Join(d.Sync(), d.Close()) -} diff --git a/sandboxd/utils/utils.go b/sandboxd/utils/utils.go index 1fd63462..a75d8812 100644 --- a/sandboxd/utils/utils.go +++ b/sandboxd/utils/utils.go @@ -73,7 +73,7 @@ func WriteFileSync(path string, data []byte, perm os.FileMode) error { // ReplaceFileSync writes data to the open temp file f, fsyncs it, renames it over path and fsyncs the directory; f is closed and removed on failure. func ReplaceFileSync(f *os.File, path string, data []byte) (err error) { - dir, tmp := filepath.Dir(path), f.Name() + tmp := f.Name() renamed := false defer func() { if !renamed { @@ -95,7 +95,12 @@ func ReplaceFileSync(f *os.File, path string, data []byte) (err error) { return fmt.Errorf("rename %s: %w", path, err) } renamed = true - d, err := os.Open(dir) //nolint:gosec // caller-owned data-dir path + return SyncDir(filepath.Dir(path)) +} + +// SyncDir fsyncs dir so an entry created, renamed or removed in it is durable. +func SyncDir(dir string) error { + d, err := os.Open(dir) //nolint:gosec // caller-owned directory path if err != nil { return fmt.Errorf("open dir: %w", err) } From 8e2c43221c78967f83f80a6ab3b0d3abaa4b286f Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 29 Oct 2026 21:40:32 +0800 Subject: [PATCH 2/5] docs: match the SDK volume options and the mount retry WithVolumesAttachOnly and mount=False no longer validate locally (sandboxd answers 400), Volume carries attach_only, a mount reporting "not a valid block device" is retried within the setup budget (#404), and the README names runtime volume registration (#382). --- README.md | 5 +++-- docs/deploy.md | 3 ++- docs/sandboxd-api.md | 9 ++++++--- docs/sdk-python.md | 4 ++-- docs/sdk.md | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a69644d0..10eccfd4 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ performance) — source in baked into the base image - `sandboxd/` — per-node control plane (Go): warm pools refilled from golden snapshot exports (online-retunable), claim/release/renew/hibernate/fork/ - promote/checkpoint HTTP API, operator-catalog dataset volumes (read-only or - writable, mounted or attach-only, with live add and remove), signed preview URLs, + promote/checkpoint HTTP API, operator-catalog dataset volumes (static or + registered at runtime, read-only or writable, mounted or attach-only, with + live add and remove), signed preview URLs, the HTTP-upgrade byte relays to silkd and to any guest port, read-only file access to hibernated or archived claims from a warm VM of their pool (opt-in per pool `reader`), claim env with diff --git a/docs/deploy.md b/docs/deploy.md index 7fd1fb89..aba0df01 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -310,7 +310,8 @@ A volume claim may consume an ordinary warm Cloud Hypervisor VM. sandboxd attaches after the warm pop or provision, waits for the device settle — a serial match under `/sys/block`, then the `/dev/` node itself, since the kernel publishes sysfs before devtmpfs creates it — up to 2 seconds, then -mounts the device before finalizing the claim: `mode: "ro"` (the default) +mounts the device, retrying `not a valid block device` within the 10-second +setup budget, before finalizing the claim: `mode: "ro"` (the default) attaches and mounts read-only; `mode: "rw"` requires the catalog entry's `writable: true` and attaches and mounts read-write. Setup failure destroys the VM without quiescing — the claim was never handed out, so no workload diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 88920c49..369ca03e 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -129,8 +129,10 @@ entries, matching the request shape. sandboxd attaches each disk, waits for the device settle — a serial match under `/sys/block`, then the `/dev/` node itself, since the kernel publishes sysfs before devtmpfs creates it — up to 2 seconds total, and mounts the filesystem — read-only, unless the entry -requested and was granted `rw` — before returning. A custom mount may shadow -an existing populated guest directory for the claim's life. +requested and was granted `rw` — before returning. A mount that reports +`not a valid block device` is retried within the 10-second setup budget, +since the new node can exist before the kernel opens the disk. A custom mount +may shadow an existing populated guest directory for the claim's life. ### Attach-only volumes @@ -168,7 +170,8 @@ done printf '%s\n' "$device" ``` -Then mount it however the workload needs. A `ro` entry is attached +Then mount it however the workload needs, retrying a mount that reports +`not a valid block device` for a short time. A `ro` entry is attached `--readonly` and stays read-only at the guest block layer no matter who mounts it, so the guarantee does not depend on your mount flags. diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 375dd0d6..31779db5 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -106,8 +106,8 @@ sb = client.new("ghcr.io/cocoonstack/sandbox/rt:24.04", |---|---|---|---| | `net` | `"none"`, `"egress"` | `"none"` | Cloud Hypervisor network shape: `none` disables the NIC and uses vsock-only I/O; `egress` attaches a bridge/CNI NIC | | `size` | `"small"`, `"medium"`, `"large"`, `"xlarge"`, `"2xlarge"` | `"small"` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G | -| `volumes` | bare names or `{name, mount?, mode?}` mappings | `None` | attach and mount up to eight unique catalog dataset disks; an omitted mount defaults to `/volumes/`; `mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; accepted by `Client.new` and `Template.new` | -| `mount` | bool | `True` | mount every requested volume. `False` attaches the devices and leaves the mounting to the workload; a mapping carrying `mount` is then a `TypeError` | +| `volumes` | bare names or `{name, mount?, mode?, attach_only?}` mappings | `None` | attach and mount up to eight unique catalog dataset disks; an omitted mount defaults to `/volumes/`; `mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; accepted by `Client.new` and `Template.new` | +| `mount` | bool | `True` | mount every requested volume. `False` attaches the devices and leaves the mounting to the workload; sandboxd then rejects a mapping carrying `mount` with 400 | | `ttl_seconds` | int | server default 5m | sandbox TTL, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes | | `deadline` | `time.monotonic()` value | `None` | keyword-only wall clock over the whole claim, redirect targets included: every request is bounded by whatever is left and a spent deadline raises `TimeoutError`. Without it each redirect candidate gets a fresh `timeout`, so a cluster with unreachable peers can cost a multiple of it. Also on `Template.new` and `Checkpoint.new` | diff --git a/docs/sdk.md b/docs/sdk.md index 382d2dcc..02d22dcc 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -210,8 +210,8 @@ defer sb.Close() |---|---|---|---| | `WithNetwork(n)` | `NetNone`, `NetEgress` | `NetNone` | Cloud Hypervisor network shape: `NetNone` disables the NIC and uses vsock-only I/O; `NetEgress` attaches a bridge/CNI NIC | | `WithSize(s)` | `Small`, `Medium`, `Large`, `XLarge`, `XXLarge` | `Small` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G | -| `WithVolumes(volumes...)` | `Volume{Name, Mount?, Mode?}` entries | none | attach and mount up to eight unique catalog dataset disks; `Mount` defaults to `/volumes/`; `Mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; supported by `Client.New` and `Template.New` | -| `WithVolumesAttachOnly()` | — | mount | attach the requested volumes without mounting them; the workload finds each device and owns the mount. Rejects a `Volume.Mount` locally | +| `WithVolumes(volumes...)` | `Volume{Name, Mount?, Mode?, AttachOnly?}` entries | none | attach and mount up to eight unique catalog dataset disks; `Mount` defaults to `/volumes/`; `Mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; supported by `Client.New` and `Template.New` | +| `WithVolumesAttachOnly()` | — | mount | legacy claim-wide form of `Volume.AttachOnly`: attach every requested volume without mounting it; the workload finds each device and owns the mount. sandboxd rejects a `Volume.Mount` with 400 | | `WithTimeout(d)` | duration | server default 5m | sandbox TTL, rounded up to seconds, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes | `New` returns when the sandbox's silkd answers: a warm hit is milliseconds, From 826b963b554a8f9216fee0b26c295ccdca52313d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Thu, 29 Oct 2026 08:49:58 +0000 Subject: [PATCH 3/5] e2e: cover controller-managed volume hibernation Complete the volume series with the external detach, hibernate, wake and reattach flow. Exercise the Python example through SDK requests and keep reservation, quiescence and access gating with the controller. --- docs/index.md | 2 + docs/live-volumes.md | 3 +- docs/sdk-python.md | 3 + docs/volume-controller.md | 80 +++++++++++ e2e/volume_catalog_test.go | 83 +++++++++++ sdk/python/examples/volume_controller.py | 63 +++++++++ sdk/python/pyproject.toml | 5 +- sdk/python/tests/test_volume_controller.py | 155 +++++++++++++++++++++ 8 files changed, 392 insertions(+), 2 deletions(-) create mode 100644 docs/volume-controller.md create mode 100644 sdk/python/examples/volume_controller.py create mode 100644 sdk/python/tests/test_volume_controller.py diff --git a/docs/index.md b/docs/index.md index 0c0e15e5..9ecbc887 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,8 @@ vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. online pool reconfigure, metrics, the usage journal - [Replace a dataset in a running sandbox](live-volumes.md) — swapping a read-only catalog volume on a running claim with the volume API +- [Hibernate with an external volume controller](volume-controller.md) — detach, + hibernate, wake, and reattach while retaining the reservation and access gate - [Go SDK](sdk.md) — connecting (single node and clusters), every option, the full sandbox surface, error handling - [Python SDK](sdk-python.md) — the guest and data-plane surface for the diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 1d42f003..3d959a48 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -7,7 +7,8 @@ Register both read-only catalog entries on the owning node and grant the sandbox's tenant access to them. Put `v1` and `v2` in each image's `/version` file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. -A sandbox token alone cannot manage volumes. +A sandbox token alone cannot manage volumes. For a complete sleep and wake sequence, +see [the external controller example](volume-controller.md). Entries [registered at runtime](sandboxd-api.md#put-v1volumesname) work the same way. The API also supports diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 31779db5..87d404df 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -137,6 +137,9 @@ entirely yours: sandboxd writes and clears no dirty marker for an attach-only `rw` claim, because it cannot verify your unmount, so releasing without unmounting cleanly leaves the image as a crash would — see [sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +For a root client managing a tenant claim, see the +[external volume controller example](volume-controller.md). It keeps the +controller reservation and access gate across detach, hibernate, and wake. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud diff --git a/docs/volume-controller.md b/docs/volume-controller.md new file mode 100644 index 00000000..65dad95d --- /dev/null +++ b/docs/volume-controller.md @@ -0,0 +1,80 @@ +# Hibernate a sandbox with an external volume controller + +Remove a data disk before hibernation and restore it after wake, keeping the same +claim and catalog registration. The controller owns reservation and access policy. + +## Prepare the claim and controller + +Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. +Keep its TTL valid across sleep. Set `replicate_seconds: 0` and +`archive_after_seconds: 0` on its pool, and `on_expire: "destroy"` on the claim. +Keep that pool configured to avoid the node-wide archive fallback. An archived +or archiving claim refuses renew, so renewal cannot replace this configuration. +Keep replication disabled even while detached, when the claim holds no volume. +Changing replication requires a daemon restart. When migrating replicated claims +or changing failover, follow [disabling archive failover](deploy.md#disabling-archive-failover) +for retained replica or archive intent and recovery risks. + +Use Cloud Hypervisor, Cocoon v0.7.2 or newer, and the current Python SDK with +[per-entry attach-only support](sdk-python.md#changing-live-volumes). +Prepare and format the backing outside sandboxd. Follow the +[registration and backing requirements](sandboxd-api.md#put-v1volumesname), +including stable paths outside daemon cleanup directories. Register on the +claim's node with `writable: true`, the configured root token, and an explicit +[tenant access list](sandboxd-api.md#volume-access-lists). + +Hold a durable external reservation throughout use, detached sleep, and recovery. +Prevent other root or tenant callers from attaching the backing while detached. +A tenant access list is not a reservation. On owner change, keep access closed +and fence the old node before attaching elsewhere. Node-local admission is not +a distributed single-writer lock. + +## Supply the application hooks + +Copy the [tested example](https://github.com/cocoonstack/sandbox/blob/main/sdk/python/examples/volume_controller.py) and implement +its `VolumeHooks` contracts. Gate and drain API and preview traffic on every node, +including guest API access and direct ports. Issued preview URLs can wake the +claim. Stopping new URL issuance does not close access. +Stop writers, flush state, and ordinarily unmount before detach. Never use lazy +or forced unmount. Keep the reservation and access gate through every step. + +Use one attach-only writable volume. The controller owns filesystem consistency +and crash recovery. Attach-only RW writes no `.dirty` marker. An automatically +mounted RW volume that the controller unmounts externally retains its marker, +because sandboxd cannot confirm the clean unmount. See the +[dirty-marker rules](sandboxd-api.md#patch-v1sandboxesidvolumes). + +Bind a root client to the existing tenant claim. The root token authorizes +volume changes. The sandbox token authorizes guest commands and hibernation. + +```python +from cocoonsandbox import Client + +root = Client(owner_addr, api_token=root_node_token) +operator = root.attach(owner_addr, sandbox_id, sandbox_token) +``` + +Call `resume_volume` for initial attachment to an empty running claim or after +sleep. It wakes silkd, attaches, mounts, and verifies before opening access. +Call `hibernate_volume` after use. Do not use a context manager that releases +the claim when the sleep function returns. Neither function releases the reservation. + +## Recover with access closed + +For attach or detach failures, follow [partial failure and recovery](sandboxd-api.md#partial-failure-and-recovery). +Require a successful remove response before hibernation. An empty GET view does +not acknowledge durable retirement. After a failed, timed-out, or lost response, +retry the same remove target until acknowledged. Any remaining desired, applied, +or per-volume state, or ongoing convergence, prevents sleep. + +On any failure or cancellation, retain the reservation and keep access closed. +Inspect the confirmed claim state and resume from that stage. Make guest cleanup +safe to retry after unmount. Do not blindly repeat writes or formatting. +`exec("true")` proves silkd readiness, not disk attachment or mount readiness. +Attach, mount, and data checks must succeed before business traffic resumes. +Recover an uncertain gate operation externally, including a partial reopen. + +After a cycle, verify the same claim, backing, serial, filesystem, and payload. +The backing is external to the memory snapshot. The mocked HTTP tests check +SDK compatibility and failure ordering. They do not prove guest unmount, disk +integrity, external gates, or fencing. Qualify those on the real deployment. diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go index 3858f3e5..22667f10 100644 --- a/e2e/volume_catalog_test.go +++ b/e2e/volume_catalog_test.go @@ -2,6 +2,7 @@ package e2e import ( "encoding/json/v2" + "errors" "net/http" "slices" "testing" @@ -72,3 +73,85 @@ func TestRuntimeVolumeRegistrationThroughHTTP(t *testing.T) { t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) } } + +func TestRuntimeVolumeDetachHibernateWakeThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "external bytes"), "writable": true, "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/home", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + claim, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer claim.Close() + other, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer other.Close() + operator := st.client.Attach(st.addr, claim.ID, "") + otherOperator := st.client.Attach(st.addr, other.ID, "") + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if _, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatal(err) + } + if err = claim.Hibernate(t.Context()); err == nil { + t.Fatal("captured an attached disk") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("attached capture refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("root admitted a second writer before detach") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("second writer refusal: %v", err) + } + view, err := operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("detach acknowledgment: %+v %v", view, err) + } + if err = claim.Hibernate(t.Context()); err != nil { + t.Fatal(err) + } + if !hibernated(t, st, claim.ID) { + t.Fatal("detach followed by hibernate did not sleep the claim") + } + registered, listErr := st.client.Volumes(t.Context()) + if listErr != nil || len(registered) != 1 || registered[0].Name != "home" || !registered[0].Writable { + t.Fatalf("registration lost across sleep: %+v %v", registered, listErr) + } + if _, err = other.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatalf("detached volume stayed reserved during sleep: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatal(err) + } + beforeWake := st.eng.volumeOpsLog() + if out, execErr := claim.Exec(t.Context(), "echo", "awake"); execErr != nil || out != "awake\n" { + t.Fatalf("implicit wake: %q %v", out, execErr) + } + if hibernated(t, st, claim.ID) { + t.Fatal("guest request did not wake the original claim") + } + if ops := st.eng.volumeOpsLog(); !slices.Equal(ops, beforeWake) { + t.Fatalf("wake unexpectedly operated on external disks: %v -> %v", beforeWake, ops) + } + view, err = claim.GetVolumes(t.Context()) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("wake unexpectedly restored volume references: %+v %v", view, err) + } + view, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}) + if err != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) || view.Converging { + t.Fatalf("controller reattachment: %+v %v", view, err) + } + if status, body := rawJSON(t, st, http.MethodDelete, "/v1/volumes/home", ""); status != http.StatusConflict { + t.Fatalf("unregister attached restored disk: %d %s", status, body) + } +} diff --git a/sdk/python/examples/volume_controller.py b/sdk/python/examples/volume_controller.py new file mode 100644 index 00000000..9a5462c9 --- /dev/null +++ b/sdk/python/examples/volume_controller.py @@ -0,0 +1,63 @@ +from collections.abc import Callable +from dataclasses import dataclass + +from cocoonsandbox import Sandbox, Volume + + +@dataclass(frozen=True) +class VolumeHooks: + """Application operations required by the controller functions. + + assert_reserved verifies exclusive backing ownership and the owning node. + Hold a durable external reservation throughout use, detached sleep, failures, + and controller restarts. Renew leases separately and serialize operations. + Fence the old node before attaching at a new placement. This assertion is + neither a lock nor lease renewal. Tenant ACLs are not a reservation. + + close_access blocks and drains business requests before returning, including + API, guest API, direct ports, and preview listeners on every serving node. + Issued preview URLs can wake the claim and have no per-URL revocation. + Stopping URL issuance is insufficient. Maintenance calls may reach silkd. + + quiesce_and_unmount stops and reaps writers, flushes application and filesystem + state, and ordinarily unmounts. Raise on failure, including a busy mount. + Never use lazy or forced unmount. Account for raw-device users, partitions, + and other mount namespaces. A retry must tolerate an already unmounted disk. + + mount_and_verify finds the device by catalog serial, mounts its existing + filesystem, and checks its identity and data. Restart the application behind + the closed gate if needed. Raise before admitting traffic if any check fails. + open_access admits traffic only after verification succeeds. + + Keep access closed on exceptions, cancellation, or uncertain results. These + functions cannot prove an external gate closed or undo a partial reopen. + Recover failed gate operations externally while retaining the reservation. + """ + + assert_reserved: Callable[[], None] + close_access: Callable[[], None] + quiesce_and_unmount: Callable[[], None] + mount_and_verify: Callable[[], None] + open_access: Callable[[], None] + + +def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + hooks.quiesce_and_unmount() + view = sb.patch_volumes(remove=[name]) + if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: + raise RuntimeError("volume references remain; keep access closed") + sb.hibernate() + + +def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + sb.exec("true") + volume: Volume = {"name": name, "mode": "rw", "attach_only": True} + view = sb.patch_volumes(add=[volume]) + if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: + raise RuntimeError("volume attachment is not settled; keep access closed") + hooks.mount_and_verify() + hooks.open_access() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 5a898e7d..0b87f3cd 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,9 +13,12 @@ readme = "README.md" [tool.setuptools.packages.find] include = ["cocoonsandbox*"] +[tool.pytest.ini_options] +pythonpath = ["."] + [tool.mypy] strict = true -files = ["cocoonsandbox"] +files = ["cocoonsandbox", "examples"] [tool.ruff] line-length = 120 diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py new file mode 100644 index 00000000..e7436052 --- /dev/null +++ b/sdk/python/tests/test_volume_controller.py @@ -0,0 +1,155 @@ +from asyncio import CancelledError +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from conftest import FakeNode + +from cocoonsandbox import Client, VolumeError +from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume + +PATH = "/v1/sandboxes/sb_1" +EMPTY = {"volumes_want": [], "volumes": [], "states": [], "converging": False} +ATTACHED = { + "volumes_want": [{"name": "home", "mode": "rw", "attach_only": True}], + "volumes": [{"name": "home", "mode": "rw", "attach_only": True}], + "states": [{"name": "home", "state": "attached"}], + "converging": False, +} + +REMOVE = ("patch", {"remove": ["home"]}) +ADD = ("patch", {"add": [{"name": "home", "mode": "rw", "attach_only": True}]}) +REMOVE_STEPS = ["close", "reserved", "unmount", REMOVE] +ADD_STEPS = ["close", "reserved", "wake", ADD] + + +@pytest.fixture +def controller(spawn_node, monkeypatch): + events = [] + replies = [] + fail = set() + gate = {"open": True} + + def hook(name): + def run(): + events.append(name) + if name == "close": + gate["open"] = False + if name in fail: + raise RuntimeError(name) + if name == "open": + gate["open"] = True + + return run + + def patch(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer root-token" + assert not gate["open"] + events.append(("patch", body)) + if replies: + return replies.pop(0) + return 200, EMPTY if "remove" in body else ATTACHED + + def hibernate(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer sandbox-token" + assert not gate["open"] + events.append("hibernate") + return 204, {} + + addr = spawn_node({("PATCH", PATH + "/volumes"): patch, ("POST", PATH + "/hibernate"): hibernate}) + client = Client(addr, api_token="root-token") + sb = client.attach(addr, "sb_1", "sandbox-token") + + def exec_true(*argv): + assert argv == ("true",) + assert not gate["open"] + events.append("wake") + return "" + + monkeypatch.setattr(sb, "exec", exec_true) + hooks = VolumeHooks(hook("reserved"), hook("close"), hook("unmount"), hook("verify"), hook("open")) + return SimpleNamespace(sb=sb, hooks=hooks, events=events, replies=replies, fail=fail, gate=gate) + + +def test_controller_cycle_opens_after_verification(controller): + hibernate_volume(controller.sb, "home", controller.hooks) + assert not controller.gate["open"] + resume_volume(controller.sb, "home", controller.hooks) + assert controller.events == REMOVE_STEPS + ["hibernate"] + ADD_STEPS + ["verify", "open"] + assert controller.gate["open"] + assert controller.sb.volumes == [{"name": "home", "mode": "rw", "attach_only": True}] + + +@pytest.mark.parametrize( + ("failures", "responses", "expected"), + [ + (("reserved",), [], ["close", "reserved"]), + (("unmount",), [], ["close", "reserved", "unmount"]), + ((), [(200, {**EMPTY, "volumes_want": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "volumes": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "states": [{"name": "other"}]})], REMOVE_STEPS), + ((), [(200, {**EMPTY, "converging": True})], REMOVE_STEPS), + ], + ids=["reservation", "unmount", "desired-remains", "applied-remains", "state-remains", "converging"], +) +def test_controller_refuses_sleep(controller, failures, responses, expected): + controller.fail.update(failures) + controller.replies.extend(responses) + with pytest.raises(RuntimeError): + hibernate_volume(controller.sb, "home", controller.hooks) + assert controller.events == expected + assert not controller.gate["open"] + + +def test_controller_failed_remove_needs_successful_retry_even_with_empty_view(controller): + controller.replies.append((503, {**EMPTY, "error": "claim write failed"})) + with pytest.raises(VolumeError) as caught: + hibernate_volume(controller.sb, "home", controller.hooks) + assert caught.value.status == 503 + assert controller.sb.volumes == controller.sb.volumes_want == [] + assert not controller.gate["open"] + assert "hibernate" not in controller.events + + hibernate_volume(controller.sb, "home", controller.hooks) + assert [event for event in controller.events if isinstance(event, tuple)] == [REMOVE] * 2 + assert controller.events[-1] == "hibernate" + assert not controller.gate["open"] + + +@pytest.mark.parametrize( + ("reply", "failures", "error", "expected"), + [ + ((400, {"error": "backing missing"}), (), VolumeError, ADD_STEPS), + ((200, {**ATTACHED, "converging": True}), (), RuntimeError, ADD_STEPS), + ((200, {**ATTACHED, "volumes": []}), (), RuntimeError, ADD_STEPS), + ((200, ATTACHED), ("verify",), RuntimeError, ADD_STEPS + ["verify"]), + ], + ids=["attach-refused", "converging", "applied-missing", "verification"], +) +def test_controller_restore_failure_keeps_access_closed(controller, reply, failures, error, expected): + controller.replies.append(reply) + controller.fail.update(failures) + with pytest.raises(error): + resume_volume(controller.sb, "home", controller.hooks) + assert controller.events == expected + assert not controller.gate["open"] + + +@pytest.mark.parametrize( + ("operation", "hook_name", "event", "expected"), + [ + (hibernate_volume, "quiesce_and_unmount", "unmount", ["close", "reserved", "unmount"]), + (resume_volume, "mount_and_verify", "verify", ADD_STEPS + ["verify"]), + ], + ids=["quiesce", "verify"], +) +def test_controller_cancellation_keeps_access_closed(controller, operation, hook_name, event, expected): + def cancel(): + controller.events.append(event) + raise CancelledError() + + hooks = replace(controller.hooks, **{hook_name: cancel}) + with pytest.raises(CancelledError): + operation(controller.sb, "home", hooks) + assert controller.events == expected + assert not controller.gate["open"] From 8a022a7fe4d4c5c4c2c1e1feabdaf0c324291121 Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 29 Oct 2026 21:58:37 +0800 Subject: [PATCH 4/5] review: tighten the volume controller example and its doc The VolumeHooks docstring repeated the doc in 26 lines; each hook's contract now lives once in docs/volume-controller.md, which also points mount_and_verify at the attach-only device discovery and mount retry. The cancellation test repeated the failure cases: the functions hold no cleanup, so any exception leaves the gate closed the same way. --- docs/volume-controller.md | 54 +++++++++++++--------- sdk/python/examples/volume_controller.py | 30 ++---------- sdk/python/tests/test_volume_controller.py | 22 --------- 3 files changed, 35 insertions(+), 71 deletions(-) diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 65dad95d..f5de0d7a 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -5,15 +5,15 @@ claim and catalog registration. The controller owns reservation and access polic ## Prepare the claim and controller -Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. -Keep its TTL valid across sleep. Set `replicate_seconds: 0` and -`archive_after_seconds: 0` on its pool, and `on_expire: "destroy"` on the claim. -Keep that pool configured to avoid the node-wide archive fallback. An archived -or archiving claim refuses renew, so renewal cannot replace this configuration. -Keep replication disabled even while detached, when the claim holds no volume. -Changing replication requires a daemon restart. When migrating replicated claims -or changing failover, follow [disabling archive failover](deploy.md#disabling-archive-failover) -for retained replica or archive intent and recovery risks. +Use a `net: none` claim on its owning node, since egress-lane claims cannot +hibernate, and keep its TTL valid across sleep. Its pool sets +`replicate_seconds: 0` and `archive_after_seconds: 0`, and the claim uses +`on_expire: "destroy"`: a detached sleeping claim holds no volume, so a replica +or an archive of it could wake on another node. Keep the pool configured, because +an unpooled key falls back to the node-wide `archive_after_seconds`. Replication +changes only at a restart; see +[disabling archive failover](deploy.md#disabling-archive-failover) before +changing it on a node with replicated claims. Use Cloud Hypervisor, Cocoon v0.7.2 or newer, and the current Python SDK with [per-entry attach-only support](sdk-python.md#changing-live-volumes). @@ -32,11 +32,22 @@ a distributed single-writer lock. ## Supply the application hooks Copy the [tested example](https://github.com/cocoonstack/sandbox/blob/main/sdk/python/examples/volume_controller.py) and implement -its `VolumeHooks` contracts. Gate and drain API and preview traffic on every node, -including guest API access and direct ports. Issued preview URLs can wake the -claim. Stopping new URL issuance does not close access. -Stop writers, flush state, and ordinarily unmount before detach. Never use lazy -or forced unmount. Keep the reservation and access gate through every step. +its `VolumeHooks`. Each hook raises on failure, and the controller keeps the +reservation and the access gate through every step. + +- `assert_reserved` checks the external reservation and the owning node. It is + not a lock or a lease renewal; serialize controller operations on one claim. +- `close_access` blocks and drains business traffic on every node, including the + guest API and direct ports. Issued preview URLs can wake the claim, and + stopping new URL issuance does not close access. +- `quiesce_and_unmount` stops writers, flushes state, and unmounts normally, + never lazily or by force. It raises on a busy mount, accounts for raw-device + users, partitions, and other mount namespaces, and tolerates a disk that is + already unmounted. +- `mount_and_verify` finds the device by its serial and mounts it as described + in [attach-only volumes](sandboxd-api.md#attach-only-volumes), then checks the + filesystem identity and data. +- `open_access` admits traffic only after verification succeeds. Use one attach-only writable volume. The controller owns filesystem consistency and crash recovery. Attach-only RW writes no `.dirty` marker. An automatically @@ -68,13 +79,12 @@ retry the same remove target until acknowledged. Any remaining desired, applied, or per-volume state, or ongoing convergence, prevents sleep. On any failure or cancellation, retain the reservation and keep access closed. -Inspect the confirmed claim state and resume from that stage. Make guest cleanup -safe to retry after unmount. Do not blindly repeat writes or formatting. -`exec("true")` proves silkd readiness, not disk attachment or mount readiness. -Attach, mount, and data checks must succeed before business traffic resumes. -Recover an uncertain gate operation externally, including a partial reopen. +Inspect the confirmed claim state and resume from that stage. Do not blindly +repeat writes or formatting. `exec("true")` proves silkd readiness, not disk +attachment or mount readiness. Recover an uncertain gate operation externally, +including a partial reopen. After a cycle, verify the same claim, backing, serial, filesystem, and payload. -The backing is external to the memory snapshot. The mocked HTTP tests check -SDK compatibility and failure ordering. They do not prove guest unmount, disk -integrity, external gates, or fencing. Qualify those on the real deployment. +The backing is external to the memory snapshot. The repository tests mock the +HTTP API, so qualify guest unmount, disk integrity, external gates, and fencing +on the real deployment. diff --git a/sdk/python/examples/volume_controller.py b/sdk/python/examples/volume_controller.py index 9a5462c9..bd5f8f0c 100644 --- a/sdk/python/examples/volume_controller.py +++ b/sdk/python/examples/volume_controller.py @@ -1,3 +1,5 @@ +"""External volume controller: detach before hibernation, reattach after wake (docs/volume-controller.md).""" + from collections.abc import Callable from dataclasses import dataclass @@ -6,33 +8,7 @@ @dataclass(frozen=True) class VolumeHooks: - """Application operations required by the controller functions. - - assert_reserved verifies exclusive backing ownership and the owning node. - Hold a durable external reservation throughout use, detached sleep, failures, - and controller restarts. Renew leases separately and serialize operations. - Fence the old node before attaching at a new placement. This assertion is - neither a lock nor lease renewal. Tenant ACLs are not a reservation. - - close_access blocks and drains business requests before returning, including - API, guest API, direct ports, and preview listeners on every serving node. - Issued preview URLs can wake the claim and have no per-URL revocation. - Stopping URL issuance is insufficient. Maintenance calls may reach silkd. - - quiesce_and_unmount stops and reaps writers, flushes application and filesystem - state, and ordinarily unmounts. Raise on failure, including a busy mount. - Never use lazy or forced unmount. Account for raw-device users, partitions, - and other mount namespaces. A retry must tolerate an already unmounted disk. - - mount_and_verify finds the device by catalog serial, mounts its existing - filesystem, and checks its identity and data. Restart the application behind - the closed gate if needed. Raise before admitting traffic if any check fails. - open_access admits traffic only after verification succeeds. - - Keep access closed on exceptions, cancellation, or uncertain results. These - functions cannot prove an external gate closed or undo a partial reopen. - Recover failed gate operations externally while retaining the reservation. - """ + """Application operations the controller calls; docs/volume-controller.md states each contract.""" assert_reserved: Callable[[], None] close_access: Callable[[], None] diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py index e7436052..c43036be 100644 --- a/sdk/python/tests/test_volume_controller.py +++ b/sdk/python/tests/test_volume_controller.py @@ -1,5 +1,3 @@ -from asyncio import CancelledError -from dataclasses import replace from types import SimpleNamespace import pytest @@ -133,23 +131,3 @@ def test_controller_restore_failure_keeps_access_closed(controller, reply, failu resume_volume(controller.sb, "home", controller.hooks) assert controller.events == expected assert not controller.gate["open"] - - -@pytest.mark.parametrize( - ("operation", "hook_name", "event", "expected"), - [ - (hibernate_volume, "quiesce_and_unmount", "unmount", ["close", "reserved", "unmount"]), - (resume_volume, "mount_and_verify", "verify", ADD_STEPS + ["verify"]), - ], - ids=["quiesce", "verify"], -) -def test_controller_cancellation_keeps_access_closed(controller, operation, hook_name, event, expected): - def cancel(): - controller.events.append(event) - raise CancelledError() - - hooks = replace(controller.hooks, **{hook_name: cancel}) - with pytest.raises(CancelledError): - operation(controller.sb, "home", hooks) - assert controller.events == expected - assert not controller.gate["open"] From 37cbc7c39941c75a371dbaea60e765b054590271 Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 29 Oct 2026 21:58:37 +0800 Subject: [PATCH 5/5] docs: list attach_only in the sandbox index row --- docs/sandboxd-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 369ca03e..98b16639 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -890,7 +890,7 @@ an SDK caller should set. Auth: node API token. Root sees every live claim; a tenant sees only its own. The index is `{"sandboxes": [{id, key, tenant?, deadline, claimed_at?, hibernated, archived?, from_checkpoint?, claim_ref?, metadata?, on_expire?, cpu_count, mem_total_bytes, -volumes?: [{name, mount?, mode?}], volumes_want?: [{name, mount?, mode?}], restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the +volumes?: [{name, mount?, mode?, attach_only?}], volumes_want?: [{name, mount?, mode?, attach_only?}], restarts?, restarted_at?, failed?}]}` — `cpu_count` and `mem_total_bytes` are the size tier's allocation; `restarts` counts the cold boots that replaced an exited VMM and `restarted_at` stamps the last one (each loses guest memory and the instance-metadata document, so a caller re-establishes guest state when