From 7ab8192830d729d407f02821047307b00f2506da Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 29 Oct 2026 17:41:27 +0800 Subject: [PATCH] test, docs: drop duplicate volume tests and the failover runbook Remove tests whose behavior another test already pins (claim-time writable plus live add, the detach-result write stage covered by the two-restart test, the reader-under-writer admission row, the direct capture refusals) and fold fork and the no-engine-call check into the capture test. The archive-failover runbook existed to explain the removed replica refusal; keep its one constraint. --- docs/deploy.md | 33 +++------------ sandboxd/pool/volume_capture_test.go | 36 ---------------- sandboxd/pool/volume_convergence_test.go | 7 +++- sandboxd/pool/volume_mutation_test.go | 53 +++--------------------- sandboxd/pool/volume_rw_test.go | 19 --------- sandboxd/server/volume_mutation_test.go | 3 -- 6 files changed, 16 insertions(+), 135 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 01114f986..7fd1fb892 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -713,33 +713,12 @@ The drain is not persisted — a restarted node serves again. ### Disabling archive failover -Stop admitting claims that use `on_expire: archive`. Set the node and affected -pools' `archive_after_seconds` to `0`, and use -[`renew`](sandboxd-api.md#post-v1sandboxesidrenew) to switch existing active claims -to `on_expire: destroy` while keeping their leases valid. These changes stop -idle-time and expiry-triggered archives. Before setting `archive_failover` to `false`: - -1. Set every affected pool's `replicate_seconds` to `0` and restart its nodes - with failover and the registry still enabled. Keep the registry reachable - while replica retirement retries. -2. Check each node's `/claims.json` for affected claims. Wait until - `replica_ck` and `replica_pending` are absent after successful journal writes. - Use `sync_claims: true` when completion must survive power loss. -3. Wake or release existing registered archives while failover remains enabled. - Wait for `archive_pending` to clear in the journal. Verify that the affected - IDs have no rows in PostgreSQL's - `sandboxd_archives` table for your `meta_store.cell`. A consumed row is a - non-adoptable tombstone; wait for its cleanup retries to finish too. -4. Set `archive_failover` to `false` and restart the nodes. - -The public API omits replica journal fields. A fixed delay or an absence of -warnings does not prove retirement. Disabling failover first stops replica -retirement and leaves residual references. -A local archive can still wake and clear `archive_ck`, but that does not clear a -retained `replica_pending` or retire an old shared row. Release can remove local -state without reconciling that row. Re-enabling failover against stale rows can -fence or drop a local claim if another node has adopted it. Resolve registry -ownership before serving that claim again. +Replicas and archive rows retire only through the registry. Before setting +`archive_failover` to `false`, set the affected pools' `replicate_seconds` to +`0` and restart with failover still on until the claims in `claims.json` carry +no `replica_ck`, `replica_pending` or `archive_pending`. Rows left behind are +not retired while failover is off, and turning it back on against them can +fence or drop a local claim that another node has adopted. ### Shared tenant database diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index 27c1b1dc7..5110e17e7 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -11,42 +11,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestVolumeClaimRejectsDirectCapture(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - sb := mustClaim(t, m, testKey) - sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - - tests := []struct { - name string - run func(*testing.T) error - }{ - {"direct hibernate", func(t *testing.T) error { return m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) }}, - {"fork", func(t *testing.T) error { - _, err := m.Fork(t.Context(), sb.ID, Cred{Token: sb.Token}, 1, time.Hour, "", "") - return err - }}, - {"checkpoint", func(t *testing.T) error { - _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "", "") - return err - }}, - {"promote", func(t *testing.T) error { - _, _, err := m.Promote(t.Context(), sb.ID, Cred{Token: sb.Token}, "volume-source", "") - return err - }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := tt.run(t); !errors.Is(err, ErrVolumeCapture) { - t.Errorf("error %v, want ErrVolumeCapture", err) - } - }) - } - if len(eng.hibernates) != 0 || len(eng.snapSaves) != 0 || len(eng.exports) != 0 { - t.Errorf("capture reached engine: hibernates=%v snapshots=%v exports=%v", eng.hibernates, eng.snapSaves, eng.exports) - } -} - func TestVolumeClaimRunningWakeUnchanged(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index db53172ac..6ce2456b7 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -343,8 +343,11 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("hibernate: %v", err) } - if len(eng.trims) != 0 { - t.Fatalf("capture trimmed held volume: %v", eng.trims) + if _, err := m.Fork(t.Context(), sb.ID, Cred{Token: sb.Token}, 1, time.Hour, "", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("fork: %v", err) + } + if len(eng.trims) != 0 || len(eng.hibernates) != 0 || len(eng.snapSaves) != 0 || len(eng.exports) != 0 { + t.Fatalf("capture reached the engine: trims=%v hibernates=%v snapshots=%v exports=%v", eng.trims, eng.hibernates, eng.snapSaves, eng.exports) } }) } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 19ce01c78..d41767fa2 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -119,32 +119,6 @@ func TestLiveVolumesCommitAndRetry(t *testing.T) { } } -func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { - for _, attachOnly := range []bool{false, true} { - t.Run(fmt.Sprintf("attach_only=%t", attachOnly), func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: "rw", AttachOnly: attachOnly}}}) - if err != nil { - t.Fatal(err) - } - original := sb.Volumes[0] - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Add: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatal(err) - } - if sb.Volumes[0] != original { - t.Fatalf("changed existing volume: %v -> %v", original, sb.Volumes[0]) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { - t.Fatalf("accepted mode change: %v", err) - } - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); err != nil || len(sb.Volumes) != 0 { - t.Fatalf("writable detach failed: %v", err) - } - }) - } -} - func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) @@ -209,49 +183,32 @@ func TestLiveAttachFailureSurvivesRestartAndCanBeDetached(t *testing.T) { } func TestLiveVolumePersistenceFailureRetainsRecoverableState(t *testing.T) { - for _, stage := range []string{"intent", "attach result", "detach result"} { + for _, stage := range []string{"intent", "attach result"} { t.Run(stage, func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) request := []types.Volume{{Name: "data"}} - if stage == "detach result" { - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err != nil { - t.Fatal(err) - } - } var restore func() if stage == "intent" { restore = breakClaimStore(t, m) } else { eng.liveVolumeHook = func(string) error { restore = breakClaimStore(t, m); return nil } } - var err error - if stage == "detach result" { - _, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) - } else { - _, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) - } - if err == nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request); err == nil { t.Fatal("expected persistence failure") } eng.liveVolumeHook = nil restore() assertLiveJournal(t, m, sb) - wantApplied := map[string]int{"intent": 0, "attach result": 1, "detach result": 0}[stage] - wantHolders := map[string]int{"intent": 1, "attach result": 1, "detach result": 0}[stage] - if len(sb.Volumes) != wantApplied || volumeHoldersOf(m, "data").readers != wantHolders { + wantApplied := map[string]int{"intent": 0, "attach result": 1}[stage] + if len(sb.Volumes) != wantApplied || volumeHoldersOf(m, "data").readers != 1 { t.Fatalf("failed write lost the physical result or its hold: volumes=%v holders=%+v", sb.Volumes, volumeHoldersOf(m, "data")) } if stage == "intent" && len(eng.volumeSpecs) != 0 { t.Fatal("failed intent reached the engine") } - var view types.SandboxVolumesResponse - if stage == "detach result" { - view, err = m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) - } else { - view, err = m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) - } + view, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, request) if err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index eaa5ee0ef..fd3c40407 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -317,25 +317,6 @@ func TestVolumeAdmissionMatrix(t *testing.T) { } } -func TestAdmitClaimExcludesReaderUnderWriter(t *testing.T) { - path := writeVolumeImage(t, "scratch.img", "data") - m := newVolumeManager(t, newFakeEngine(), []config.VolumeSpec{{Name: "scratch", Path: path, Writable: true}}) - resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}) - if err != nil { - t.Fatal(err) - } - m.mu.Lock() - m.adoptVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch", Mode: types.VolumeModeRW}}) - m.mu.Unlock() - - if err := m.admitClaim("", resolved, appliedVolumes(resolved)); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) - } - if holders := volumeHoldersOf(m, "scratch"); holders != (volumeHolders{writers: 1}) { - t.Errorf("registry after the refusal=%+v, want the writer alone", holders) - } -} - func TestVolumeAdmissionRefusalHoldsNothing(t *testing.T) { held := writeVolumeImage(t, "held.img", "held") free := writeVolumeImage(t, "free.img", "free") diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 528914909..8fcab0a8e 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -182,10 +182,7 @@ func TestClaimVolumeCompatibilityExpansion(t *testing.T) { want []types.Volume err bool }{ - {"default mount", ClaimRequest{Volumes: []types.Volume{{Name: "data"}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}}, false}, {"legacy all devices", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data"}, {Name: "work", Mode: types.VolumeModeRW}}}, []types.Volume{{Name: "data", AttachOnly: true}, {Name: "work", Mode: types.VolumeModeRW, AttachOnly: true}}, false}, - {"mixed", ClaimRequest{Volumes: []types.Volume{{Name: "data"}, {Name: "work", AttachOnly: true}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}, {Name: "work", AttachOnly: true}}, false}, - {"mixed reversed", ClaimRequest{Volumes: []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data"}}}, []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data", Mount: "/volumes/data"}}, false}, {"empty legacy", ClaimRequest{VolumesAttachOnly: true}, nil, true}, {"legacy mount conflict", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data", Mount: "/data"}}}, nil, true}, } {