From 0037d338c9911ed98ff63c4e074dacea6762cb34 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:02:19 +0800 Subject: [PATCH 01/53] metastore: lift the archive registry's compare-and-set query helpers into CAS --- sandboxd/archives/pg/pg.go | 50 +++++++++----------------------------- sandboxd/metastore/cas.go | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 38 deletions(-) create mode 100644 sandboxd/metastore/cas.go diff --git a/sandboxd/archives/pg/pg.go b/sandboxd/archives/pg/pg.go index 04afb9b2..33afe230 100644 --- a/sandboxd/archives/pg/pg.go +++ b/sandboxd/archives/pg/pg.go @@ -5,7 +5,6 @@ import ( "context" _ "embed" "errors" - "fmt" "time" "github.com/jackc/pgx/v5" @@ -29,6 +28,8 @@ var ( //go:embed nodes.sql nodesSchema string + + cas = metastore.CAS{Lost: archives.ErrLost, Unavailable: archives.ErrUnavailable} ) var _ archives.Registry = (*Registry)(nil) @@ -53,7 +54,7 @@ func Open(ctx context.Context, dsn, cell string) (*Registry, error) { } func (r *Registry) Put(ctx context.Context, rec archives.Record) (int64, error) { - return r.advance(ctx, `INSERT INTO sandboxd_archives (id, tenant, token_sha256, owner, epoch, checkpoint, deadline, claim, cell) + return cas.Advance(ctx, r.pool, `INSERT INTO sandboxd_archives (id, tenant, token_sha256, owner, epoch, checkpoint, deadline, claim, cell) VALUES ($1, $2, $3, $4, nextval('sandboxd_archives_epoch'), $5, $6, $7, $8) ON CONFLICT (cell, id) DO UPDATE SET tenant = EXCLUDED.tenant, token_sha256 = EXCLUDED.token_sha256, epoch = EXCLUDED.epoch, consumed = false, checkpoint = EXCLUDED.checkpoint, @@ -67,20 +68,20 @@ func (r *Registry) Replace(ctx context.Context, rec archives.Record, expect int6 if expect == 0 { return r.Put(ctx, rec) } - return r.advance(ctx, `UPDATE sandboxd_archives SET tenant = $5, token_sha256 = $6, epoch = nextval('sandboxd_archives_epoch'), + return cas.Advance(ctx, r.pool, `UPDATE sandboxd_archives SET tenant = $5, token_sha256 = $6, epoch = nextval('sandboxd_archives_epoch'), checkpoint = $7, deadline = $8, claim = $9, updated_at = now() WHERE cell = $1 AND id = $2 AND epoch = $3 AND owner = $4 AND NOT consumed RETURNING epoch`, r.cell, rec.ID, expect, rec.Owner, rec.Tenant, rec.TokenSHA256, rec.Checkpoint, pgtype.Timestamptz{Time: rec.Deadline, Valid: !rec.Deadline.IsZero()}, []byte(rec.Claim)) } func (r *Registry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { - return r.advance(ctx, `UPDATE sandboxd_archives SET owner = $4, epoch = nextval('sandboxd_archives_epoch'), updated_at = now() + return cas.Advance(ctx, r.pool, `UPDATE sandboxd_archives SET owner = $4, epoch = nextval('sandboxd_archives_epoch'), updated_at = now() WHERE cell = $1 AND id = $2 AND epoch = $3 AND NOT consumed AND (owner = $4 OR `+leaseExpired+`) RETURNING epoch`, r.cell, id, expect, owner) } func (r *Registry) Release(ctx context.Context, id string, expect int64, owner string) (int64, error) { - return r.advance(ctx, `UPDATE sandboxd_archives SET owner = '', epoch = nextval('sandboxd_archives_epoch'), updated_at = now() + return cas.Advance(ctx, r.pool, `UPDATE sandboxd_archives SET owner = '', epoch = nextval('sandboxd_archives_epoch'), updated_at = now() WHERE cell = $1 AND id = $2 AND epoch = $3 AND owner = $4 AND NOT consumed RETURNING epoch`, r.cell, id, expect, owner) } @@ -105,22 +106,13 @@ func (r *Registry) TakeOwned(ctx context.Context, owner string, expect map[strin } func (r *Registry) Consume(ctx context.Context, id string, expect int64) (int64, error) { - return r.advance(ctx, `UPDATE sandboxd_archives SET consumed = true, epoch = nextval('sandboxd_archives_epoch'), updated_at = now() + return cas.Advance(ctx, r.pool, `UPDATE sandboxd_archives SET consumed = true, epoch = nextval('sandboxd_archives_epoch'), updated_at = now() WHERE cell = $1 AND id = $2 AND epoch = $3 AND NOT consumed RETURNING epoch`, r.cell, id, expect) } func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - tag, err := r.pool.Exec(qctx, `DELETE FROM sandboxd_archives WHERE cell = $3 AND id = $1 AND epoch = $2`, id, epoch, r.cell) - switch { - case err != nil: - return unavailable(err) - case tag.RowsAffected() == 0: - return archives.ErrLost - } - return nil + return cas.Delete(ctx, r.pool, `DELETE FROM sandboxd_archives WHERE cell = $3 AND id = $1 AND epoch = $2`, id, epoch, r.cell) } func (r *Registry) Get(ctx context.Context, id string) (archives.Record, bool, error) { @@ -132,7 +124,7 @@ func (r *Registry) Get(ctx context.Context, id string) (archives.Record, bool, e case errors.Is(err, pgx.ErrNoRows): return archives.Record{}, false, nil case err != nil: - return archives.Record{}, false, unavailable(err) + return archives.Record{}, false, cas.Down(err) } return rec, true, nil } @@ -162,7 +154,7 @@ func (r *Registry) Renew(ctx context.Context, node string, ttl time.Duration) er VALUES ($1, $2, now() + $3::float8 * interval '1 second') ON CONFLICT (cell, node) DO UPDATE SET lease_until = EXCLUDED.lease_until`, r.cell, node, ttl.Seconds()) if err != nil { - return unavailable(err) + return cas.Down(err) } return nil } @@ -172,30 +164,16 @@ func (r *Registry) Close() { r.pool.Close() } -func (r *Registry) advance(ctx context.Context, sql string, args ...any) (int64, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - var epoch int64 - err := r.pool.QueryRow(qctx, sql, args...).Scan(&epoch) - switch { - case errors.Is(err, pgx.ErrNoRows): - return 0, archives.ErrLost - case err != nil: - return 0, unavailable(err) - } - return epoch, nil -} - func (r *Registry) collect(ctx context.Context, sql string, args ...any) ([]archives.Record, error) { qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) defer cancel() rows, err := r.pool.Query(qctx, sql, args...) if err != nil { - return nil, unavailable(err) + return nil, cas.Down(err) } recs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (archives.Record, error) { return scan(row) }) if err != nil { - return nil, unavailable(err) + return nil, cas.Down(err) } return recs, nil } @@ -212,7 +190,3 @@ func scan(row pgx.Row) (archives.Record, error) { rec.Deadline, rec.Claim = deadline.Time, claim return rec, nil } - -func unavailable(err error) error { - return fmt.Errorf("%w: %w", archives.ErrUnavailable, err) -} diff --git a/sandboxd/metastore/cas.go b/sandboxd/metastore/cas.go new file mode 100644 index 00000000..d4313214 --- /dev/null +++ b/sandboxd/metastore/cas.go @@ -0,0 +1,50 @@ +package metastore + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// CAS runs a registry's compare-and-set queries and answers with its sentinels. +type CAS struct { + Lost error + Unavailable error +} + +// Advance runs sql, which returns the new epoch of the one row it moves; Lost when it moves none. +func (c CAS) Advance(ctx context.Context, pool *pgxpool.Pool, sql string, args ...any) (int64, error) { + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + var epoch int64 + err := pool.QueryRow(qctx, sql, args...).Scan(&epoch) + switch { + case errors.Is(err, pgx.ErrNoRows): + return 0, c.Lost + case err != nil: + return 0, c.Down(err) + } + return epoch, nil +} + +// Delete runs sql, which removes the one row it matches; Lost when it removes none. +func (c CAS) Delete(ctx context.Context, pool *pgxpool.Pool, sql string, args ...any) error { + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + tag, err := pool.Exec(qctx, sql, args...) + switch { + case err != nil: + return c.Down(err) + case tag.RowsAffected() == 0: + return c.Lost + } + return nil +} + +// Down wraps err in Unavailable. +func (c CAS) Down(err error) error { + return fmt.Errorf("%w: %w", c.Unavailable, err) +} From 9cd27ee369b899c3bcb8864a5863a367a9b7e414 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:07:14 +0800 Subject: [PATCH 02/53] moves: a record of its own for a live move, with a PostgreSQL store and a contract suite --- sandboxd/moves/moves.go | 65 ++++ sandboxd/moves/movestest/movestest.go | 366 +++++++++++++++++++++ sandboxd/moves/movestest/movestest_test.go | 11 + sandboxd/moves/pg/pg.go | 103 ++++++ sandboxd/moves/pg/pg_test.go | 74 +++++ sandboxd/moves/pg/schema.sql | 11 + 6 files changed, 630 insertions(+) create mode 100644 sandboxd/moves/moves.go create mode 100644 sandboxd/moves/movestest/movestest.go create mode 100644 sandboxd/moves/movestest/movestest_test.go create mode 100644 sandboxd/moves/pg/pg.go create mode 100644 sandboxd/moves/pg/pg_test.go create mode 100644 sandboxd/moves/pg/schema.sql diff --git a/sandboxd/moves/moves.go b/sandboxd/moves/moves.go new file mode 100644 index 00000000..3ba42d77 --- /dev/null +++ b/sandboxd/moves/moves.go @@ -0,0 +1,65 @@ +// Package moves keeps one cluster-wide record per live move of a running claim from a source node to a target node. +package moves + +import ( + "context" + "errors" + "fmt" + "time" +) + +const ( + Pending State = "pending" + Final State = "final" + Committed State = "committed" + Aborted State = "aborted" +) + +var ( + // ErrUnavailable answers a call the registry cannot serve now; a retry may succeed. + ErrUnavailable = errors.New("move registry down") + // ErrLost answers a compare-and-set whose record is gone, at another epoch, or in another state. + ErrLost = errors.New("move record moved on") + + advances = map[[2]State]bool{ + {Pending, Final}: true, + {Final, Committed}: true, + {Pending, Aborted}: true, + {Final, Aborted}: true, + } +) + +// State is where a move stands; Committed and Aborted end it. +type State string + +// Record is one claim's move. +type Record struct { + ID string + From string + To string + State State + Epoch int64 + UpdatedAt time.Time +} + +// Registry keeps the records; every write is a compare-and-set, and every epoch it hands out is new. +type Registry interface { + // Begin writes a Pending record for id at a new epoch and returns it; ErrLost when a Pending or Final record holds id. + Begin(ctx context.Context, id, from, to string) (int64, error) + // Advance moves the record at epoch expect in state from to state to at a new epoch and returns it. + Advance(ctx context.Context, id string, expect int64, from, to State) (int64, error) + // Get returns the record of id; false when there is none. + Get(ctx context.Context, id string) (Record, bool, error) + // Delete removes the record only while it is at epoch. + Delete(ctx context.Context, id string, epoch int64) error + // Stale returns the records last written before before. + Stale(ctx context.Context, before time.Time) ([]Record, error) +} + +// VerifyAdvance refuses a state change no move makes. +func VerifyAdvance(from, to State) error { + if !advances[[2]State{from, to}] { + return fmt.Errorf("move state %s cannot advance to %s", from, to) + } + return nil +} diff --git a/sandboxd/moves/movestest/movestest.go b/sandboxd/moves/movestest/movestest.go new file mode 100644 index 00000000..5a69e10c --- /dev/null +++ b/sandboxd/moves/movestest/movestest.go @@ -0,0 +1,366 @@ +// Package movestest is the contract every moves.Registry passes, and an in-memory registry for tests. +package movestest + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/moves" +) + +const racers = 8 + +var states = []moves.State{moves.Pending, moves.Final, moves.Committed, moves.Aborted} + +// Opener returns an empty registry; each call is a fresh table. +type Opener func(t *testing.T) moves.Registry + +var _ moves.Registry = (*Mem)(nil) + +// Mem is an in-memory registry. +type Mem struct { + mu sync.Mutex + recs map[string]moves.Record + seq int64 + down bool +} + +// NewMem returns an empty in-memory registry. +func NewMem() *Mem { + return &Mem{recs: map[string]moves.Record{}} +} + +func (m *Mem) Begin(_ context.Context, id, from, to string) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return 0, moves.ErrUnavailable + } + if old, ok := m.recs[id]; ok && old.State != moves.Committed && old.State != moves.Aborted { + return 0, moves.ErrLost + } + return m.write(moves.Record{ID: id, From: from, To: to, State: moves.Pending}), nil +} + +func (m *Mem) Advance(_ context.Context, id string, expect int64, from, to moves.State) (int64, error) { + if err := moves.VerifyAdvance(from, to); err != nil { + return 0, err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return 0, moves.ErrUnavailable + } + rec, ok := m.recs[id] + if !ok || rec.Epoch != expect || rec.State != from { + return 0, moves.ErrLost + } + rec.State = to + return m.write(rec), nil +} + +func (m *Mem) Get(_ context.Context, id string) (moves.Record, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return moves.Record{}, false, moves.ErrUnavailable + } + rec, ok := m.recs[id] + return rec, ok, nil +} + +func (m *Mem) Delete(_ context.Context, id string, epoch int64) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return moves.ErrUnavailable + } + if rec, ok := m.recs[id]; !ok || rec.Epoch != epoch { + return moves.ErrLost + } + delete(m.recs, id) + return nil +} + +func (m *Mem) Stale(_ context.Context, before time.Time) ([]moves.Record, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return nil, moves.ErrUnavailable + } + var out []moves.Record + for _, rec := range m.recs { + if rec.UpdatedAt.Before(before) { + out = append(out, rec) + } + } + return out, nil +} + +// SetDown makes every call answer ErrUnavailable while down is set. +func (m *Mem) SetDown(down bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.down = down +} + +func (m *Mem) write(rec moves.Record) int64 { + m.seq++ + rec.Epoch, rec.UpdatedAt = m.seq, time.Now() + m.recs[rec.ID] = rec + return rec.Epoch +} + +// Contract checks the behavior every backend shares. +func Contract(t *testing.T, open Opener) { + t.Run("begin get", func(t *testing.T) { testBeginGet(t, open(t)) }) + t.Run("every allowed advance", func(t *testing.T) { testAdvance(t, open(t)) }) + t.Run("advance is a compare-and-set", func(t *testing.T) { testAdvanceFences(t, open(t)) }) + t.Run("advance refuses a state change no move makes", func(t *testing.T) { testAdvanceRefusesPairs(t, open(t)) }) + t.Run("one of concurrent advances wins", func(t *testing.T) { testAdvanceRace(t, open(t)) }) + t.Run("begin replaces an ended record", func(t *testing.T) { testBeginOverEnded(t, open(t)) }) + t.Run("begin refuses a live record", func(t *testing.T) { testBeginOverLive(t, open(t)) }) + t.Run("delete only at its epoch", func(t *testing.T) { testDelete(t, open(t)) }) + t.Run("epochs are never reused", func(t *testing.T) { testEpochsNeverRepeat(t, open(t)) }) + t.Run("stale lists records written before a time", func(t *testing.T) { testStale(t, open(t)) }) +} + +func testBeginGet(t *testing.T, r moves.Registry) { + ctx := t.Context() + if _, ok, err := r.Get(ctx, "sb_1"); ok || err != nil { + t.Fatalf("get before begin: %v %v, want none", ok, err) + } + epoch := mustBegin(t, r, "sb_1") + got, ok, err := r.Get(ctx, "sb_1") + if err != nil || !ok { + t.Fatalf("get: %v %v", ok, err) + } + if got.ID != "sb_1" || got.From != "s" || got.To != "t" || got.State != moves.Pending || got.Epoch != epoch || got.UpdatedAt.IsZero() { + t.Errorf("get %+v, want s to t pending at epoch %d", got, epoch) + } +} + +func testAdvance(t *testing.T, r moves.Registry) { + ctx := t.Context() + for i, path := range [][]moves.State{ + {moves.Final, moves.Committed}, + {moves.Aborted}, + {moves.Final, moves.Aborted}, + } { + id := "sb_" + strconv.Itoa(i) + epoch := mustBegin(t, r, id) + from := moves.Pending + for _, to := range path { + next, err := r.Advance(ctx, id, epoch, from, to) + if err != nil || next <= epoch { + t.Fatalf("%s advance %s to %s: %d %v, want an epoch above %d", id, from, to, next, err, epoch) + } + epoch, from = next, to + if got, _, _ := r.Get(ctx, id); got.State != to || got.Epoch != epoch || got.From != "s" || got.To != "t" { + t.Fatalf("%s after advance: %+v, want %s at %d", id, got, to, epoch) + } + } + } +} + +func testAdvanceFences(t *testing.T, r moves.Registry) { + ctx := t.Context() + if _, err := r.Advance(ctx, "sb_none", 1, moves.Pending, moves.Final); !errors.Is(err, moves.ErrLost) { + t.Errorf("advance of no record: %v, want ErrLost", err) + } + first := mustBegin(t, r, "sb_1") + if _, err := r.Advance(ctx, "sb_1", first-1, moves.Pending, moves.Final); !errors.Is(err, moves.ErrLost) { + t.Errorf("advance at a stale epoch: %v, want ErrLost", err) + } + if _, err := r.Advance(ctx, "sb_1", first, moves.Final, moves.Committed); !errors.Is(err, moves.ErrLost) { + t.Errorf("advance from a state the record is not in: %v, want ErrLost", err) + } + final := mustAdvance(t, r, "sb_1", first, moves.Pending, moves.Final) + if _, err := r.Advance(ctx, "sb_1", first, moves.Pending, moves.Aborted); !errors.Is(err, moves.ErrLost) { + t.Errorf("abort at the superseded epoch: %v, want ErrLost", err) + } + if got, _, _ := r.Get(ctx, "sb_1"); got.State != moves.Final || got.Epoch != final { + t.Errorf("record after refused advances: %s@%d, want final@%d", got.State, got.Epoch, final) + } + committed := mustAdvance(t, r, "sb_1", final, moves.Final, moves.Committed) + if _, err := r.Advance(ctx, "sb_1", committed, moves.Final, moves.Aborted); !errors.Is(err, moves.ErrLost) { + t.Errorf("abort of a committed move: %v, want ErrLost", err) + } + if _, ok, _ := r.Get(ctx, "sb_none"); ok { + t.Errorf("a refused advance inserted a record") + } +} + +func testAdvanceRefusesPairs(t *testing.T, r moves.Registry) { + ctx := t.Context() + for _, from := range states { + for _, to := range states { + if moves.VerifyAdvance(from, to) == nil { + continue + } + id := "sb_" + string(from) + "_" + string(to) + epoch := mustReach(t, r, id, from) + _, err := r.Advance(ctx, id, epoch, from, to) + if err == nil || errors.Is(err, moves.ErrLost) || errors.Is(err, moves.ErrUnavailable) { + t.Errorf("advance %s to %s: %v, want a plain error", from, to, err) + } + if got, _, _ := r.Get(ctx, id); got.State != from || got.Epoch != epoch { + t.Errorf("%s after a refused pair: %s@%d, want %s@%d", id, got.State, got.Epoch, from, epoch) + } + } + } +} + +func testAdvanceRace(t *testing.T, r moves.Registry) { + ctx := t.Context() + final := mustAdvance(t, r, "sb_1", mustBegin(t, r, "sb_1"), moves.Pending, moves.Final) + var ( + mu sync.Mutex + wg sync.WaitGroup + won []int64 + ends []moves.State + lost int + ) + for i := range racers { + end := moves.Committed + if i%2 == 1 { + end = moves.Aborted + } + wg.Go(func() { + epoch, err := r.Advance(ctx, "sb_1", final, moves.Final, end) + mu.Lock() + defer mu.Unlock() + switch { + case err == nil: + won, ends = append(won, epoch), append(ends, end) + case errors.Is(err, moves.ErrLost): + lost++ + default: + t.Errorf("advance: %v", err) + } + }) + } + wg.Wait() + if len(won) != 1 || lost != racers-1 { + t.Fatalf("%d advances won and %d lost, want 1 and %d", len(won), lost, racers-1) + } + if got, _, _ := r.Get(ctx, "sb_1"); got.Epoch != won[0] || got.State != ends[0] { + t.Errorf("record after the race %s@%d, want the winner's %s@%d", got.State, got.Epoch, ends[0], won[0]) + } +} + +func testBeginOverEnded(t *testing.T, r moves.Registry) { + ctx := t.Context() + for _, end := range []moves.State{moves.Committed, moves.Aborted} { + id := "sb_" + string(end) + ended := mustReach(t, r, id, end) + again, err := r.Begin(ctx, id, "t", "u") + if err != nil || again <= ended { + t.Fatalf("begin over %s: %d %v, want an epoch above %d", end, again, err, ended) + } + if got, _, _ := r.Get(ctx, id); got.State != moves.Pending || got.Epoch != again || got.From != "t" || got.To != "u" { + t.Errorf("record after begin over %s: %+v, want t to u pending at %d", end, got, again) + } + } +} + +func testBeginOverLive(t *testing.T, r moves.Registry) { + ctx := t.Context() + for _, live := range []moves.State{moves.Pending, moves.Final} { + id := "sb_" + string(live) + epoch := mustReach(t, r, id, live) + if _, err := r.Begin(ctx, id, "t", "u"); !errors.Is(err, moves.ErrLost) { + t.Errorf("begin over %s: %v, want ErrLost", live, err) + } + if got, _, _ := r.Get(ctx, id); got.State != live || got.Epoch != epoch || got.From != "s" || got.To != "t" { + t.Errorf("record after a refused begin over %s: %+v, want s to t %s at %d", live, got, live, epoch) + } + } +} + +func testDelete(t *testing.T, r moves.Registry) { + ctx := t.Context() + if err := r.Delete(ctx, "sb_1", 1); !errors.Is(err, moves.ErrLost) { + t.Errorf("delete of no record: %v, want ErrLost", err) + } + first := mustBegin(t, r, "sb_1") + aborted := mustAdvance(t, r, "sb_1", first, moves.Pending, moves.Aborted) + if err := r.Delete(ctx, "sb_1", first); !errors.Is(err, moves.ErrLost) { + t.Errorf("delete at a stale epoch: %v, want ErrLost", err) + } + if err := r.Delete(ctx, "sb_1", aborted); err != nil { + t.Fatalf("delete at the epoch: %v", err) + } + if _, ok, err := r.Get(ctx, "sb_1"); ok || err != nil { + t.Errorf("get after delete: %v %v, want none", ok, err) + } +} + +func testEpochsNeverRepeat(t *testing.T, r moves.Registry) { + ctx := t.Context() + first := mustBegin(t, r, "sb_1") + if err := r.Delete(ctx, "sb_1", first); err != nil { + t.Fatalf("delete: %v", err) + } + if again := mustBegin(t, r, "sb_1"); again <= first { + t.Errorf("begin after delete at epoch %d, want above %d", again, first) + } + if other := mustBegin(t, r, "sb_2"); other <= first { + t.Errorf("another record at epoch %d, want above %d", other, first) + } +} + +func testStale(t *testing.T, r moves.Registry) { + ctx := t.Context() + mustBegin(t, r, "sb_1") + mustReach(t, r, "sb_2", moves.Committed) + if got, err := r.Stale(ctx, time.Now().Add(-time.Hour)); err != nil || len(got) != 0 { + t.Errorf("stale before the records were written: %+v %v, want none", got, err) + } + got, err := r.Stale(ctx, time.Now().Add(time.Hour)) + ids := map[string]bool{} + for _, rec := range got { + ids[rec.ID] = rec.From == "s" && rec.To == "t" && rec.Epoch > 0 + } + if err != nil || len(got) != 2 || !ids["sb_1"] || !ids["sb_2"] { + t.Errorf("stale after the records were written: %+v %v, want sb_1 and sb_2", got, err) + } +} + +func mustBegin(t *testing.T, r moves.Registry, id string) int64 { + t.Helper() + epoch, err := r.Begin(t.Context(), id, "s", "t") + if err != nil || epoch <= 0 { + t.Fatalf("begin %s: %d %v", id, epoch, err) + } + return epoch +} + +func mustAdvance(t *testing.T, r moves.Registry, id string, expect int64, from, to moves.State) int64 { + t.Helper() + epoch, err := r.Advance(t.Context(), id, expect, from, to) + if err != nil || epoch <= expect { + t.Fatalf("advance %s from %s to %s: %d %v", id, from, to, epoch, err) + } + return epoch +} + +func mustReach(t *testing.T, r moves.Registry, id string, state moves.State) int64 { + t.Helper() + epoch := mustBegin(t, r, id) + path := map[moves.State][]moves.State{ + moves.Final: {moves.Final}, + moves.Committed: {moves.Final, moves.Committed}, + moves.Aborted: {moves.Aborted}, + }[state] + from := moves.Pending + for _, to := range path { + epoch = mustAdvance(t, r, id, epoch, from, to) + from = to + } + return epoch +} diff --git a/sandboxd/moves/movestest/movestest_test.go b/sandboxd/moves/movestest/movestest_test.go new file mode 100644 index 00000000..9c1d74df --- /dev/null +++ b/sandboxd/moves/movestest/movestest_test.go @@ -0,0 +1,11 @@ +package movestest + +import ( + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/moves" +) + +func TestMemContract(t *testing.T) { + Contract(t, func(*testing.T) moves.Registry { return NewMem() }) +} diff --git a/sandboxd/moves/pg/pg.go b/sandboxd/moves/pg/pg.go new file mode 100644 index 00000000..0ffdf787 --- /dev/null +++ b/sandboxd/moves/pg/pg.go @@ -0,0 +1,103 @@ +// Package pg keeps the move registry in one shared PostgreSQL table. +package pg + +import ( + "context" + _ "embed" + "errors" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/cocoonstack/sandbox/sandboxd/metastore" + "github.com/cocoonstack/sandbox/sandboxd/moves" +) + +const table = "sandboxd_moves" + +var ( + //go:embed schema.sql + schema string + + cas = metastore.CAS{Lost: moves.ErrLost, Unavailable: moves.ErrUnavailable} +) + +var _ moves.Registry = (*Registry)(nil) + +// Registry is the shared move table. +type Registry struct { + pool *pgxpool.Pool + cell string +} + +// Open connects to cell's records and creates the table when it is missing. +func Open(ctx context.Context, dsn, cell string) (*Registry, error) { + pool, _, err := metastore.Open(ctx, dsn, table, schema) + if err != nil { + return nil, err + } + return &Registry{pool: pool, cell: cell}, nil +} + +func (r *Registry) Begin(ctx context.Context, id, from, to string) (int64, error) { + return cas.Advance(ctx, r.pool, `INSERT INTO sandboxd_moves (cell, id, from_node, to_node, state, epoch) + VALUES ($1, $2, $3, $4, 'pending', nextval('sandboxd_moves_epoch')) + ON CONFLICT (cell, id) DO UPDATE SET from_node = EXCLUDED.from_node, to_node = EXCLUDED.to_node, + state = EXCLUDED.state, epoch = EXCLUDED.epoch, updated_at = now() + WHERE sandboxd_moves.state IN ('committed', 'aborted') + RETURNING epoch`, r.cell, id, from, to) +} + +func (r *Registry) Advance(ctx context.Context, id string, expect int64, from, to moves.State) (int64, error) { + if err := moves.VerifyAdvance(from, to); err != nil { + return 0, err + } + return cas.Advance(ctx, r.pool, `UPDATE sandboxd_moves SET state = $5, epoch = nextval('sandboxd_moves_epoch'), updated_at = now() + WHERE cell = $1 AND id = $2 AND epoch = $3 AND state = $4 + RETURNING epoch`, r.cell, id, expect, from, to) +} + +func (r *Registry) Get(ctx context.Context, id string) (moves.Record, bool, error) { + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + defer cancel() + rec, err := scan(r.pool.QueryRow(qctx, `SELECT id, from_node, to_node, state, epoch, updated_at + FROM sandboxd_moves WHERE cell = $1 AND id = $2`, r.cell, id)) + switch { + case errors.Is(err, pgx.ErrNoRows): + return moves.Record{}, false, nil + case err != nil: + return moves.Record{}, false, cas.Down(err) + } + return rec, true, nil +} + +func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { + return cas.Delete(ctx, r.pool, `DELETE FROM sandboxd_moves WHERE cell = $1 AND id = $2 AND epoch = $3`, r.cell, id, epoch) +} + +func (r *Registry) Stale(ctx context.Context, before time.Time) ([]moves.Record, error) { + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + defer cancel() + rows, err := r.pool.Query(qctx, `SELECT id, from_node, to_node, state, epoch, updated_at + FROM sandboxd_moves WHERE cell = $1 AND updated_at < $2`, r.cell, before) + if err != nil { + return nil, cas.Down(err) + } + recs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (moves.Record, error) { return scan(row) }) + if err != nil { + return nil, cas.Down(err) + } + return recs, nil +} + +// Close releases the connection pool. +func (r *Registry) Close() { + r.pool.Close() +} + +func scan(row pgx.Row) (moves.Record, error) { + var rec moves.Record + err := row.Scan(&rec.ID, &rec.From, &rec.To, &rec.State, &rec.Epoch, &rec.UpdatedAt) + return rec, err +} diff --git a/sandboxd/moves/pg/pg_test.go b/sandboxd/moves/pg/pg_test.go new file mode 100644 index 00000000..528c2ba8 --- /dev/null +++ b/sandboxd/moves/pg/pg_test.go @@ -0,0 +1,74 @@ +package pg + +import ( + "errors" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest" + "github.com/cocoonstack/sandbox/sandboxd/moves" + "github.com/cocoonstack/sandbox/sandboxd/moves/movestest" +) + +func TestRegistryContract(t *testing.T) { + movestest.Contract(t, func(t *testing.T) moves.Registry { return open(t, metastoretest.PGSchema(t), "c1") }) +} + +func TestCellsKeepTheirOwnRecords(t *testing.T) { + dsn := metastoretest.PGSchema(t) + c1, c2 := open(t, dsn, "c1"), open(t, dsn, "c2") + ctx := t.Context() + epoch, err := c1.Begin(ctx, "sb_1", "s", "t") + if err != nil { + t.Fatalf("begin in c1: %v", err) + } + if _, ok, getErr := c2.Get(ctx, "sb_1"); ok || getErr != nil { + t.Errorf("c2 sees c1's record: %v %v", ok, getErr) + } + if _, err = c2.Advance(ctx, "sb_1", epoch, moves.Pending, moves.Aborted); !errors.Is(err, moves.ErrLost) { + t.Errorf("c2 aborts c1's move: %v, want ErrLost", err) + } + if err = c2.Delete(ctx, "sb_1", epoch); !errors.Is(err, moves.ErrLost) { + t.Errorf("c2 deletes c1's record: %v, want ErrLost", err) + } + if recs, listErr := c2.Stale(ctx, time.Now().Add(time.Hour)); listErr != nil || len(recs) != 0 { + t.Errorf("c2 lists c1's record as stale: %+v %v", recs, listErr) + } + if _, err = c2.Begin(ctx, "sb_1", "u", "v"); err != nil { + t.Errorf("begin of the same id in c2: %v, want its own record", err) + } + if rec, _, _ := c1.Get(ctx, "sb_1"); rec.Epoch != epoch || rec.State != moves.Pending || rec.From != "s" { + t.Errorf("c1's record after c2's begin: %+v, want s pending at %d", rec, epoch) + } +} + +func TestADownDatabaseIsUnavailable(t *testing.T) { + r := open(t, metastoretest.PGSchema(t), "c1") + r.Close() + ctx := t.Context() + if _, err := r.Begin(ctx, "sb_1", "s", "t"); !errors.Is(err, moves.ErrUnavailable) { + t.Errorf("begin on a closed pool: %v, want ErrUnavailable", err) + } + if _, err := r.Advance(ctx, "sb_1", 1, moves.Pending, moves.Final); !errors.Is(err, moves.ErrUnavailable) { + t.Errorf("advance on a closed pool: %v, want ErrUnavailable", err) + } + if _, _, err := r.Get(ctx, "sb_1"); !errors.Is(err, moves.ErrUnavailable) { + t.Errorf("get on a closed pool: %v, want ErrUnavailable", err) + } + if err := r.Delete(ctx, "sb_1", 1); !errors.Is(err, moves.ErrUnavailable) { + t.Errorf("delete on a closed pool: %v, want ErrUnavailable", err) + } + if _, err := r.Stale(ctx, time.Now()); !errors.Is(err, moves.ErrUnavailable) { + t.Errorf("stale on a closed pool: %v, want ErrUnavailable", err) + } +} + +func open(t *testing.T, dsn, cell string) *Registry { + t.Helper() + r, err := Open(t.Context(), dsn, cell) + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(r.Close) + return r +} diff --git a/sandboxd/moves/pg/schema.sql b/sandboxd/moves/pg/schema.sql new file mode 100644 index 00000000..bde23c02 --- /dev/null +++ b/sandboxd/moves/pg/schema.sql @@ -0,0 +1,11 @@ +CREATE SEQUENCE IF NOT EXISTS sandboxd_moves_epoch; +CREATE TABLE IF NOT EXISTS sandboxd_moves ( + cell text NOT NULL, + id text NOT NULL, + from_node text NOT NULL, + to_node text NOT NULL, + state text NOT NULL CHECK (state IN ('pending', 'final', 'committed', 'aborted')), + epoch bigint NOT NULL CHECK (epoch > 0), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (cell, id) +); From 985af4de1ff5234d28214f7f63e0d2e9cfe6ad86 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:12:22 +0800 Subject: [PATCH 03/53] engine: drive cocoon's live migration capture, receive and abort --- sandboxd/engine/engine.go | 34 +++++++++---- sandboxd/engine/engine_test.go | 15 +++++- sandboxd/engine/migrate.go | 61 +++++++++++++++++++++++ sandboxd/engine/migrate_test.go | 86 +++++++++++++++++++++++++++++++++ sandboxd/engine/version_test.go | 30 ++++++++---- sandboxd/engine/volume_live.go | 7 +-- sandboxd/types/types.go | 15 ++++++ 7 files changed, 221 insertions(+), 27 deletions(-) create mode 100644 sandboxd/engine/migrate.go create mode 100644 sandboxd/engine/migrate_test.go diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index ad5432fe..1373f505 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -34,6 +34,8 @@ const ( RequiredCocoon = "v0.6.9" // LiveVolumeCocoon adds idempotent disk attach and detach for live volume changes. LiveVolumeCocoon = "v0.7.2" + // MigrateCocoon adds live migration: capture rounds, receive and abort. + MigrateCocoon = "v0.7.7" StaleCreateCollected StaleCreateOutcome = "collected" StaleCreateBusy StaleCreateOutcome = "busy" @@ -167,7 +169,7 @@ func (e *Engine) Remove(ctx context.Context, name string) error { return err } -// Start cold-boots a stopped VM from its own disk; guest memory starts empty. +// Start cold-boots a stopped VM from its own disk, or resumes the paused guest of a received VM. func (e *Engine) Start(ctx context.Context, name string) error { _, err := e.run(ctx, "vm", "start", name) return err @@ -196,12 +198,14 @@ func (e *Engine) ReconcileStaleCreate(ctx context.Context, name string) (StaleCr // SnapshotSave snapshots a running VM under snapName. func (e *Engine) SnapshotSave(ctx context.Context, vmName, snapName string) error { - return e.runDump(ctx, "snapshot", "save", argName, snapName, vmName) + _, err := e.runDump(ctx, "snapshot", "save", argName, snapName, vmName) + return err } // Hibernate atomically snapshots a running VM under snapName and stops it. func (e *Engine) Hibernate(ctx context.Context, vmName, snapName string) error { - return e.runDump(ctx, "vm", "hibernate", argName, snapName, vmName) + _, err := e.runDump(ctx, "vm", "hibernate", argName, snapName, vmName) + return err } // Restore resumes a VM from a snapshot, returning its vsock UDS. @@ -383,9 +387,12 @@ func (e *Engine) probe(ctx context.Context, vsockSocket string, port int, timeou } func (e *Engine) cloneArgs(name string, key types.PoolKey, source ...string) []string { - args := append([]string{"vm", "clone"}, source...) + return append(append([]string{"vm", "clone"}, source...), e.cloneFlags(name, key)...) +} + +func (e *Engine) cloneFlags(name string, key types.PoolKey) []string { // --pull: a checkpoint/template export carries only COW + memory, not the base image. - args = append(args, argName, name, "--pull", argOutput, formatJSON, e.directIOArg()) + args := []string{argName, name, "--pull", argOutput, formatJSON, e.directIOArg()} args = append(args, e.restoreArgs()...) return append(args, e.netArgs(name, key, false)...) } @@ -465,15 +472,14 @@ func (e *Engine) begin() bool { return !e.closed } -func (e *Engine) runDump(ctx context.Context, args ...string) error { +func (e *Engine) runDump(ctx context.Context, args ...string) ([]byte, error) { select { case e.dumps <- struct{}{}: case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() } defer func() { <-e.dumps }() - _, err := e.runFor(ctx, dumpTimeout, saveTermGrace, args...) - return err + return e.runFor(ctx, dumpTimeout, saveTermGrace, args...) } func (e *Engine) runRecord(ctx context.Context, timeout, termGrace time.Duration, args ...string) (types.VMRecord, error) { @@ -516,6 +522,16 @@ func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string, port int return nil } +func (e *Engine) missesFloor(ctx context.Context, floor string) (string, bool) { + version, err := e.Version(ctx) + if err != nil { + log.WithFunc("engine.missesFloor").Warnf(ctx, "query cocoon version: %v", err) + return "", true + } + below, known := belowFloor(version, floor) + return version, known && below +} + // EgressSocketPath is the host UDS the VMM connects when the guest dials CID2:egressPort. func EgressSocketPath(vsockSocket string) string { return fmt.Sprintf("%s_%d", vsockSocket, egressPort) diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index d70d6026..64528423 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -259,6 +259,14 @@ func TestAMemoryCommandGetsItsLongerTermGraceAndAColdBootTheCommandOne(t *testin {"snapshot save", func(ctx context.Context, e *Engine) error { return e.SnapshotSave(ctx, "vm", "snap") }, "exit status 3"}, + {"migrate capture", func(ctx context.Context, e *Engine) error { + _, err := e.MigrateCapture(ctx, "vm", "s1", true) + return err + }, "exit status 3"}, + {"migrate receive", func(ctx context.Context, e *Engine) error { + _, err := e.MigrateReceive(ctx, "dir", "vm", types.PoolKey{}) + return err + }, "exit status 3"}, {"cold boot", func(ctx context.Context, e *Engine) error { _, err := e.RunCold(ctx, "vm", types.PoolKey{}, "") return err @@ -345,10 +353,13 @@ func TestMemoryDumpsRunAtMostEightAtOnce(t *testing.T) { var wg sync.WaitGroup for i := range 40 { wg.Go(func() { - if i%2 == 0 { + switch i % 3 { + case 0: _ = e.Hibernate(t.Context(), "vm", "snap") - } else { + case 1: _ = e.SnapshotSave(t.Context(), "vm", "snap") + default: + _, _ = e.MigrateCapture(t.Context(), "vm", "", false) } }) } diff --git a/sandboxd/engine/migrate.go b/sandboxd/engine/migrate.go new file mode 100644 index 00000000..60628da9 --- /dev/null +++ b/sandboxd/engine/migrate.go @@ -0,0 +1,61 @@ +package engine + +import ( + "cmp" + "context" + "encoding/json/v2" + "fmt" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const argSeries = "--series" + +// MigrateUnsupported reports a runtime without cocoon's live migration. +type MigrateUnsupported struct { + Version string +} + +func (e *MigrateUnsupported) Error() string { + return fmt.Sprintf("live migration requires Cocoon >= %s; installed version is %s", MigrateCocoon, cmp.Or(e.Version, "unavailable")) +} + +// CheckMigrateSupport rejects runtimes without live migration. +func (e *Engine) CheckMigrateSupport(ctx context.Context) error { + if version, misses := e.missesFloor(ctx, MigrateCocoon); misses { + return &MigrateUnsupported{Version: version} + } + return nil +} + +// MigrateCapture captures one round of a VM's series; an empty series starts one, and final leaves the guest paused. +func (e *Engine) MigrateCapture(ctx context.Context, vmName, series string, final bool) (types.MigrateRound, error) { + args := []string{"vm", "migrate", "capture", vmName} + if series != "" { + args = append(args, argSeries, series) + } + if final { + args = append(args, "--final") + } + out, err := e.runDump(ctx, append(args, argOutput, formatJSON)...) + if err != nil { + return types.MigrateRound{}, err + } + var round types.MigrateRound + if err := json.Unmarshal(out, &round); err != nil { + return types.MigrateRound{}, fmt.Errorf("parse migrate capture: %w", err) + } + return round, nil +} + +// MigrateReceive restores a transferred series into a VM whose guest stays paused until Start. +func (e *Engine) MigrateReceive(ctx context.Context, fromDir, name string, key types.PoolKey) (types.VMRecord, error) { + args := append([]string{"vm", "migrate", "receive", "--from-dir", fromDir}, e.cloneFlags(name, key)...) + return e.runRecord(ctx, dumpTimeout, dumpTermGrace, args...) +} + +// MigrateAbort ends a series and resumes the guest; a repeat or an old series does nothing. +func (e *Engine) MigrateAbort(ctx context.Context, vmName, series string) error { + _, err := e.run(ctx, "vm", "migrate", "abort", argSeries, series, vmName) + return err +} diff --git a/sandboxd/engine/migrate_test.go b/sandboxd/engine/migrate_test.go new file mode 100644 index 00000000..027a49bc --- /dev/null +++ b/sandboxd/engine/migrate_test.go @@ -0,0 +1,86 @@ +package engine + +import ( + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const migrateCocoon = `#!/bin/sh +echo "$*" >> "$COCOON_TEST_TRACE" +case "$3" in +capture) echo '{"series":"s1","dir":"/run/xfer/s1","round":2,"paused":true,"files":[{"name":"state.json","size":3},{"name":"cow.raw","size":4096}]}' ;; +receive) echo '{"state":"created","vsock_socket":"/run/sbx-1/vsock.sock","config":{"name":"sbx-1"}}' ;; +esac +` + +func TestMigrateCallsBuildCocoonsArgsAndParseItsJSON(t *testing.T) { + trace := filepath.Join(t.TempDir(), "trace") + t.Setenv("COCOON_TEST_TRACE", trace) + fakeCocoon(t, migrateCocoon) + e := New("cocoon", []string{"br0"}, nil, false, false, types.RestoreMmap) + none := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} + egress := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall} + + if _, err := e.MigrateCapture(t.Context(), "sbx-1", "", false); err != nil { + t.Fatalf("baseline capture: %v", err) + } + round, err := e.MigrateCapture(t.Context(), "sbx-1", "s1", true) + if err != nil { + t.Fatalf("final capture: %v", err) + } + want := types.MigrateRound{Series: "s1", Dir: "/run/xfer/s1", Round: 2, Paused: true, Files: []types.MigrateFile{{Name: "state.json", Size: 3}, {Name: "cow.raw", Size: 4096}}} + if !reflect.DeepEqual(round, want) { + t.Errorf("round = %+v, want %+v", round, want) + } + for _, key := range []types.PoolKey{none, egress} { + rec, receiveErr := e.MigrateReceive(t.Context(), "/xfer/s1", "sbx-1", key) + if receiveErr != nil || rec.VsockSocket != "/run/sbx-1/vsock.sock" || rec.Config.Name != "sbx-1" { + t.Fatalf("receive on %s = %+v, %v", key.Net, rec, receiveErr) + } + } + if err = e.MigrateAbort(t.Context(), "sbx-1", "s1"); err != nil { + t.Fatalf("abort: %v", err) + } + + data, err := os.ReadFile(trace) + if err != nil { + t.Fatal(err) + } + receive := "vm migrate receive --from-dir /xfer/s1 --name sbx-1 --pull --output json --no-direct-io=false --restore-mode mmap" + wantCalls := []string{ + "vm migrate capture sbx-1 --output json", + "vm migrate capture sbx-1 --series s1 --final --output json", + receive, + receive + " --bridge br0", + "vm migrate abort --series s1 sbx-1", + } + if calls := strings.Split(strings.TrimSpace(string(data)), "\n"); !slices.Equal(calls, wantCalls) { + t.Errorf("cocoon calls\n%q\nwant\n%q", calls, wantCalls) + } +} + +func TestMigrateCallsSurfaceCocoonsStderr(t *testing.T) { + fakeCocoon(t, "#!/bin/sh\necho 'Error: capture refused: run dir has no reflink' >&2\nexit 1\n") + e := newTestEngine() + _, captureErr := e.MigrateCapture(t.Context(), "sbx-1", "", false) + _, receiveErr := e.MigrateReceive(t.Context(), "/xfer/s1", "sbx-1", types.PoolKey{Net: types.NetNone}) + abortErr := e.MigrateAbort(t.Context(), "sbx-1", "s1") + for name, err := range map[string]error{"capture": captureErr, "receive": receiveErr, "abort": abortErr} { + if err == nil || !strings.Contains(err.Error(), "cocoon vm migrate") || !strings.Contains(err.Error(), "run dir has no reflink") { + t.Errorf("%s error = %v, want cocoon's stderr tail", name, err) + } + } +} + +func TestMigrateCaptureRejectsOutputThatIsNotARound(t *testing.T) { + fakeCocoon(t, "#!/bin/sh\necho 'series s1 round 0'\n") + if _, err := newTestEngine().MigrateCapture(t.Context(), "sbx-1", "", false); err == nil || !strings.Contains(err.Error(), "parse migrate capture") { + t.Errorf("capture of prose = %v, want a parse error", err) + } +} diff --git a/sandboxd/engine/version_test.go b/sandboxd/engine/version_test.go index 9e6174dc..f07bb053 100644 --- a/sandboxd/engine/version_test.go +++ b/sandboxd/engine/version_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "fmt" "strings" @@ -35,19 +36,26 @@ func TestBelowFloorComparesReleasesAndLeavesOthersUnknown(t *testing.T) { } } -func TestLiveVolumeSupportFollowsTheCocoonVersion(t *testing.T) { +func TestFeatureSupportFollowsTheCocoonVersion(t *testing.T) { + liveVolume, migrate := (*Engine).CheckLiveVolumeSupport, (*Engine).CheckMigrateSupport for _, tt := range []struct { + check func(*Engine, context.Context) error + floor string version string fail, allowed bool }{ - {"v0.7.1", false, false}, - {"v0.7.2-rc.1", false, false}, - {"v0.7.2", false, true}, - {"v0.8.0", false, true}, - {"master-abcdef", false, true}, - {"v0.7.2", true, false}, + {liveVolume, LiveVolumeCocoon, "v0.7.1", false, false}, + {liveVolume, LiveVolumeCocoon, "v0.7.2-rc.1", false, false}, + {liveVolume, LiveVolumeCocoon, "v0.7.2", false, true}, + {liveVolume, LiveVolumeCocoon, "v0.8.0", false, true}, + {liveVolume, LiveVolumeCocoon, "master-abcdef", false, true}, + {liveVolume, LiveVolumeCocoon, "v0.7.2", true, false}, + {migrate, MigrateCocoon, "v0.7.6", false, false}, + {migrate, MigrateCocoon, "v0.7.7", false, true}, + {migrate, MigrateCocoon, "master-ba2020c", false, true}, + {migrate, MigrateCocoon, "v0.7.7", true, false}, } { - t.Run(fmt.Sprintf("%s/fail=%v", tt.version, tt.fail), func(t *testing.T) { + t.Run(fmt.Sprintf("floor=%s/%s/fail=%v", tt.floor, tt.version, tt.fail), func(t *testing.T) { t.Setenv("COCOON_TEST_VERSION", tt.version) t.Setenv("COCOON_TEST_VERSION_FAIL", "") if tt.fail { @@ -58,14 +66,16 @@ if [ "$#" != 1 ] || [ "$1" != version ]; then exit 2; fi if [ -n "$COCOON_TEST_VERSION_FAIL" ]; then echo /private/cocoon >&2; exit 1; fi printf 'Version: %s\n' "$COCOON_TEST_VERSION" `) - err := newTestEngine().CheckLiveVolumeSupport(t.Context()) + err := tt.check(newTestEngine(), t.Context()) if tt.allowed { if err != nil { t.Fatal(err) } return } - if _, ok := errors.AsType[*LiveVolumeUnsupported](err); !ok || !strings.Contains(err.Error(), LiveVolumeCocoon) || strings.Contains(err.Error(), "/private") { + _, isVolume := errors.AsType[*LiveVolumeUnsupported](err) + _, isMigrate := errors.AsType[*MigrateUnsupported](err) + if !isVolume && !isMigrate || !strings.Contains(err.Error(), tt.floor) || strings.Contains(err.Error(), "/private") { t.Fatalf("error %v", err) } }) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index a82d0d15..5a81c433 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -36,12 +36,7 @@ type volumeMount struct { // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. func (e *Engine) CheckLiveVolumeSupport(ctx context.Context) error { - version, err := e.Version(ctx) - if err != nil { - log.WithFunc("engine.CheckLiveVolumeSupport").Warnf(ctx, "query cocoon version: %v", err) - return &LiveVolumeUnsupported{} - } - if below, known := belowFloor(version, LiveVolumeCocoon); known && below { + if version, misses := e.missesFloor(ctx, LiveVolumeCocoon); misses { return &LiveVolumeUnsupported{Version: version} } return nil diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 12bac378..4c57f5cd 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -454,6 +454,21 @@ func (r VMRecord) TapDevice() string { return r.NetworkConfigs[0].TAP } +// MigrateFile is one file a migrate round wrote, named relative to the round's Dir. +type MigrateFile struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +// MigrateRound is what `cocoon vm migrate capture` prints for one round; round 0 is the baseline. +type MigrateRound struct { + Series string `json:"series"` + Dir string `json:"dir"` + Round int `json:"round"` + Paused bool `json:"paused"` + Files []MigrateFile `json:"files"` +} + // TemplateGossipHash scopes a key hash to its owner; TemplateRootScope covers every tenant. func TemplateGossipHash(keyHash, tenant string) string { sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) From 040951859d02a1f55ba13520a6855df637875b11 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:12:22 +0800 Subject: [PATCH 04/53] peer: stream a migrate round's files to the target over the control port --- sandboxd/store/peer/move.go | 137 +++++++++++++++++ sandboxd/store/peer/move_test.go | 244 +++++++++++++++++++++++++++++++ sandboxd/store/peer/transport.go | 24 +-- 3 files changed, 395 insertions(+), 10 deletions(-) create mode 100644 sandboxd/store/peer/move.go create mode 100644 sandboxd/store/peer/move_test.go diff --git a/sandboxd/store/peer/move.go b/sandboxd/store/peer/move.go new file mode 100644 index 00000000..53c3ea39 --- /dev/null +++ b/sandboxd/store/peer/move.go @@ -0,0 +1,137 @@ +package peer + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + + "github.com/cocoonstack/sandbox/sandboxd/store" +) + +const partialSuffix = ".partial" + +// SendRound PUTs the named files of dir to a peer as one tar stream, in the given order, sparse files kept sparse. +func SendRound(ctx context.Context, addr, token, path, dir string, names []string) error { + pr, pw := io.Pipe() + written := make(chan error, 1) + go func() { + err := writeRound(dir, names, pw) + _ = pw.CloseWithError(err) + written <- err + }() + err := putRound(ctx, peerURL(addr, path), token, pr) + _ = pr.Close() + if writeErr := <-written; writeErr != nil && !errors.Is(writeErr, io.ErrClosedPipe) { + return fmt.Errorf("send round to %s: %w", addr, writeErr) + } + if err != nil { + return fmt.Errorf("send round to %s: %w", addr, err) + } + return nil +} + +// ReceiveRound writes a SendRound stream into dst, replacing same-named files; a file takes its name only once all of it arrived. +func ReceiveRound(r io.Reader, dst string) error { + open, err := receiveFiles(tar.NewReader(r), dst) + if err != nil && open != "" { + _ = os.Remove(open + partialSuffix) + } + return err +} + +func putRound(ctx context.Context, target, token string, body io.Reader) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body) + if err != nil { + return err + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + if resp.StatusCode/100 != 2 { + return fmt.Errorf("unexpected status %d", resp.StatusCode) + } + return nil +} + +func writeRound(dir string, names []string, w io.Writer) error { + tw := tar.NewWriter(w) + scratch := sync.OnceValue(func() []byte { return make([]byte, store.DigestChunkSize) }) + for _, name := range names { + path := filepath.Join(dir, name) + fi, err := os.Stat(path) + if err != nil { + return err + } + if err = tarFile(path, name, fi, tw, scratch); err != nil { + return fmt.Errorf("tar %s: %w", name, err) + } + } + if err := tw.WriteHeader(&tar.Header{Name: recordTrailer, Mode: 0o600, Typeflag: tar.TypeReg}); err != nil { + return err + } + return tw.Close() +} + +// receiveFiles returns the target whose partial file is still on disk. +func receiveFiles(tr *tar.Reader, dst string) (string, error) { + sizes := map[string]int64{} + open := "" + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return open, errors.New("receive round: no completion marker (transfer cut short)") + } + if err != nil { + return open, fmt.Errorf("receive round: %w", err) + } + if hdr.Name == recordTrailer { + return open, commitPartial(open) + } + target, err := safeJoin(dst, hdr.Name) + if err != nil { + return open, err + } + if raw, ok := hdr.PAXRecords[paxChunk]; ok && target == open { + if err = writeChunk(open+partialSuffix, raw, tr, sizes); err != nil { + return open, err + } + continue + } + rawSize, ok := hdr.PAXRecords[paxFileSize] + if !ok { + return open, fmt.Errorf("receive round: unexpected entry %q", hdr.Name) + } + if err = commitPartial(open); err != nil { + return open, err + } + open = target + if err = os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return open, fmt.Errorf("receive round mkdir %s: %w", filepath.Dir(hdr.Name), err) + } + mode := os.FileMode(hdr.Mode).Perm() //nolint:gosec // Perm masks to 0777 + if err = createSparse(target+partialSuffix, rawSize, mode, sizes); err != nil { + return open, err + } + } +} + +func commitPartial(target string) error { + if target == "" { + return nil + } + return os.Rename(target+partialSuffix, target) +} diff --git a/sandboxd/store/peer/move_test.go b/sandboxd/store/peer/move_test.go new file mode 100644 index 00000000..ffd87b01 --- /dev/null +++ b/sandboxd/store/peer/move_test.go @@ -0,0 +1,244 @@ +package peer + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/store/storetest" +) + +func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing.T) { + src := t.TempDir() + size := 6*store.DigestChunkSize + 4096 + storetest.WriteSparse(t, filepath.Join(src, "cow.raw"), size, map[int64][]byte{ + 0: []byte("head"), + 4*store.DigestChunkSize + 9: []byte("mid"), + }) + dense := bytes.Repeat([]byte("guest-memory-"), 300000) + for name, body := range map[string][]byte{"memory-ranges": dense, "state.json": []byte(`{"round":1}`), "unlisted": []byte("x")} { + if err := os.WriteFile(filepath.Join(src, name), body, 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + names := []string{"state.json", "memory-ranges", "cow.raw"} + var method, path, auth string + var body bytes.Buffer + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method, path, auth = r.Method, r.URL.Path, r.Header.Get("Authorization") + _, _ = io.Copy(&body, r.Body) + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + if err := SendRound(t.Context(), srv.URL, "fleet-token", "/v1/moves/m1/rounds/1", src, names); err != nil { + t.Fatalf("SendRound: %v", err) + } + if method != http.MethodPut || path != "/v1/moves/m1/rounds/1" || auth != "Bearer fleet-token" { + t.Errorf("request = %s %s auth %q, want PUT /v1/moves/m1/rounds/1 with the fleet token", method, path, auth) + } + if got, want := entryNames(t, body.Bytes()), append(slices.Clone(names), recordTrailer); !slices.Equal(got, want) { + t.Errorf("entries on the wire = %q, want %q", got, want) + } + sent := int64(body.Len()) + dst := t.TempDir() + if err := ReceiveRound(&body, dst); err != nil { + t.Fatalf("ReceiveRound: %v", err) + } + for _, name := range names { + if sha256OfFile(t, filepath.Join(dst, name)) != sha256OfFile(t, filepath.Join(src, name)) { + t.Errorf("%s differs from the source", name) + } + } + info, err := os.Stat(filepath.Join(dst, "cow.raw")) + if err != nil || info.Size() != size { + t.Fatalf("received cow.raw = %v, %v, want %d bytes", info, err, size) + } + if allocated := info.Sys().(*syscall.Stat_t).Blocks * 512; allocated > sent { + t.Errorf("received cow.raw allocates %d bytes, more than the %d sent", allocated, sent) + } + entries, _ := os.ReadDir(dst) + if len(entries) != len(names) { + t.Errorf("dst holds %d entries, want exactly the %d listed files", len(entries), len(names)) + } +} + +func TestReceiveRoundReplacesFilesOfTheSameName(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + size := 3 * store.DigestChunkSize + storetest.WriteSparse(t, filepath.Join(src, "cow.raw"), size, map[int64][]byte{store.DigestChunkSize: []byte("new")}) + if err := os.WriteFile(filepath.Join(src, "state.json"), []byte("new"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dst, "state.json"), []byte("an older and longer state"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dst, "cow.raw"), bytes.Repeat([]byte{0xee}, int(size)), 0o600); err != nil { + t.Fatal(err) + } + + if err := ReceiveRound(bytes.NewReader(roundStream(t, src, "state.json", "cow.raw")), dst); err != nil { + t.Fatalf("ReceiveRound: %v", err) + } + if body, err := os.ReadFile(filepath.Join(dst, "state.json")); err != nil || string(body) != "new" { + t.Errorf("state.json = %q, %v, want the new round's bytes", body, err) + } + if sha256OfFile(t, filepath.Join(dst, "cow.raw")) != sha256OfFile(t, filepath.Join(src, "cow.raw")) { + t.Error("cow.raw keeps bytes of the replaced file in its holes") + } +} + +func TestReceiveRoundOfACutStreamLeavesNoCompleteLookingFile(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "state.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + storetest.WriteSparse(t, filepath.Join(src, "memory-ranges"), 4*store.DigestChunkSize, map[int64][]byte{ + 0: bytes.Repeat([]byte("a"), 4096), + store.DigestChunkSize: bytes.Repeat([]byte("b"), 4096), + 3 * store.DigestChunkSize: bytes.Repeat([]byte("c"), 4096), + }) + full := roundStream(t, src, "state.json", "memory-ranges") + var noTrailer bytes.Buffer + tw := tar.NewWriter(&noTrailer) + tr := tar.NewReader(bytes.NewReader(full)) + for { + hdr, err := tr.Next() + if err != nil || hdr.Name == recordTrailer { + break + } + if err = tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err = io.Copy(tw, tr); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + for name, stream := range map[string][]byte{ + "no completion marker": noTrailer.Bytes(), + "cut inside a chunk": full[:len(full)-int(store.DigestChunkSize)/2], + } { + t.Run(name, func(t *testing.T) { + dst := t.TempDir() + if err := ReceiveRound(bytes.NewReader(stream), dst); err == nil { + t.Fatal("ReceiveRound accepted a cut stream") + } + if _, err := os.Stat(filepath.Join(dst, "memory-ranges")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("memory-ranges exists after a cut stream (%v); a partial file must not take its name", err) + } + if _, err := os.Stat(filepath.Join(dst, "memory-ranges"+partialSuffix)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("partial file left behind (%v)", err) + } + if body, err := os.ReadFile(filepath.Join(dst, "state.json")); err != nil || string(body) != "{}" { + t.Errorf("state.json = %q, %v, want the file that arrived whole", body, err) + } + }) + } +} + +func TestReceiveRoundRefusesNonLocalAndNonRegularEntries(t *testing.T) { + sized := map[string]string{paxFileSize: "4"} + for _, hdr := range []*tar.Header{ + {Name: "../escape", Mode: 0o600, Typeflag: tar.TypeReg, PAXRecords: sized}, + {Name: "/abs/escape", Mode: 0o600, Typeflag: tar.TypeReg, PAXRecords: sized}, + {Name: `..\escape`, Mode: 0o600, Typeflag: tar.TypeReg, PAXRecords: sized}, + {Name: "link", Linkname: "/etc/passwd", Mode: 0o777, Typeflag: tar.TypeSymlink}, + {Name: "sub/", Mode: 0o700, Typeflag: tar.TypeDir}, + {Name: "plain", Mode: 0o600, Typeflag: tar.TypeReg}, + } { + t.Run(hdr.Name, func(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if err := tw.WriteHeader(&tar.Header{Name: recordTrailer, Mode: 0o600, Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + root := t.TempDir() + dst := filepath.Join(root, "dst") + if err := os.Mkdir(dst, 0o700); err != nil { + t.Fatal(err) + } + if err := ReceiveRound(&buf, dst); err == nil { + t.Fatalf("ReceiveRound accepted %q", hdr.Name) + } + if entries, _ := os.ReadDir(dst); len(entries) != 0 { + t.Errorf("dst holds %d entries after a refused stream", len(entries)) + } + if entries, _ := os.ReadDir(root); len(entries) != 1 { + t.Errorf("dst's parent holds %d entries after a refused stream, want only dst", len(entries)) + } + }) + } +} + +func TestSendRoundFailsOnARefusedPutOrAMissingFile(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "state.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer fleet-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + if err := ReceiveRound(r.Body, t.TempDir()); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + if err := SendRound(t.Context(), srv.URL, "wrong", "/r", src, []string{"state.json"}); err == nil || !strings.Contains(err.Error(), "401") { + t.Errorf("refused PUT = %v, want the 401", err) + } + if err := SendRound(t.Context(), srv.URL, "fleet-token", "/r", src, []string{"state.json", "missing"}); err == nil || !errors.Is(err, os.ErrNotExist) { + t.Errorf("round with a missing file = %v, want the stat error", err) + } +} + +func roundStream(t *testing.T, dir string, names ...string) []byte { + t.Helper() + var buf bytes.Buffer + if err := writeRound(dir, names, &buf); err != nil { + t.Fatalf("writeRound: %v", err) + } + return buf.Bytes() +} + +func entryNames(t *testing.T, raw []byte) []string { + t.Helper() + var names []string + tr := tar.NewReader(bytes.NewReader(raw)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return names + } + if err != nil { + t.Fatalf("read entries: %v", err) + } + if len(names) == 0 || names[len(names)-1] != hdr.Name { + names = append(names, hdr.Name) + } + } +} diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index c1e30bc9..b37a537b 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -246,15 +246,7 @@ func tarInto(src, prefix string, tw *tar.Writer) error { Typeflag: tar.TypeDir, }) } - if err = tw.WriteHeader(&tar.Header{ - Name: name, - Mode: int64(fi.Mode().Perm()), - Typeflag: tar.TypeReg, - PAXRecords: map[string]string{paxFileSize: strconv.FormatInt(fi.Size(), 10)}, - }); err != nil { - return err - } - return tarChunks(path, name, fi.Size(), tw, scratch) + return tarFile(path, name, fi, tw, scratch) }) if err != nil { return fmt.Errorf("tar %s: %w", src, err) @@ -262,8 +254,20 @@ func tarInto(src, prefix string, tw *tar.Writer) error { return nil } +func tarFile(path, name string, fi fs.FileInfo, tw *tar.Writer, scratch func() []byte) error { + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Mode: int64(fi.Mode().Perm()), + Typeflag: tar.TypeReg, + PAXRecords: map[string]string{paxFileSize: strconv.FormatInt(fi.Size(), 10)}, + }); err != nil { + return err + } + return tarChunks(path, name, fi.Size(), tw, scratch) +} + func tarChunks(path, name string, size int64, tw *tar.Writer, scratch func() []byte) error { - f, openErr := os.Open(path) //nolint:gosec // path comes from WalkDir over src + f, openErr := os.Open(path) //nolint:gosec // path is a file of a directory this node sends if openErr != nil { return openErr } From ae5556ada1a8f7fefdc0f1415f7b78449b742ab6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:21:06 +0800 Subject: [PATCH 05/53] engine: leave the restore mode to cocoon on a receive under ondemand cocoon's hypervisor rejects ondemand for a series with deltas. --- sandboxd/engine/engine.go | 6 +++--- sandboxd/engine/migrate.go | 7 ++++++- sandboxd/engine/migrate_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 1373f505..53f7c422 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -387,13 +387,13 @@ func (e *Engine) probe(ctx context.Context, vsockSocket string, port int, timeou } func (e *Engine) cloneArgs(name string, key types.PoolKey, source ...string) []string { - return append(append([]string{"vm", "clone"}, source...), e.cloneFlags(name, key)...) + return append(append([]string{"vm", "clone"}, source...), e.cloneFlags(name, key, e.restoreArgs())...) } -func (e *Engine) cloneFlags(name string, key types.PoolKey) []string { +func (e *Engine) cloneFlags(name string, key types.PoolKey, restore []string) []string { // --pull: a checkpoint/template export carries only COW + memory, not the base image. args := []string{argName, name, "--pull", argOutput, formatJSON, e.directIOArg()} - args = append(args, e.restoreArgs()...) + args = append(args, restore...) return append(args, e.netArgs(name, key, false)...) } diff --git a/sandboxd/engine/migrate.go b/sandboxd/engine/migrate.go index 60628da9..d169c189 100644 --- a/sandboxd/engine/migrate.go +++ b/sandboxd/engine/migrate.go @@ -50,7 +50,12 @@ func (e *Engine) MigrateCapture(ctx context.Context, vmName, series string, fina // MigrateReceive restores a transferred series into a VM whose guest stays paused until Start. func (e *Engine) MigrateReceive(ctx context.Context, fromDir, name string, key types.PoolKey) (types.VMRecord, error) { - args := append([]string{"vm", "migrate", "receive", "--from-dir", fromDir}, e.cloneFlags(name, key)...) + restore := e.restoreArgs() + if e.restoreMode == types.RestoreOnDemand { + // cocoon's hypervisor rejects ondemand for a series with deltas, so its default applies + restore = nil + } + args := append([]string{"vm", "migrate", "receive", "--from-dir", fromDir}, e.cloneFlags(name, key, restore)...) return e.runRecord(ctx, dumpTimeout, dumpTermGrace, args...) } diff --git a/sandboxd/engine/migrate_test.go b/sandboxd/engine/migrate_test.go index 027a49bc..a7740a65 100644 --- a/sandboxd/engine/migrate_test.go +++ b/sandboxd/engine/migrate_test.go @@ -84,3 +84,28 @@ func TestMigrateCaptureRejectsOutputThatIsNotARound(t *testing.T) { t.Errorf("capture of prose = %v, want a parse error", err) } } + +func TestMigrateReceiveLeavesTheOndemandRestoreModeToCocoon(t *testing.T) { + trace := filepath.Join(t.TempDir(), "trace") + t.Setenv("COCOON_TEST_TRACE", trace) + fakeCocoon(t, migrateCocoon) + key := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} + for _, mode := range []types.RestoreMode{types.RestoreOnDemand, types.RestoreCopy} { + e := New("cocoon", nil, nil, false, false, mode) + if _, err := e.MigrateReceive(t.Context(), "/xfer/s1", "sbx-1", key); err != nil { + t.Fatalf("receive under %s: %v", mode, err) + } + _, _ = e.Clone(t.Context(), "/golden", "sbx-2", key) + } + data, err := os.ReadFile(trace) + if err != nil { + t.Fatal(err) + } + calls := strings.Split(strings.TrimSpace(string(data)), "\n") + receive := "vm migrate receive --from-dir /xfer/s1 --name sbx-1 --pull --output json --no-direct-io=false" + clone := "vm clone --from-dir /golden --name sbx-2 --pull --output json --no-direct-io=false --restore-mode " + want := []string{receive, clone + "ondemand", receive + " --restore-mode copy", clone + "copy"} + if !slices.Equal(calls, want) { + t.Errorf("cocoon calls\n%q\nwant\n%q", calls, want) + } +} From 1b75753e49932c16e046f370e9e1db49b37b307f Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:21:55 +0800 Subject: [PATCH 06/53] config: move_concurrency, reloadable, and the move registry beside the archive registry Moves need archive_failover and an encrypted mesh; a node that sets move_concurrency without them refuses to load, one that leaves it unset just cannot move. --- sandboxd/backends.go | 6 ++++++ sandboxd/config/config.go | 15 +++++++++++++-- sandboxd/config/config_test.go | 24 ++++++++++++++++++++++++ sandboxd/config/reload.go | 1 + sandboxd/config/reload_test.go | 3 ++- sandboxd/pool/pool.go | 8 +++++++- sandboxd/pool/view.go | 2 ++ 7 files changed, 55 insertions(+), 4 deletions(-) diff --git a/sandboxd/backends.go b/sandboxd/backends.go index a2c1cbc7..e1adbb54 100644 --- a/sandboxd/backends.go +++ b/sandboxd/backends.go @@ -9,6 +9,7 @@ import ( archivespg "github.com/cocoonstack/sandbox/sandboxd/archives/pg" "github.com/cocoonstack/sandbox/sandboxd/config" + movespg "github.com/cocoonstack/sandbox/sandboxd/moves/pg" "github.com/cocoonstack/sandbox/sandboxd/pool" poolpg "github.com/cocoonstack/sandbox/sandboxd/poolset/pg" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -44,6 +45,11 @@ func openBackends(ctx context.Context, cfg *config.Config) (pool.Backends, error return pool.Backends{}, err } } + if cfg.Movable() { + if b.Moves, err = movespg.Open(ctx, os.Getenv(ms.DSNEnv), ms.Cell); err != nil { + return pool.Backends{}, err + } + } return b, nil } diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index e06e784c..d9ab5103 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -275,8 +275,10 @@ type Config struct { EgressUsageBytes bool `json:"egress_usage_bytes,omitzero"` EgressConnRate *EgressConnRate `json:"egress_conn_rate,omitempty"` - MaxClaims int `json:"max_claims,omitzero"` - MaxRunning int `json:"max_running,omitzero"` + MaxClaims int `json:"max_claims,omitzero"` + MaxRunning int `json:"max_running,omitzero"` + // MoveConcurrency caps the live moves this node runs as source or target at once; zero takes the default. + MoveConcurrency int `json:"move_concurrency,omitzero"` MaxForkCount int `json:"max_fork_count,omitzero"` RefillConcurrency int `json:"refill_concurrency,omitzero"` ReleaseDelaySeconds int `json:"release_delay_seconds,omitzero"` @@ -297,6 +299,11 @@ func (c *Config) HasEgress() bool { return len(c.Bridges) > 0 || len(c.Networks) > 0 } +// Movable reports whether the node can run live moves: they need the move registry and an encrypted mesh. +func (c *Config) Movable() bool { + return c.ArchiveFailover && c.Mesh != nil && c.Mesh.ClusterKey != "" +} + func (c *Config) S3Checkpoints() bool { return c.CheckpointStore != nil && c.CheckpointStore.Kind == "s3" } @@ -338,6 +345,7 @@ func (c *Config) validate() error { {"checkpoint_ttl_hours", c.CheckpointTTLHours}, {"max_claims", c.MaxClaims}, {"max_running", c.MaxRunning}, + {"move_concurrency", c.MoveConcurrency}, {"refill_concurrency", c.RefillConcurrency}, {"release_delay_seconds", c.ReleaseDelaySeconds}, } { @@ -389,6 +397,9 @@ func (c *Config) validate() error { if c.CheckpointPeerHeal && c.APIToken == "" { return fmt.Errorf("checkpoint_peer_heal requires api_token: without it resolveScope leaves the raw checkpoint blob GET reachable with no credential") } + if c.MoveConcurrency > 0 && !c.Movable() { + return fmt.Errorf("move_concurrency requires archive_failover and an encrypted mesh (set mesh.cluster_key)") + } for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMemoryLimit, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { if err := check(); err != nil { return err diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 9670d1dc..75c259d5 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -261,6 +261,30 @@ func TestLoadReplicateSeconds(t *testing.T) { } } +func TestLoadMoveConcurrency(t *testing.T) { + t.Setenv("SANDBOX_TEST_DSN", "postgres://sandboxd@db/sandboxd") + failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},` + key := `"cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="` + body := func(on, mesh, moves string) string { + return `{"api_token":"r","advertise_addr":"node1:7777",` + on + `"mesh":{"bind":"node1:7946"` + mesh + `},` + moves + `"pools":[]}` + } + cfg := mustLoad(t, writeConfig(t, body(failover, ","+key, `"move_concurrency":3,`))) + if cfg.MoveConcurrency != 3 || !cfg.Movable() { + t.Fatalf("Load: move_concurrency %d movable %v, want 3 and movable", cfg.MoveConcurrency, cfg.Movable()) + } + for name, tt := range map[string]struct{ on, mesh, moves, want string }{ + "no failover": {"", "," + key, `"move_concurrency":2,`, "move_concurrency requires"}, + "plain mesh": {failover, "", `"move_concurrency":2,`, "move_concurrency requires"}, + "negative": {failover, "," + key, `"move_concurrency":-1,`, "must not be negative"}, + "unset on a plain mesh": {failover, "", "", ""}, + } { + cfg, err := Load(writeConfig(t, body(tt.on, tt.mesh, tt.moves))) + if tt.want == "" && (err != nil || cfg.Movable()) || tt.want != "" && (err == nil || !strings.Contains(err.Error(), tt.want)) { + t.Errorf("%s: %v, want %q", name, err, tt.want) + } + } +} + func TestLoadEgressUpstream(t *testing.T) { t.Setenv("SANDBOX_TEST_RES", "http://user:pw@res.example:3128") t.Setenv("SANDBOX_TEST_DC", "socks5://10.1.0.5:1080") diff --git a/sandboxd/config/reload.go b/sandboxd/config/reload.go index 6919c8bd..6d642113 100644 --- a/sandboxd/config/reload.go +++ b/sandboxd/config/reload.go @@ -22,6 +22,7 @@ var reloadableFields = map[string]bool{ "max_claims": true, "max_running": true, + "move_concurrency": true, "max_fork_count": true, "release_delay_seconds": true, "vmm_restart": true, diff --git a/sandboxd/config/reload_test.go b/sandboxd/config/reload_test.go index 0041dc10..08ad22f8 100644 --- a/sandboxd/config/reload_test.go +++ b/sandboxd/config/reload_test.go @@ -25,7 +25,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { next.EgressConnRate = &EgressConnRate{PerSecond: 30, Burst: 600} next.DiskReserve = &DiskReserve{Free: "100G"} next.MemoryLimit = &MemoryLimit{Anon: "300G"} - next.MaxClaims, next.MaxRunning, next.MaxForkCount = 200, 12, 4 + next.MaxClaims, next.MaxRunning, next.MaxForkCount, next.MoveConcurrency = 200, 12, 4, 3 changed, ignored, err := cur.ReloadDiff(&next) if err != nil { t.Fatalf("ReloadDiff: %v", err) @@ -36,6 +36,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { "egress_conn_rate", "max_claims", "max_running", + "move_concurrency", "max_fork_count", "disk_reserve", "memory_limit", diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index c851f119..d42781b9 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -25,6 +25,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/moves" "github.com/cocoonstack/sandbox/sandboxd/outbound" "github.com/cocoonstack/sandbox/sandboxd/poolset" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -63,7 +64,8 @@ const ( refillBackoffBase = 250 * time.Millisecond refillBackoffMax = buildRetryDelay // Fallbacks when a Manager is built from a Config that skipped config.Load's defaulting - defaultRefill = 4 + defaultRefill = 4 + defaultMoveConcurrency = 2 // A heal pulls a whole guest memory image, so a few saturate disk and NIC. maxConcurrentHeals = 4 @@ -232,6 +234,7 @@ type Backends struct { Tenants tenants.Source CellPools poolset.Shared Archives archives.Registry + Moves moves.Registry } // PeerDeleteFunc broadcasts a checkpoint delete to every peer. @@ -283,6 +286,8 @@ type Manager struct { unfencing atomic.Bool replicating atomic.Bool replicas map[string]time.Time + // moves is the cluster move registry, nil unless the node can run live moves. + moves moves.Registry view atomic.Pointer[configView] // cfg is the config the view was built from; reloadMu orders reloads, the only writers of cfg and view. @@ -365,6 +370,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg poolStore: newPoolStore(cfg.DataDir), poolShared: b.CellPools, archives: b.Archives, + moves: b.Moves, tenants: b.Tenants, ckpts: b.Checkpoints, ckptsShared: b.SharedCheckpoints, diff --git a/sandboxd/pool/view.go b/sandboxd/pool/view.go index 4ee18773..0d37e1a1 100644 --- a/sandboxd/pool/view.go +++ b/sandboxd/pool/view.go @@ -26,6 +26,7 @@ type configView struct { maxClaims int maxRunning int + moveConcurrency int maxFork int releaseDelay time.Duration vmmRestart types.VMMRestart @@ -47,6 +48,7 @@ func newConfigView(cfg *config.Config, secrets *egress.SecretStore) *configView maxClaims: cfg.MaxClaims, maxRunning: cfg.MaxRunning, + moveConcurrency: cmp.Or(cfg.MoveConcurrency, defaultMoveConcurrency), maxFork: cmp.Or(cfg.MaxForkCount, config.DefaultMaxForkCount), releaseDelay: time.Duration(cfg.ReleaseDelaySeconds) * time.Second, vmmRestart: cfg.VMMRestart, From 15249a59288ab74363890182b408aa9a9589bf84 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:35:59 +0800 Subject: [PATCH 07/53] pool: move a running claim to a named peer with cocoon's live migration The target admits the move like a new claim and holds a reservation that quotaErr and runningErr count until its commit swaps it for the claim under one m.mu. The source journals the move, sends the rounds, drops a replicated claim's row before the final round, and resolves any failure through the move record before it resumes the guest. --- sandboxd/pool/capture.go | 3 + sandboxd/pool/claim.go | 15 +- sandboxd/pool/claims.go | 4 +- sandboxd/pool/drain.go | 4 +- sandboxd/pool/failover.go | 3 + sandboxd/pool/failover_test.go | 4 +- sandboxd/pool/hibernate.go | 6 + sandboxd/pool/liveness.go | 2 +- sandboxd/pool/move.go | 113 +++++++ sandboxd/pool/move_in.go | 266 ++++++++++++++++ sandboxd/pool/move_out.go | 376 +++++++++++++++++++++++ sandboxd/pool/move_test.go | 542 +++++++++++++++++++++++++++++++++ sandboxd/pool/pool.go | 23 +- sandboxd/pool/pool_test.go | 80 +++++ sandboxd/pool/reap.go | 2 +- sandboxd/pool/replica.go | 5 +- sandboxd/types/types.go | 17 ++ 17 files changed, 1452 insertions(+), 13 deletions(-) create mode 100644 sandboxd/pool/move.go create mode 100644 sandboxd/pool/move_in.go create mode 100644 sandboxd/pool/move_out.go create mode 100644 sandboxd/pool/move_test.go diff --git a/sandboxd/pool/capture.go b/sandboxd/pool/capture.go index 66681887..56ba133b 100644 --- a/sandboxd/pool/capture.go +++ b/sandboxd/pool/capture.go @@ -34,6 +34,9 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { if m.claimed[sb.ID] != sb { return ErrUnknownSandbox } + if _, moving := m.moving[sb.ID]; moving { + return ErrEvacuating + } if _, err := m.fenceWatch(sb); err != nil { return err } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 0739ef9e..37cc9970 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -118,6 +118,9 @@ func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { if !ok { return ErrUnknownSandbox } + if m.movingNow(id) { + return ErrEvacuating + } if m.archives != nil { return m.releaseRowed(ctx, id, sb) } @@ -391,8 +394,8 @@ func (m *Manager) quotaErr(extra int, tenant string) error { if m.memoryLow.Load() { return fmt.Errorf("%w: %s", ErrQuota, memoryLowReason) } - if limit := m.view.Load().maxClaims; limit > 0 && len(m.claimed)+extra > limit { - return fmt.Errorf("%w: %d live claims, cap %d", ErrQuota, len(m.claimed), limit) + if limit, live := m.view.Load().maxClaims, len(m.claimed)+len(m.incoming); limit > 0 && live+extra > limit { + return fmt.Errorf("%w: %d live claims, cap %d", ErrQuota, live, limit) } if err := m.runningErr(extra); err != nil { return err @@ -405,7 +408,11 @@ func (m *Manager) quotaErr(extra int, tenant string) error { if limit <= 0 { return nil } - if live := m.tenantLive[tenant]; live+extra > limit { + live := m.tenantLive[tenant] + if len(m.incoming) > 0 { + live += m.incomingOf(tenant) + } + if live+extra > limit { return fmt.Errorf("%w: tenant %s at %d live claims, cap %d", ErrQuota, tenant, live, limit) } return nil @@ -417,7 +424,7 @@ func (m *Manager) runningErr(extra int) error { if limit == 0 { return nil } - if running := m.store.runningCount(); running+extra > limit { + if running := m.store.runningCount() + len(m.incoming); running+extra > limit { return fmt.Errorf("%w: %d running claims, cap %d", ErrQuota, running, limit) } return nil diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index e998d7c0..e01f4cbd 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -64,6 +64,8 @@ type claimDTO struct { Restarts int `json:"restarts,omitzero"` RestartedAt time.Time `json:"restarted_at,omitzero"` Failed string `json:"failed,omitempty"` + MoveOut *types.MoveOut `json:"move_out,omitempty"` + MoveIn *types.MoveIn `json:"move_in,omitempty"` } // claimRow keeps its encoded member until set replaces the row; set never edits one. @@ -394,7 +396,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ReaderSafe: sb.ReaderSafe, ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, ReplicaPending: sb.ReplicaPending, FromCheckpoint: sb.FromCheckpoint, - Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, + Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, MoveOut: sb.MoveOut, MoveIn: sb.MoveIn, } } diff --git a/sandboxd/pool/drain.go b/sandboxd/pool/drain.go index d4e5e9db..e3e35c7b 100644 --- a/sandboxd/pool/drain.go +++ b/sandboxd/pool/drain.go @@ -102,7 +102,9 @@ func (m *Manager) evacuateClaim(ctx context.Context, sb *types.Sandbox) error { return errNoFailover } m.mu.Lock() - _, busy := m.archiving[sb.ID] + _, archiving := m.archiving[sb.ID] + _, moving := m.moving[sb.ID] + busy := archiving || moving gone := m.claimed[sb.ID] != sb || !m.evacuatingLocked(sb.ID) archived, tenantGone := sb.ArchiveCk != "", m.tenantGone(sb.Tenant) pause := !gone && !tenantGone && !busy && !archived diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index d1b17436..2591c292 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -325,6 +325,9 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox) { func (m *Manager) releaseRowed(ctx context.Context, id string, sb *types.Sandbox) error { sb.Transition.Lock() defer sb.Transition.Unlock() + if m.movingNow(id) { + return ErrEvacuating + } if err := m.resolveIntent(context.WithoutCancel(ctx), sb); err != nil { return err } diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 98e66b43..62af2f95 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -17,6 +17,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/archives/archivestest" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/moves" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -1869,6 +1870,7 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { type failoverCluster struct { reg *archivestest.Mem + moves moves.Registry ckDir string replicate int volumes []config.VolumeSpec @@ -1940,7 +1942,7 @@ func (c *failoverCluster) nodeOn(t *testing.T, eng *fakeEngine, name, dataDir st cfg.EgressCA = writeTestEgressCA(t) } b := testBackends(t, cfg) - b.Archives = c.reg + b.Archives, b.Moves = c.reg, c.moves m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), b) if err != nil { t.Fatalf("manager %s: %v", name, err) diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 3e8092c8..06588d71 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -62,6 +62,9 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error if sb.ArchiveCk != "" { return nil // already off the node } + if m.movingNow(sb.ID) { + return ErrEvacuating + } if _, err := m.fenceWatch(sb); err != nil { return err } @@ -151,6 +154,9 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, } func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bool) (string, error) { + if m.movingFinal(sb.ID) { + return "", ErrEvacuating + } sb.Transition.Lock() defer sb.Transition.Unlock() if sb.ArchiveCk != "" { diff --git a/sandboxd/pool/liveness.go b/sandboxd/pool/liveness.go index ffd34727..cd9bb224 100644 --- a/sandboxd/pool/liveness.go +++ b/sandboxd/pool/liveness.go @@ -310,7 +310,7 @@ func (m *Manager) leaseLapsed(sb *types.Sandbox, now time.Time) bool { // runningShaped reports whether sb's VM should be up; callers hold m.mu. func runningShaped(sb *types.Sandbox) bool { - return sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.Failed == "" + return sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.Failed == "" && sb.MoveIn == nil } // coldBootBlocker names why sb's VM cannot be cold-booted, "" when it can. diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go new file mode 100644 index 00000000..07217bdc --- /dev/null +++ b/sandboxd/pool/move.go @@ -0,0 +1,113 @@ +package pool + +import ( + "context" + "fmt" + "path/filepath" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/moves" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const ( + // DefaultMoveDowntime is the final round's pause a move allows when the caller names none. + DefaultMoveDowntime = 2 * time.Second + + movesDir = "moves" + // A reservation with no round or commit for this long belongs to a source that gave up. + moveReservationIdle = 2 * time.Minute + moveRetryInterval = 2 * time.Second + moveDeltaLimit = 4 + // Receive and vm start on the target add this to the final round's transfer. + moveStartAllowance = 400 * time.Millisecond + moveStaleAge = time.Hour +) + +// outMove is a source's move in flight; final is guarded by m.mu, the rest belongs to the goroutine driving the move. +type outMove struct { + addr string + to string + epoch int64 + series string + rounds int + final bool + locked bool +} + +// MoveAdmission is what a source asks a target to reserve for a live move. +type MoveAdmission struct { + Tenant string + Key types.PoolKey + From string +} + +// MoveResult reports a finished live move: the rounds sent and the guest's pause from the final capture to the target's commit. +type MoveResult struct { + Rounds int + Downtime time.Duration +} + +// MovePeer reaches the target node of a live move. +type MovePeer interface { + // Admit reserves a claim slot on the target and returns its node id. + Admit(ctx context.Context, addr, id string, a MoveAdmission) (string, error) + SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error + // Commit hands the target the claim record and the final epoch and returns the committed epoch. + Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) (int64, error) + Drop(ctx context.Context, addr, id string) error +} + +// WithMovePeer wires the transport a move uses to reach its target; set it before serving. +func (m *Manager) WithMovePeer(p MovePeer) { + m.movePeer = p +} + +// moveBudgetErr refuses a move past move_concurrency, counting both directions; callers hold m.mu. +func (m *Manager) moveBudgetErr() error { + limit := m.view.Load().moveConcurrency + if n := len(m.moving) + len(m.incoming); n >= limit { + return fmt.Errorf("%w: %d moves in flight, move_concurrency %d", ErrQuota, n, limit) + } + return nil +} + +func (m *Manager) moveDir(id string) string { + return filepath.Join(m.dataDir, movesDir, id) +} + +// sweepStaleMoves ends this node's move records no move in flight holds. +func (m *Manager) sweepStaleMoves(ctx context.Context) { + logger := log.WithFunc("pool.sweepStaleMoves") + recs, err := m.moves.Stale(ctx, time.Now().Add(-moveStaleAge)) + if err != nil { + logger.Warnf(ctx, "list stale moves: %v", err) + return + } + for _, rec := range recs { + if rec.From != m.nodeID && rec.To != m.nodeID || m.moveHeld(rec.ID) { + continue + } + epoch := rec.Epoch + if rec.State == moves.Pending || rec.State == moves.Final { + if epoch, err = m.moves.Advance(ctx, rec.ID, rec.Epoch, rec.State, moves.Aborted); err != nil { + logger.Warnf(ctx, "abort stale move %s: %v", rec.ID, err) + continue + } + } + if err = m.moves.Delete(ctx, rec.ID, epoch); err != nil { + logger.Warnf(ctx, "delete stale move %s: %v", rec.ID, err) + } + } +} + +func (m *Manager) moveHeld(id string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, out := m.moving[id] + _, in := m.incoming[id] + sb := m.claimed[id] + return out || in || sb != nil && sb.MoveIn != nil +} diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go new file mode 100644 index 00000000..e1abdfdc --- /dev/null +++ b/sandboxd/pool/move_in.go @@ -0,0 +1,266 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/moves" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/usage" +) + +const reasonMoveStartFailed = "vmm did not start after a live move" + +// inMove is a target's reservation for a move in flight; guarded by m.mu. +type inMove struct { + tenant string + from string + touched time.Time +} + +// AdmitMove reserves a claim slot for a live move of id to this node and returns this node's id. +func (m *Manager) AdmitMove(id string, a MoveAdmission) (string, error) { + if m.moves == nil { + return "", errMovesOff + } + if err := m.validate(a.Key); err != nil { + return "", fmt.Errorf("%w: %w", ErrNotMovable, err) + } + m.mu.Lock() + err := m.admitMoveLocked(id, a) + m.mu.Unlock() + if err != nil { + return "", err + } + dir := m.moveDir(id) + if err = os.RemoveAll(dir); err == nil { + err = os.MkdirAll(dir, 0o700) + } + if err != nil { + m.DropMove(id) + return "", fmt.Errorf("move %s: staging dir: %w", id, err) + } + return m.nodeID, nil +} + +// ReceiveMoveRound writes one round of a reserved move into its staging dir. +func (m *Manager) ReceiveMoveRound(id string, body io.Reader) error { + if !m.touchReservation(id) { + return ErrUnknownSandbox + } + if err := peer.ReceiveRound(body, m.moveDir(id)); err != nil { + return fmt.Errorf("move %s: %w", id, err) + } + m.touchReservation(id) + return nil +} + +// CommitMove receives the staged series as the claim, commits the move record and starts the guest; it returns the committed epoch. +func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch int64) (int64, error) { + ctx = context.WithoutCancel(ctx) + if !m.touchReservation(id) { + return 0, fmt.Errorf("%w: no reservation for the move of %s", ErrQuota, id) + } + if m.memoryLow.Load() { + return 0, fmt.Errorf("%w: %s", ErrQuota, memoryLowReason) + } + sb := &types.Sandbox{} + if err := json.Unmarshal(claim, sb); err != nil || sb.ID != id { + return 0, fmt.Errorf("%w: the claim record does not decode as %s", ErrNotMovable, id) + } + name := vmName(sb.Key) + rec, err := m.eng.MigrateReceive(ctx, m.moveDir(id), name, sb.Key) + if err != nil { + m.removeStaleVM(ctx, name, types.VMRecord{State: vmStateCreating}) + return 0, fmt.Errorf("move %s: receive: %w", id, err) + } + sb.VMName, sb.VsockSocket, sb.TAP, sb.HibernateSnap, sb.PendingSnap = name, rec.VsockSocket, "", "", "" + sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk, sb.ReplicaPending = "", 0, "", "", "" + sb.MoveOut = nil + sb.SetEnv(sb.Env) + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + in := m.incoming[id] + if in == nil || m.claimed[id] != nil { + m.mu.Unlock() + m.destroy(ctx, name) + return 0, fmt.Errorf("%w: the reservation for %s ended", ErrNotMovable, id) + } + sb.MoveIn = &types.MoveIn{From: in.from, Epoch: epoch} + delete(m.incoming, id) + m.addClaimLocked(sb) + js := m.store.set(sb) + m.mu.Unlock() + if err = m.store.commit(js); err != nil { + m.dropMoveIn(ctx, sb) + return 0, fmt.Errorf("move %s: persist claims: %w", id, err) + } + committed, err := m.moves.Advance(ctx, id, epoch, moves.Final, moves.Committed) + switch { + case errors.Is(err, moves.ErrLost): + m.dropMoveIn(ctx, sb) + return 0, fmt.Errorf("%w: the source ended the move of %s", ErrNotMovable, id) + case err != nil: + go m.settleMoveIn(ctx, sb) + return 0, fmt.Errorf("move %s: %w", id, err) + } + m.startMoveIn(ctx, sb) + return committed, nil +} + +// DropMove ends a reservation and its staging dir; a claim a commit already journaled is left to its record. +func (m *Manager) DropMove(id string) { + m.mu.Lock() + _, held := m.incoming[id] + delete(m.incoming, id) + m.mu.Unlock() + if held { + _ = os.RemoveAll(m.moveDir(id)) + } +} + +// startMoveIn starts a committed claim's guest and serves it; a guest that cannot start fails the claim here, never back to the source. The caller holds sb.Transition. +func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox) { + logger := log.WithFunc("pool.startMoveIn") + from := sb.MoveIn.From + var err error + for range 2 { + if err = m.eng.Start(ctx, sb.VMName); err == nil { + break + } + logger.Warnf(ctx, "start moved claim %s: %v", sb.ID, err) + } + if err != nil { + _, _ = m.commitIfLive(ctx, sb, func() { sb.MoveIn = nil }) + sb.ExitRecorded = true + m.recordUsage(ctx, usage.Event{Event: "vmm_exit", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) + m.failLocked(ctx, sb, reasonMoveStartFailed) + _ = os.RemoveAll(m.moveDir(sb.ID)) + return + } + if sock, probeErr := m.probeReady(ctx, sb.VMName, sb.VsockSocket, claimProbeTimeout); probeErr != nil { + logger.Warnf(ctx, "probe moved claim %s: %v", sb.ID, probeErr) + } else { + m.mu.Lock() + sb.VsockSocket = sock + m.mu.Unlock() + } + if proxyErr := m.out.ArmProxy(ctx, sb); proxyErr != nil { + logger.Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) + } + live, err := m.commitIfLive(ctx, sb, func() { sb.MoveIn = nil }) + if !live { + m.out.Disarm(sb.ID, true) + return + } + if err != nil { + logger.Warnf(ctx, "persist the start of moved claim %s: %v", sb.ID, err) + } + sb.Touch() + m.recordUsage(ctx, usage.Event{Event: "move_in", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant, Deadline: m.deadlineOf(sb)}) + _ = os.RemoveAll(m.moveDir(sb.ID)) + logger.Infof(ctx, "sandbox %s moved here from %s", sb.ID, from) +} + +// dropMoveIn forgets a received claim whose move did not commit and removes its VM. +func (m *Manager) dropMoveIn(ctx context.Context, sb *types.Sandbox) { + if holds, ok := m.forgetClaim(ctx, sb); ok { + m.out.Disarm(sb.ID, true) + m.removeOrRetry(ctx, sb.VMName, "", volumeTeardown{holds: holds}) + } + _ = os.RemoveAll(m.moveDir(sb.ID)) +} + +// settleMoveIn resolves a received claim through its record until the registry answers. +func (m *Manager) settleMoveIn(ctx context.Context, sb *types.Sandbox) { + for !m.resolveMoveIn(ctx, sb) { + time.Sleep(moveRetryInterval) + } +} + +// resolveMoveIn starts a received claim whose record committed to this node and drops any other; false while the registry is down. +func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + pending := m.claimed[sb.ID] == sb && sb.MoveIn != nil + m.mu.Unlock() + if !pending { + return true + } + rec, ok, err := m.moves.Get(ctx, sb.ID) + if err != nil { + log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "resolve moved claim %s, retrying: %v", sb.ID, err) + return false + } + if ok && rec.State == moves.Committed && rec.To == m.nodeID { + m.startMoveIn(ctx, sb) + return true + } + m.dropMoveIn(ctx, sb) + return true +} + +func (m *Manager) touchReservation(id string) bool { + m.mu.Lock() + defer m.mu.Unlock() + in := m.incoming[id] + if in != nil { + in.touched = time.Now() + } + return in != nil +} + +// dropIdleReservations ends the reservations of sources that went quiet. +func (m *Manager) dropIdleReservations() { + now := time.Now() + var idle []string + m.mu.Lock() + for id, in := range m.incoming { + if now.Sub(in.touched) > moveReservationIdle { + idle = append(idle, id) + } + } + m.mu.Unlock() + for _, id := range idle { + m.DropMove(id) + } +} + +// incomingOf counts tenant's reservations; callers hold m.mu. +func (m *Manager) incomingOf(tenant string) int { + n := 0 + for _, in := range m.incoming { + if in.tenant == tenant { + n++ + } + } + return n +} + +// admitMoveLocked counts a move like a new claim and reserves its slot; callers hold m.mu. +func (m *Manager) admitMoveLocked(id string, a MoveAdmission) error { + if _, held := m.claimed[id]; held { + return fmt.Errorf("%w: this node already holds %s", ErrNotMovable, id) + } + if _, held := m.incoming[id]; held { + return fmt.Errorf("%w: a move of %s to this node is already running", ErrNotMovable, id) + } + if err := m.moveBudgetErr(); err != nil { + return err + } + if err := m.quotaErr(1, a.Tenant); err != nil { + return err + } + m.incoming[id] = &inMove{tenant: a.Tenant, from: a.From, touched: time.Now()} + return nil +} diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go new file mode 100644 index 00000000..8b1f481f --- /dev/null +++ b/sandboxd/pool/move_out.go @@ -0,0 +1,376 @@ +package pool + +import ( + "cmp" + "context" + "encoding/json/v2" + "errors" + "fmt" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/moves" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/usage" +) + +type movedAt struct { + epoch int64 + downtime time.Duration +} + +// Move hands a running claim to the peer at addr with cocoon's live migration, pausing its guest for about maxDowntime at most. +func (m *Manager) Move(ctx context.Context, id, addr string, maxDowntime time.Duration) (MoveResult, error) { + sb, ok := m.byID(id) + if !ok { + return MoveResult{}, ErrUnknownSandbox + } + if m.moves == nil || m.movePeer == nil { + return MoveResult{}, errMovesOff + } + if _, err := m.fenceWatch(sb); err != nil { + return MoveResult{}, err + } + if err := m.eng.CheckMigrateSupport(ctx); err != nil { + return MoveResult{}, fmt.Errorf("%w: %w", ErrNotMovable, err) + } + mv := &outMove{addr: addr} + if err := m.markMoving(sb, mv); err != nil { + return MoveResult{}, err + } + ctx = context.WithoutCancel(ctx) + paused, err := m.moveOut(ctx, sb, mv, cmp.Or(maxDowntime, DefaultMoveDowntime)) + if err == nil { + m.finishMoveOut(ctx, sb, mv, paused.epoch) + return MoveResult{Rounds: mv.rounds, Downtime: paused.downtime}, nil + } + return m.failMoveOut(ctx, sb, mv, err) +} + +// moveOut runs the rounds up to the target's commit; on any error the caller resolves the move through its record. +func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, budget time.Duration) (movedAt, error) { + m.mu.Lock() + admission, vm := MoveAdmission{Tenant: sb.Tenant, Key: sb.Key, From: m.nodeID}, sb.VMName + m.mu.Unlock() + to, err := m.movePeer.Admit(ctx, mv.addr, sb.ID, admission) + if err != nil { + return movedAt{}, err + } + mv.to = to + epoch, err := m.moves.Begin(ctx, sb.ID, m.nodeID, to) + if errors.Is(err, moves.ErrLost) { + return movedAt{}, fmt.Errorf("%w: another move of %s is on record", ErrNotMovable, sb.ID) + } + if err != nil { + return movedAt{}, err + } + mv.epoch = epoch + if err = m.journalMoveOut(ctx, sb, mv); err != nil { + return movedAt{}, err + } + round, err := m.eng.MigrateCapture(ctx, vm, "", false) + if err != nil { + return movedAt{}, fmt.Errorf("%w: baseline capture: %w", ErrNotMovable, err) + } + mv.series = round.Series + if err = m.journalMoveOut(ctx, sb, mv); err != nil { + return movedAt{}, err + } + var sent int64 + var sending time.Duration + fits := false + for i := 0; ; i++ { + start := time.Now() + if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { + return movedAt{}, err + } + sending += time.Since(start) + size := roundBytes(round) + sent += size + if i > 0 && finalPause(size, sent, sending) <= budget { + fits = true + } + if fits || i == moveDeltaLimit { + break + } + if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, false); err != nil { + return movedAt{}, fmt.Errorf("delta capture: %w", err) + } + } + if !fits { + return movedAt{}, fmt.Errorf("%w: %d deltas did not shrink under %s", ErrMoveDowntime, moveDeltaLimit, budget) + } + m.mu.Lock() + mv.final = true + m.mu.Unlock() + sb.Transition.Lock() + mv.locked = true + if err = m.unrowForMove(ctx, sb); err != nil { + return movedAt{}, err + } + pausedAt := time.Now() + if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, true); err != nil { + return movedAt{}, fmt.Errorf("final capture: %w", err) + } + final, err := m.moves.Advance(ctx, sb.ID, mv.epoch, moves.Pending, moves.Final) + if err != nil { + return movedAt{}, err + } + if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { + return movedAt{}, err + } + m.mu.Lock() + dto := dtoOf(sb) + m.mu.Unlock() + dto.MoveOut = nil + claim, err := json.Marshal(&dto) + if err != nil { + return movedAt{}, fmt.Errorf("encode claim: %w", err) + } + committed, err := m.movePeer.Commit(ctx, mv.addr, sb.ID, claim, final) + if err != nil { + return movedAt{}, err + } + return movedAt{epoch: committed, downtime: time.Since(pausedAt)}, nil +} + +func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round types.MigrateRound) error { + mv.rounds++ + names := make([]string, len(round.Files)) + for i, f := range round.Files { + names[i] = f.Name + } + return m.movePeer.SendRound(ctx, mv.addr, id, mv.rounds, round.Dir, names) +} + +// markMoving admits a move of sb and gates its captures, hibernates, releases and reaps until the move ends. +func (m *Manager) markMoving(sb *types.Sandbox, mv *outMove) error { + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[sb.ID] != sb { + return ErrUnknownSandbox + } + if reason := m.moveBlocker(sb); reason != "" { + return fmt.Errorf("%w: %s", ErrNotMovable, reason) + } + switch { + case m.evacuatingLocked(sb.ID): + return ErrEvacuating + case m.tenantGone(sb.Tenant): + return ErrTenantRemoved + } + if err := m.moveBudgetErr(); err != nil { + return err + } + m.moving[sb.ID] = mv + return nil +} + +// moveBlocker names why sb cannot move, "" when it can; callers hold m.mu. +func (m *Manager) moveBlocker(sb *types.Sandbox) string { + _, archiving := m.archiving[sb.ID] + _, moving := m.moving[sb.ID] + switch { + case sb.ArchiveCk != "" || archiving: + return "it is archived or archiving" + case sb.HibernateSnap != "" || sb.PendingSnap != "": + return "it is hibernated" + case sb.Failed != "": + return "it failed" + case sb.Waking: + return "it is waking" + case sb.MoveIn != nil || moving: + return "a move of it is already running" + case hasVolumes(sb) || m.hasVolumeHolds(sb): + return "it has volumes" + case sb.Key.Net != types.NetNone: + return "an egress-lane claim does not move" + } + return "" +} + +func (m *Manager) journalMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) error { + journal := &types.MoveOut{To: mv.to, Epoch: mv.epoch, Series: mv.series} + live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = journal }) + if !live { + return ErrUnknownSandbox + } + if err != nil { + return fmt.Errorf("journal the move of %s: %w", sb.ID, err) + } + return nil +} + +// unrowForMove deletes a replicated claim's row and replica, so no third node adopts a stale copy while the target runs it; the caller holds sb.Transition. +func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox) error { + if m.archives == nil { + return nil + } + if err := m.settleReplicaIntent(ctx, sb); err != nil { + return err + } + m.mu.Lock() + replica, epoch := sb.ReplicaCk, sb.ArchiveEpoch + m.mu.Unlock() + if epoch == 0 { + return nil + } + if err := m.archives.Delete(ctx, sb.ID, epoch); errors.Is(err, archives.ErrLost) { + return fmt.Errorf("%w: its replica row moved under it", ErrNotMovable) + } else if err != nil { + return err + } + live, err := m.commitIfLive(ctx, sb, func() { sb.ReplicaCk, sb.ArchiveEpoch = "", 0 }) + if !live { + return ErrUnknownSandbox + } + if replica != "" { + m.deleteOrphanArchiveCk(ctx, replica) + } + return err +} + +// finishMoveOut drops a claim the target committed, like a claim another node adopted. +func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, committed int64) { + m.mu.Lock() + vm := sb.VMName + m.mu.Unlock() + if teardown := m.forgetAdopted(ctx, sb); teardown != nil { + teardown() + m.recordUsage(ctx, usage.Event{Event: "move_out", ID: sb.ID, VMName: vm, Tenant: sb.Tenant}) + } + if err := m.moves.Delete(ctx, sb.ID, committed); err != nil { + log.WithFunc("pool.finishMoveOut").Warnf(ctx, "delete the committed move of %s: %v", sb.ID, err) + } + m.endMoveOut(sb, mv) +} + +// failMoveOut resolves a failed move through its record: committed finishes it, anything else aborts and resumes the guest. +func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, cause error) (MoveResult, error) { + if mv.epoch == 0 { + m.abortMoveOut(ctx, sb, mv) + return MoveResult{}, cause + } + committed, err := m.settleMoveOut(ctx, sb, mv) + switch { + case err != nil: + log.WithFunc("pool.failMoveOut").Warnf(ctx, "move of %s failed (%v) and its abort waits for the registry: %v", sb.ID, cause, err) + go m.retryMoveOut(ctx, sb, mv) + return MoveResult{}, fmt.Errorf("%w: %w", ErrMoveStoreDown, cause) + case committed: + return MoveResult{Rounds: mv.rounds}, nil + } + return MoveResult{}, cause +} + +func (m *Manager) retryMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) { + for { + time.Sleep(moveRetryInterval) + _, err := m.settleMoveOut(ctx, sb, mv) + if err == nil { + return + } + log.WithFunc("pool.retryMoveOut").Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) + } +} + +// settleMoveOut reads the record: committed drops the claim, pending or final is aborted before the guest resumes; an error leaves the move as it is. +func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) (bool, error) { + for range 2 { + rec, ok, err := m.moves.Get(ctx, sb.ID) + if err != nil { + return false, err + } + ours := ok && rec.From == m.nodeID && rec.Epoch >= mv.epoch + switch { + case ours && rec.State == moves.Committed: + m.finishMoveOut(ctx, sb, mv, rec.Epoch) + return true, nil + case ours && (rec.State == moves.Pending || rec.State == moves.Final): + aborted, err := m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted) + if errors.Is(err, moves.ErrLost) { + continue + } + if err != nil { + return false, err + } + rec.Epoch, rec.State = aborted, moves.Aborted + } + if ours && rec.State == moves.Aborted { + if err = m.moves.Delete(ctx, sb.ID, rec.Epoch); err != nil && !errors.Is(err, moves.ErrLost) { + return false, err + } + } + return false, m.abortMoveOut(ctx, sb, mv) + } + return false, fmt.Errorf("the move record of %s kept changing", sb.ID) +} + +// abortMoveOut ends the series, which resumes a paused guest, and releases the target and the budget slot. +func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) error { + if mv.series != "" { + m.mu.Lock() + vm := sb.VMName + m.mu.Unlock() + if err := m.eng.MigrateAbort(ctx, vm, mv.series); err != nil { + return fmt.Errorf("abort series %s: %w", mv.series, err) + } + } + if mv.addr != "" && mv.to != "" { + if err := m.movePeer.Drop(ctx, mv.addr, sb.ID); err != nil { + log.WithFunc("pool.abortMoveOut").Warnf(ctx, "drop the reservation for %s on %s: %v", sb.ID, mv.addr, err) + } + } + if _, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = nil }); err != nil { + log.WithFunc("pool.abortMoveOut").Warnf(ctx, "clear the move journal of %s: %v", sb.ID, err) + } + m.endMoveOut(sb, mv) + return nil +} + +func (m *Manager) endMoveOut(sb *types.Sandbox, mv *outMove) { + m.mu.Lock() + if m.moving[sb.ID] == mv { + delete(m.moving, sb.ID) + } + m.mu.Unlock() + if mv.locked { + mv.locked = false + sb.Transition.Unlock() + } +} + +// movingFinal reports whether sb's guest is in the final round of a move, when dials and wakes answer ErrEvacuating. +func (m *Manager) movingFinal(id string) bool { + m.mu.Lock() + defer m.mu.Unlock() + mv := m.moving[id] + return mv != nil && mv.final +} + +func (m *Manager) movingNow(id string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, moving := m.moving[id] + return moving +} + +func roundBytes(round types.MigrateRound) int64 { + var n int64 + for _, f := range round.Files { + n += f.Size + } + return n +} + +// finalPause estimates the final round's pause from the last delta's size at the rate the rounds so far reached. +func finalPause(last, sent int64, sending time.Duration) time.Duration { + if sent <= 0 { + return moveStartAllowance + } + return moveStartAllowance + time.Duration(float64(sending)*float64(last)/float64(sent)) +} diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go new file mode 100644 index 00000000..2f7e718e --- /dev/null +++ b/sandboxd/pool/move_test.go @@ -0,0 +1,542 @@ +package pool + +import ( + "cmp" + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/moves" + "github.com/cocoonstack/sandbox/sandboxd/moves/movestest" + "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestMoveHandsARunningClaimToThePeerWithBothNodesCountingIt(t *testing.T) { + mc := newMoveCluster(t) + sb := claimTenant(t, mc.s, "acme") + vm := sb.VMName + var during [][2][3]int + mc.p.roundHook = func(int) error { + during = append(during, [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}) + return nil + } + mc.p.commitHook = mc.p.roundHook + + res, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute) + if err != nil { + t.Fatalf("move: %v", err) + } + if res.Rounds != 3 || res.Downtime <= 0 { + t.Errorf("result %+v, want a baseline, one delta and the final round with a downtime", res) + } + for i, counts := range during { + if counts != [2][3]int{{1, 1, 1}, {1, 1, 1}} { + t.Errorf("step %d counts source/target %v, want 1/1/1 on both", i, counts) + } + } + if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + t.Errorf("counts after the move %v, want 0/0/0 on the source and 1/1/1 on the target", got) + } + moved, ok := mc.t.byID(sb.ID) + if !ok || moved.Token != sb.Token || moved.Tenant != "acme" || moved.MoveIn != nil || !moved.Deadline.Equal(sb.Deadline) { + t.Fatalf("target claim %+v %v, want the source's record started", moved, ok) + } + if _, held := mc.s.byID(sb.ID); held || !fakeOf(mc.s).removed(vm) { + t.Errorf("source still holds %s or its VM %s", sb.ID, vm) + } + if fakeOf(mc.t).paused[moved.VMName] || !slices.Contains(fakeOf(mc.t).starts, moved.VMName) { + t.Errorf("target VM %s not started", moved.VMName) + } + if _, ok, _ := mc.moveReg.Get(t.Context(), sb.ID); ok { + t.Error("the committed move record outlived the move") + } + if got := usageEventsOf(t, mc.s, sb.ID); got != "claim,move_out" { + t.Errorf("source usage %s, want claim,move_out", got) + } + if got := usageEventsOf(t, mc.t, sb.ID); got != "move_in" { + t.Errorf("target usage %s, want move_in", got) + } + assertNoMoveResidue(t, mc.t, sb.ID) +} + +func TestMoveDisarmsTheSourceDoorsAndArmsTheTarget(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + dialEgress(t, sb.VsockSocket) + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + if _, err := net.Dial("unix", engine.EgressSocketPath(sb.VsockSocket)); err == nil { + t.Error("the source still serves the moved claim's egress door") + } + moved, _ := mc.t.byID(sb.ID) + dialEgress(t, moved.VsockSocket) +} + +func TestMoveRefusedByTheTargetLeavesTheClaimRunningOnTheSource(t *testing.T) { + for name, refuse := range map[string]func(t *testing.T, mc *moveCluster){ + "max_claims": func(t *testing.T, mc *moveCluster) { + tuneView(mc.t, func(v *configView) { v.maxClaims = 1 }) + mustClaim(t, mc.t, testKey) + }, + "max_running": func(t *testing.T, mc *moveCluster) { + tuneView(mc.t, func(v *configView) { v.maxRunning = 1 }) + mustClaim(t, mc.t, testKey) + }, + "tenant cap": func(t *testing.T, mc *moveCluster) { + setTenantCaps(t, mc.t, map[string]int{"acme": 1}) + claimTenant(t, mc.t, "acme") + }, + "memory": func(_ *testing.T, mc *moveCluster) { mc.t.memoryLow.Store(true) }, + "move budget": func(t *testing.T, mc *moveCluster) { reserveMove(t, mc.t, "sb_other", "zed") }, + } { + t.Run(name, func(t *testing.T) { + mc := newMoveCluster(t) + tuneView(mc.t, func(v *configView) { v.moveConcurrency = 1 }) + sb := claimTenant(t, mc.s, "acme") + refuse(t, mc) + before := moveCounts(mc.t, "acme") + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrQuota) { + t.Fatalf("move: %v, want ErrQuota", err) + } + assertStillOnSource(t, mc, sb) + if got := moveCounts(mc.t, "acme"); got != before { + t.Errorf("target counts %v after the refusal, want %v", got, before) + } + assertNoMoveResidue(t, mc.t, sb.ID) + }) + } +} + +func TestMoveFailingAtAnyStepKeepsOneOwner(t *testing.T) { + for name, tt := range map[string]struct { + budget time.Duration + fail func(mc *moveCluster) + want error + }{ + "admission": {fail: func(mc *moveCluster) { mc.p.admitErr = ErrMoveTarget }, want: ErrMoveTarget}, + "move record": {fail: func(mc *moveCluster) { mc.moveReg.SetDown(true) }, want: ErrMoveStoreDown}, + "baseline": {fail: func(mc *moveCluster) { fakeOf(mc.s).captureErr = errors.New("no reflink") }, want: ErrNotMovable}, + "round transfer": {fail: func(mc *moveCluster) { mc.p.roundHook = failRound(2) }, want: ErrMoveTarget}, + "downtime budget": {budget: time.Millisecond, want: ErrMoveDowntime}, + "final capture": {fail: func(mc *moveCluster) { fakeOf(mc.s).finalCaptureErr = errors.New("pause failed") }}, + "final transfer": {fail: func(mc *moveCluster) { mc.p.roundHook = failRound(3) }, want: ErrMoveTarget}, + "receive": {fail: func(mc *moveCluster) { fakeOf(mc.t).receiveErr = errors.New("restore failed") }}, + "target memory": {fail: func(mc *moveCluster) { mc.p.commitHook = func(int) error { mc.t.memoryLow.Store(true); return nil } }, want: ErrQuota}, + "source aborted first": {fail: func(mc *moveCluster) { + mc.p.commitHook = func(int) error { abortRecord(mc); return nil } + }, want: ErrNotMovable}, + } { + t.Run(name, func(t *testing.T) { + mc := newMoveCluster(t) + sb := claimTenant(t, mc.s, "acme") + if tt.fail != nil { + tt.fail(mc) + } + _, err := mc.s.Move(t.Context(), sb.ID, "b", cmp.Or(tt.budget, time.Minute)) + if err == nil || tt.want != nil && !errors.Is(err, tt.want) { + t.Fatalf("move: %v, want %v", err, tt.want) + } + mc.moveReg.SetDown(false) + assertStillOnSource(t, mc, sb) + if got := moveCounts(mc.t, "acme"); got != [3]int{} { + t.Errorf("target counts %v, want none", got) + } + assertNoMoveResidue(t, mc.t, sb.ID) + if _, ok, _ := mc.moveReg.Get(t.Context(), sb.ID); ok { + t.Error("a move record outlived the failed move") + } + }) + } +} + +func TestMoveWhoseCommitAnswerIsLostResolvesThroughTheRecord(t *testing.T) { + mc := newMoveCluster(t) + sb := claimTenant(t, mc.s, "acme") + mc.p.loseCommitReply = true + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move whose commit answer was lost: %v, want the committed move", err) + } + if _, held := mc.s.byID(sb.ID); held { + t.Error("the source kept a claim the target committed") + } + if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn != nil { + t.Errorf("target claim %+v %v, want it started", moved, ok) + } + if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + t.Errorf("counts %v, want the claim on the target only", got) + } +} + +func TestMoveKeepsThePausedGuestUntilItsAbortLands(t *testing.T) { + mc := newMoveCluster(t) + sb := claimTenant(t, mc.s, "acme") + mc.p.commitHook = func(int) error { + mc.moveReg.SetDown(true) + return ErrMoveTarget + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveStoreDown) { + t.Fatalf("move with the registry down after the final round: %v, want ErrMoveStoreDown", err) + } + if !pausedOn(mc.s, sb.VMName) { + t.Fatal("the source resumed its guest before the abort landed") + } + if _, _, _, err := mc.s.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrEvacuating) { + t.Errorf("dial during the unresolved final round: %v, want ErrEvacuating", err) + } + time.Sleep(2 * moveRetryInterval) + if !pausedOn(mc.s, sb.VMName) { + t.Fatal("the source resumed its guest while the registry was down") + } + mc.moveReg.SetDown(false) + waitFor(t, func() bool { return !pausedOn(mc.s, sb.VMName) && !mc.s.movingNow(sb.ID) }) + assertStillOnSource(t, mc, sb) + if rec, ok, _ := mc.moveReg.Get(t.Context(), sb.ID); ok { + t.Errorf("record %+v after the abort, want it deleted", rec) + } +} + +func TestMoveTargetCountsAReservationLikeAClaim(t *testing.T) { + mc := newMoveCluster(t) + tuneView(mc.t, func(v *configView) { v.maxClaims = 1 }) + reserveMove(t, mc.t, "sb_r", "acme") + if _, err := claimAny(t.Context(), mc.t, testKey, ClaimOptions{}); !errors.Is(err, ErrQuota) { + t.Fatalf("claim beside a reservation at max_claims 1: %v, want ErrQuota", err) + } + if got := mc.t.Sandboxes("", "", nil); len(got) != 0 { + t.Errorf("listing shows %d claims, want the reservation invisible", len(got)) + } + mc.t.DropMove("sb_r") + mustClaim(t, mc.t, testKey) +} + +func TestMoveTargetDropsAReservationItsSourceAbandoned(t *testing.T) { + mc := newMoveCluster(t) + reserveMove(t, mc.t, "sb_r", "acme") + mc.t.mu.Lock() + mc.t.incoming["sb_r"].touched = time.Now().Add(-2 * moveReservationIdle) + mc.t.mu.Unlock() + mc.t.dropIdleReservations() + assertNoMoveResidue(t, mc.t, "sb_r") +} + +func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) + third := mc.moveNode(t, "c") + sb := mustClaim(t, mc.s, testKey) + replicateNow(t, mc.s, sb) + mc.s.mu.Lock() + replica := sb.ReplicaCk + mc.s.mu.Unlock() + if _, ok, _ := mc.failoverCluster.reg.Get(t.Context(), sb.ID); !ok || replica == "" { + t.Fatal("setup: the claim has no replica row") + } + mc.p.commitHook = func(int) error { + if _, ok, _ := mc.failoverCluster.reg.Get(t.Context(), sb.ID); ok { + t.Error("the replica row is still adoptable at the commit") + } + mc.failoverCluster.reg.Expire("a") + if adopted, err := third.AdoptArchived(t.Context(), sb.ID, sb.Token); adopted || err != nil { + t.Errorf("a third node adopts the moving claim: %v %v", adopted, err) + } + _ = mc.failoverCluster.reg.Renew(t.Context(), "a", time.Hour) + return nil + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + if ckExists(t, mc.s, replica) { + t.Error("the stale replica outlived the move") + } +} + +func TestMoveHoldsOffCapturesReleasesAndEvacuationWhileItRuns(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + other := mustClaim(t, mc.s, testKey) + cred := Cred{Token: sb.Token} + mc.p.roundHook = func(round int) error { + if round != 2 { + return nil + } + _, forkErr := mc.s.Fork(t.Context(), sb.ID, cred, 1, time.Minute, "", "") + _, ckErr := mc.s.Checkpoint(t.Context(), sb.ID, cred, "ck", "") + _, _, promoteErr := mc.s.Promote(t.Context(), sb.ID, cred, "tpl", "") + for op, err := range map[string]error{ + "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, + "hibernate": mc.s.Hibernate(t.Context(), sb.ID, cred), + "release": mc.s.Release(t.Context(), sb.ID, cred), + } { + if !errors.Is(err, ErrEvacuating) { + t.Errorf("%s during a move: %v, want ErrEvacuating", op, err) + } + } + mc.s.mu.Lock() + replicable := mc.s.replicable(sb) + mc.s.mu.Unlock() + if replicable { + t.Error("a replica round may capture the moving claim") + } + expire(mc.s, sb) + mc.s.reapOnce(t.Context()) + if _, held := mc.s.byID(sb.ID); !held { + t.Error("the reap took the moving claim") + } + tuneView(mc.s, func(v *configView) { v.moveConcurrency = 1 }) + if _, err := mc.s.Move(t.Context(), other.ID, "b", time.Minute); !errors.Is(err, ErrQuota) { + t.Errorf("a second move past a lowered move_concurrency: %v, want ErrQuota", err) + } + return nil + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("the move the lowered budget did not stop: %v", err) + } + if _, held := mc.t.byID(sb.ID); !held { + t.Error("the target does not hold the moved claim") + } +} + +func TestMoveRefusesClaimsThatCannotMove(t *testing.T) { + mc := newMoveCluster(t) + hibernatedSb := mustClaim(t, mc.s, testKey) + hibernateClaim(t, mc.s, hibernatedSb) + if _, err := mc.s.Move(t.Context(), hibernatedSb.ID, "b", time.Minute); !errors.Is(err, ErrNotMovable) { + t.Errorf("move of a hibernated claim: %v, want ErrNotMovable", err) + } + fakeOf(mc.s).migrateSupportErr = errors.New("cocoon too old") + sb := mustClaim(t, mc.s, testKey) + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrNotMovable) { + t.Errorf("move on a cocoon without migration: %v, want ErrNotMovable", err) + } + if _, err := mc.s.Move(t.Context(), "sb_none", "b", time.Minute); !errors.Is(err, ErrUnknownSandbox) { + t.Errorf("move of an unknown claim: %v, want ErrUnknownSandbox", err) + } + if len(fakeOf(mc.s).captures) != 0 { + t.Errorf("a refused move captured %v", fakeOf(mc.s).captures) + } +} + +func BenchmarkQuotaErr(b *testing.B) { + for _, incoming := range []int{0, 2} { + b.Run(map[int]string{0: "no reservations", 2: "two reservations"}[incoming], func(b *testing.B) { + m := newTestManager(b, newFakeEngine()) + for i := range incoming { + m.incoming["sb_"+string(rune('a'+i))] = &inMove{tenant: "acme", touched: time.Now()} + } + tuneView(m, func(v *configView) { v.maxClaims, v.maxRunning = 1000, 1000 }) + for b.Loop() { + m.mu.Lock() + _ = m.quotaErr(1, "") + m.mu.Unlock() + } + }) + } +} + +type moveCluster struct { + *failoverCluster + moveReg *movestest.Mem + p *fakeMovePeer + s, t *Manager +} + +func newMoveCluster(t *testing.T) *moveCluster { + t.Helper() + return newMoveClusterWith(t, func(*failoverCluster) {}) +} + +func newMoveClusterWith(t *testing.T, edit func(*failoverCluster)) *moveCluster { + t.Helper() + c := newFailoverCluster(t) + c.egressPol = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + edit(c) + reg := movestest.NewMem() + c.moves = reg + mc := &moveCluster{failoverCluster: c, moveReg: reg, p: newFakeMovePeer(t, c)} + mc.s, mc.t = mc.moveNode(t, "a"), mc.moveNode(t, "b") + return mc +} + +func (mc *moveCluster) moveNode(t *testing.T, name string) *Manager { + t.Helper() + eng := newFakeEngine() + eng.sockRoot, eng.migrateDir = outboundtest.SockRoot(t), t.TempDir() + m := mc.nodeOn(t, eng, name, t.TempDir(), keepAnHour) + m.liftFence() + m.WithMovePeer(mc.p) + setTenantCaps(t, m, map[string]int{"acme": 0}) + return m +} + +type fakeMovePeer struct { + c *failoverCluster + srv *httptest.Server + + mu sync.Mutex + admitErr error + roundHook func(round int) error + commitHook func(round int) error + loseCommitReply bool +} + +func newFakeMovePeer(t *testing.T, c *failoverCluster) *fakeMovePeer { + t.Helper() + p := &fakeMovePeer{c: c} + p.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + node, id, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + if err := p.target(node).ReceiveMoveRound(id, r.Body); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(p.srv.Close) + return p +} + +func (p *fakeMovePeer) Admit(_ context.Context, addr, id string, a MoveAdmission) (string, error) { + if p.admitErr != nil { + return "", p.admitErr + } + return p.target(addr).AdmitMove(id, a) +} + +func (p *fakeMovePeer) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error { + if hook := p.roundHook; hook != nil { + if err := hook(round); err != nil { + return err + } + } + if err := peer.SendRound(ctx, p.srv.URL, "", "/"+addr+"/"+id, dir, files); err != nil { + return errors.Join(ErrMoveTarget, err) + } + return nil +} + +func (p *fakeMovePeer) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) (int64, error) { + if hook := p.commitHook; hook != nil { + if err := hook(0); err != nil { + return 0, err + } + } + committed, err := p.target(addr).CommitMove(ctx, id, claim, epoch) + if err == nil && p.loseCommitReply { + return 0, ErrMoveTarget + } + return committed, err +} + +func (p *fakeMovePeer) Drop(_ context.Context, addr, id string) error { + p.target(addr).DropMove(id) + return nil +} + +func (p *fakeMovePeer) target(addr string) *Manager { + p.mu.Lock() + defer p.mu.Unlock() + i := slices.IndexFunc(p.c.nodes, func(m *Manager) bool { return m.nodeID == addr }) + return p.c.nodes[i] +} + +func moveCounts(m *Manager, tenant string) [3]int { + m.mu.Lock() + defer m.mu.Unlock() + return [3]int{len(m.claimed) + len(m.incoming), m.store.runningCount() + len(m.incoming), m.tenantLive[tenant] + m.incomingOf(tenant)} +} + +func reserveMove(t *testing.T, m *Manager, id, tenant string) { + t.Helper() + if _, err := m.AdmitMove(id, MoveAdmission{Tenant: tenant, Key: testKey, From: "zz"}); err != nil { + t.Fatalf("reserve %s: %v", id, err) + } +} + +func assertStillOnSource(t *testing.T, mc *moveCluster, sb *types.Sandbox) { + t.Helper() + held, ok := mc.s.byID(sb.ID) + if !ok || held != sb { + t.Fatalf("the source lost %s", sb.ID) + } + mc.s.mu.Lock() + journal, moving, vm := sb.MoveOut, mc.s.moving[sb.ID], sb.VMName + mc.s.mu.Unlock() + if journal != nil || moving != nil || pausedOn(mc.s, vm) || fakeOf(mc.s).removed(vm) { + t.Errorf("source claim journal %+v moving %v paused %v, want running and unmarked", journal, moving, pausedOn(mc.s, vm)) + } + if _, held := mc.t.byID(sb.ID); held { + t.Error("the target holds the claim too") + } +} + +func assertNoMoveResidue(t *testing.T, m *Manager, id string) { + t.Helper() + m.mu.Lock() + _, reserved := m.incoming[id] + m.mu.Unlock() + if reserved { + t.Errorf("node %s keeps a reservation for %s", m.nodeID, id) + } + if _, err := os.Stat(m.moveDir(id)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("node %s keeps the staging dir of %s: %v", m.nodeID, id, err) + } + if _, claimed := m.byID(id); claimed { + return + } + eng := fakeOf(m) + eng.mu.Lock() + defer eng.mu.Unlock() + for _, name := range eng.receives { + if _, live := eng.vms[name]; live { + t.Errorf("node %s keeps the received VM %s", m.nodeID, name) + } + } +} + +func dialEgress(t *testing.T, sock string) { + t.Helper() + conn, err := net.Dial("unix", engine.EgressSocketPath(sock)) + if err != nil { + t.Fatalf("no egress door at %s: %v", sock, err) + } + _ = conn.Close() +} + +func pausedOn(m *Manager, vm string) bool { + eng := fakeOf(m) + eng.mu.Lock() + defer eng.mu.Unlock() + return eng.paused[vm] +} + +func failRound(n int) func(int) error { + return func(round int) error { + if round == n { + return ErrMoveTarget + } + return nil + } +} + +func abortRecord(mc *moveCluster) { + ctx := context.Background() + for _, rec := range mustStale(mc) { + _, _ = mc.moveReg.Advance(ctx, rec.ID, rec.Epoch, rec.State, moves.Aborted) + } +} + +func mustStale(mc *moveCluster) []moves.Record { + recs, _ := mc.moveReg.Stale(context.Background(), time.Now().Add(time.Hour)) + return recs +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index d42781b9..38bbcb1a 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -120,6 +120,10 @@ var ( ErrQuota = errors.New("node claim quota reached") ErrSleeping = errors.New("sandbox is hibernated or archived and a reader may serve it") ErrNoReader = errors.New("no reader for the sandbox") + ErrNotMovable = errors.New("sandbox cannot move") + ErrMoveTarget = errors.New("move target failed") + ErrMoveDowntime = errors.New("the final round would exceed max_downtime_ms") + ErrMoveStoreDown = moves.ErrUnavailable errNotSleeping = errors.New("sandbox is no longer hibernated or archived") errWokeMeanwhile = errors.New("woke between sweep and hibernate") @@ -128,6 +132,7 @@ var ( errNoArchiveRow = errors.New("archived before archive_failover was on, so no registry row names it") errArchiveBusy = errors.New("an archive of the claim is in flight") errNoAdopter = errors.New("no peer adopted the claim") + errMovesOff = fmt.Errorf("%w: live moves need archive_failover and an encrypted mesh", ErrNotMovable) ) // Engine is the slice of the cocoon driver the manager consumes. @@ -169,6 +174,10 @@ type Engine interface { DialReader(ctx context.Context, vsockSocket string) (net.Conn, error) WriteInstanceMetadata(ctx context.Context, vsockSocket string, doc []byte) error WriteGuestEnv(ctx context.Context, vsockSocket string, doc []byte) error + CheckMigrateSupport(ctx context.Context) error + MigrateCapture(ctx context.Context, vmName, series string, final bool) (types.MigrateRound, error) + MigrateReceive(ctx context.Context, fromDir, name string, key types.PoolKey) (types.VMRecord, error) + MigrateAbort(ctx context.Context, vmName, series string) error } type VolumeMountConflict = engine.VolumeMountConflict @@ -287,7 +296,8 @@ type Manager struct { replicating atomic.Bool replicas map[string]time.Time // moves is the cluster move registry, nil unless the node can run live moves. - moves moves.Registry + moves moves.Registry + movePeer MovePeer view atomic.Pointer[configView] // cfg is the config the view was built from; reloadMu orders reloads, the only writers of cfg and view. @@ -334,6 +344,9 @@ type Manager struct { pools map[types.PoolKey]*pool claimed map[string]*types.Sandbox pendingRemovals map[string]pendingRemoval + // moving holds this node's outgoing moves, incoming its reservations for moves to it; neither enters claimed. + moving map[string]*outMove + incoming map[string]*inMove // Node-wide, unlike the per-pool streak backoff: the exhausted resource is shared atCapacityUntil time.Time @@ -378,6 +391,8 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg pools: make(map[types.PoolKey]*pool, len(cfg.Pools)), claimed: map[string]*types.Sandbox{}, pendingRemovals: map[string]pendingRemoval{}, + moving: map[string]*outMove{}, + incoming: map[string]*inMove{}, tenantLive: map[string]int{}, readers: &readerSet{live: map[string]*reader{}}, archiving: map[string]struct{}{}, @@ -486,6 +501,7 @@ func (m *Manager) Run(ctx context.Context) { case <-m.refillKick: m.refillOnce(ctx) case <-reap: + m.dropIdleReservations() m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) @@ -503,6 +519,9 @@ func (m *Manager) Run(ctx context.Context) { if m.archives != nil { go m.sweepOrphanArchives(ctx) } + if m.moves != nil { + go m.sweepStaleMoves(ctx) + } } } } @@ -709,7 +728,7 @@ func logSweepResult(ctx context.Context, logger *log.Fields, err error, okMsg, f func benignSweepErr(err error) bool { return errors.Is(err, ErrUnknownSandbox) || errors.Is(err, errWokeMeanwhile) || - errors.Is(err, ErrNoEgressHibernate) || errors.Is(err, ErrQuota) || errors.Is(err, ErrLeaseLost) + errors.Is(err, ErrNoEgressHibernate) || errors.Is(err, ErrQuota) || errors.Is(err, ErrLeaseLost) || errors.Is(err, ErrEvacuating) } func vmName(key types.PoolKey) string { diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 99a72b3b..dc75e63f 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -1334,6 +1335,20 @@ type fakeEngine struct { metadataErr error guestEnvs map[string]string guestEnvErr error + + migrateDir string + migrateSeries map[string]string + migrateRound map[string]int + migrateSizes []int64 + paused map[string]bool + captures []string + receives []string + aborts []string + migrateSupportErr error + captureErr error + finalCaptureErr error + receiveErr error + abortErr error } func newFakeEngine() *fakeEngine { @@ -1341,6 +1356,7 @@ func newFakeEngine() *fakeEngine { vms: map[string]string{}, stopped: map[string]bool{}, creating: map[string]bool{}, pids: map[string]int{}, attachDirty: map[string]bool{}, removeSeenOps: map[string][]string{}, removeSeenDirty: map[string][]string{}, metadataDocs: map[string]string{}, guestEnvs: map[string]string{}, sockRoot: "/vsock", + migrateSeries: map[string]string{}, migrateRound: map[string]int{}, paused: map[string]bool{}, } } @@ -1829,6 +1845,70 @@ func (f *fakeEngine) DialReader(_ context.Context, vsockSocket string) (net.Conn return conn, nil } +func (f *fakeEngine) CheckMigrateSupport(context.Context) error { return f.migrateSupportErr } + +func (f *fakeEngine) MigrateCapture(_ context.Context, vmName, series string, final bool) (types.MigrateRound, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.captures = append(f.captures, fmt.Sprintf("%s:%s:%v", vmName, series, final)) + switch { + case f.captureErr != nil: + return types.MigrateRound{}, f.captureErr + case final && f.finalCaptureErr != nil: + f.paused[vmName] = true + return types.MigrateRound{}, f.finalCaptureErr + case series == "": + series = "s-" + vmName + "-" + strconv.Itoa(len(f.captures)) + f.migrateSeries[vmName], f.migrateRound[vmName] = series, 0 + case f.migrateSeries[vmName] != series: + return types.MigrateRound{}, fmt.Errorf("series %s is not live on %s", series, vmName) + } + round := f.migrateRound[vmName] + f.migrateRound[vmName]++ + dir := filepath.Join(f.migrateDir, series, strconv.Itoa(round)) + if err := os.MkdirAll(dir, 0o750); err != nil { + return types.MigrateRound{}, err + } + if err := os.WriteFile(filepath.Join(dir, "state.json"), []byte(vmName+" round "+strconv.Itoa(round)), 0o600); err != nil { + return types.MigrateRound{}, err + } + size := int64(1) + if len(f.migrateSizes) > 0 { + size, f.migrateSizes = f.migrateSizes[0], f.migrateSizes[1:] + } + f.paused[vmName] = final + return types.MigrateRound{Series: series, Dir: dir, Round: round, Paused: final, Files: []types.MigrateFile{{Name: "state.json", Size: size}}}, nil +} + +func (f *fakeEngine) MigrateReceive(_ context.Context, fromDir, name string, _ types.PoolKey) (types.VMRecord, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.receives = append(f.receives, name) + if f.receiveErr != nil { + f.vms[name], f.creating[name] = f.sockRoot+"/"+name, true + return types.VMRecord{}, f.receiveErr + } + if _, err := os.Stat(filepath.Join(fromDir, "state.json")); err != nil { + return types.VMRecord{}, err + } + f.vms[name], f.stopped[name] = f.sockRoot+"/"+name, true + return f.record(name), nil +} + +func (f *fakeEngine) MigrateAbort(_ context.Context, vmName, series string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.aborts = append(f.aborts, vmName+":"+series) + if f.abortErr != nil { + return f.abortErr + } + if f.migrateSeries[vmName] == series { + delete(f.migrateSeries, vmName) + f.paused[vmName] = false + } + return nil +} + func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/reap.go b/sandboxd/pool/reap.go index 5b0b0303..d53fbd23 100644 --- a/sandboxd/pool/reap.go +++ b/sandboxd/pool/reap.go @@ -36,7 +36,7 @@ func (m *Manager) reapOnce(ctx context.Context) { var unrows []*types.Sandbox for id, sb := range m.claimed { // A zero deadline means no expiry (an archived claim kept forever). - if sb.Deadline.IsZero() || !now.After(sb.Deadline) { + if _, moving := m.moving[id]; moving || sb.Deadline.IsZero() || !now.After(sb.Deadline) { continue } switch { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 5e8099ba..92926146 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -67,7 +67,8 @@ func (m *Manager) replicateOnce(ctx context.Context) { // replicable excludes a claim that is paused, archived or archiving, failed, or whose tenant was removed; callers hold m.mu. func (m *Manager) replicable(sb *types.Sandbox) bool { _, archiving := m.archiving[sb.ID] - return !archiving && runningShaped(sb) && sb.ArchivePending == "" && !m.tenantGone(sb.Tenant) + _, moving := m.moving[sb.ID] + return !archiving && !moving && runningShaped(sb) && sb.ArchivePending == "" && !m.tenantGone(sb.Tenant) } func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { @@ -108,7 +109,7 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { var intent claimSnapshot var refusal error switch { - case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "": + case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "" || m.moving[sb.ID] != nil: refusal = errWokeMeanwhile case m.hasVolumeHolds(sb): refusal = ErrVolumeCapture diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4c57f5cd..ec2a83fb 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -304,6 +304,10 @@ type Sandbox struct { RestartedAt time.Time `json:"restarted_at,omitzero"` // Failed names why the VMM is down and not restarted; empty means the claim is serviceable. Failed string `json:"failed,omitempty"` + // MoveOut journals a live move this node runs as the source; nil when none. + MoveOut *MoveOut `json:"move_out,omitempty"` + // MoveIn marks a claim a live move delivered here and this node has not started yet. + MoveIn *MoveIn `json:"move_in,omitempty"` // RestartLog holds the recent restart attempts that the loop budget counts; guarded by Transition. RestartLog []time.Time `json:"-"` // ExitRecorded marks a VMM exit already journaled as vmm_exit; guarded by Transition. @@ -454,6 +458,19 @@ func (r VMRecord) TapDevice() string { return r.NetworkConfigs[0].TAP } +// MoveOut is the source's journal of a live move: the target node, the record's first epoch and cocoon's series. +type MoveOut struct { + To string `json:"to"` + Epoch int64 `json:"epoch"` + Series string `json:"series,omitempty"` +} + +// MoveIn is the target's journal of a received claim: the source node and the epoch its commit wrote. +type MoveIn struct { + From string `json:"from"` + Epoch int64 `json:"epoch"` +} + // MigrateFile is one file a migrate round wrote, named relative to the round's Dir. type MigrateFile struct { Name string `json:"name"` From 352b97428ae7fe8347c7baa7e381037b586594fa Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:36:08 +0800 Subject: [PATCH 08/53] pool: resolve a journaled move through its record after a restart A source with move_out keeps the claim gated until the record answers: committed drops it, pending or final is aborted before the guest resumes. A target with move_in starts the received VM only when the record committed to it, and keeps only those claims' staging dirs. --- sandboxd/pool/move_test.go | 87 ++++++++++++++++++++++++++++++++++++++ sandboxd/pool/reconcile.go | 41 ++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 2f7e718e..59ca323c 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "slices" "strings" "sync" @@ -328,6 +329,92 @@ func TestMoveRefusesClaimsThatCannotMove(t *testing.T) { } } +func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { + for name, tt := range map[string]struct { + node string + reach moves.State + journal func(epoch int64) *types.Sandbox + wantClaim bool + check func(t *testing.T, eng *fakeEngine) + }{ + "source at pending": {node: "a", reach: moves.Pending, wantClaim: true, journal: func(epoch int64) *types.Sandbox { + return &types.Sandbox{MoveOut: &types.MoveOut{To: "b", Epoch: epoch, Series: "s1"}} + }, check: func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.aborts, "sbx-moved:s1") { + t.Errorf("aborts %v, want the series aborted", eng.aborts) + } + }}, + "source at final": {node: "a", reach: moves.Final, wantClaim: true, journal: func(epoch int64) *types.Sandbox { + return &types.Sandbox{MoveOut: &types.MoveOut{To: "b", Epoch: epoch, Series: "s1"}} + }, check: func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.aborts, "sbx-moved:s1") { + t.Errorf("aborts %v, want the series aborted", eng.aborts) + } + }}, + "source at committed": {node: "a", reach: moves.Committed, journal: func(epoch int64) *types.Sandbox { + return &types.Sandbox{MoveOut: &types.MoveOut{To: "b", Epoch: epoch, Series: "s1"}} + }, check: func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.removes, "sbx-moved") { + t.Errorf("removes %v, want the moved-away VM removed", eng.removes) + } + }}, + "target at committed": {node: "b", reach: moves.Committed, wantClaim: true, journal: func(epoch int64) *types.Sandbox { + return &types.Sandbox{MoveIn: &types.MoveIn{From: "a", Epoch: epoch}} + }, check: func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.starts, "sbx-moved") { + t.Errorf("starts %v, want the received VM started", eng.starts) + } + }}, + "target at aborted": {node: "b", reach: moves.Aborted, journal: func(epoch int64) *types.Sandbox { + return &types.Sandbox{MoveIn: &types.MoveIn{From: "a", Epoch: epoch}} + }, check: func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.removes, "sbx-moved") || slices.Contains(eng.starts, "sbx-moved") { + t.Errorf("removes %v starts %v, want the received VM removed unstarted", eng.removes, eng.starts) + } + }}, + } { + t.Run(name, func(t *testing.T) { + c := newFailoverCluster(t) + reg := movestest.NewMem() + c.moves = reg + ctx := t.Context() + epoch, _ := reg.Begin(ctx, "sb_moved", "a", "b") + path := map[moves.State][]moves.State{moves.Final: {moves.Final}, moves.Committed: {moves.Final, moves.Committed}, moves.Aborted: {moves.Aborted}}[tt.reach] + from, at := moves.Pending, epoch + for _, to := range path { + at, _ = reg.Advance(ctx, "sb_moved", at, from, to) + from = to + } + sb := tt.journal(epoch) + sb.ID, sb.VMName, sb.Key, sb.Token, sb.Deadline = "sb_moved", "sbx-moved", testKey, "tok", time.Now().Add(time.Hour) + dataDir := t.TempDir() + seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) + if err := os.MkdirAll(filepath.Join(dataDir, movesDir, sb.ID), 0o700); err != nil { + t.Fatal(err) + } + eng := newFakeEngine() + eng.vms["sbx-moved"], eng.stopped["sbx-moved"] = "/vsock/sbx-moved", sb.MoveIn != nil + m := c.nodeOn(t, eng, tt.node, dataDir, keepAnHour) + m.liftFence() + if err := m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + waitFor(t, func() bool { return !m.moveHeld(sb.ID) }) + waitFor(t, func() bool { _, ok, _ := reg.Get(ctx, sb.ID); return !ok || tt.node == "b" }) + held, ok := m.byID(sb.ID) + if ok != tt.wantClaim { + t.Fatalf("claim held %v, want %v", ok, tt.wantClaim) + } + if ok && (held.MoveOut != nil || held.MoveIn != nil) { + t.Errorf("claim journal out %+v in %+v, want both cleared", held.MoveOut, held.MoveIn) + } + eng.mu.Lock() + defer eng.mu.Unlock() + tt.check(t, eng) + }) + } +} + func BenchmarkQuotaErr(b *testing.B) { for _, incoming := range []int{0, 2} { b.Run(map[int]string{0: "no reservations", 2: "two reservations"}[incoming], func(b *testing.B) { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index be5b1c8f..7189f9d3 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -75,6 +75,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } + outs, ins := m.pendingMovesLocked() saveErr := m.store.save(m.claimed) pooled := make(map[string]bool, len(m.pools)) for _, p := range m.pools { @@ -110,6 +111,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { // A crash mid-export leaves a fork-* staging dir nothing in this life reuses _ = utils.RemoveDirEntries(m.dataDir, func(name string) bool { return strings.HasPrefix(name, "fork-") }) _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, readerDir), nil) + _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, movesDir), func(id string) bool { return claims[id] == nil || claims[id].MoveIn == nil }) if fenced { m.reclaimOrphanArchiveCks(ctx, claims) @@ -133,11 +135,50 @@ func (m *Manager) Reconcile(ctx context.Context) error { }).Wait() } m.resyncEgress(ctx, live, removed) + m.resumeMoves(context.WithoutCancel(ctx), outs, ins) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) return saveErr } +// pendingMovesLocked marks the journaled moves of loaded claims: a source keeps its claim gated until the record resolves; callers hold m.mu. +func (m *Manager) pendingMovesLocked() (map[*types.Sandbox]*outMove, []*types.Sandbox) { + if m.moves == nil { + return nil, nil + } + outs := map[*types.Sandbox]*outMove{} + var ins []*types.Sandbox + for id, sb := range m.claimed { + switch { + case sb.MoveOut != nil: + mv := &outMove{to: sb.MoveOut.To, epoch: sb.MoveOut.Epoch, series: sb.MoveOut.Series, final: true} + m.moving[id], outs[sb] = mv, mv + case sb.MoveIn != nil: + ins = append(ins, sb) + } + } + return outs, ins +} + +// resumeMoves resolves each journaled move through its record in the background, retrying while the registry is down. +func (m *Manager) resumeMoves(ctx context.Context, outs map[*types.Sandbox]*outMove, ins []*types.Sandbox) { + for sb, mv := range outs { + go func() { + sb.Transition.Lock() + mv.locked = true + for { + if _, err := m.settleMoveOut(ctx, sb, mv); err == nil { + return + } + time.Sleep(moveRetryInterval) + } + }() + } + for _, sb := range ins { + go m.settleMoveIn(ctx, sb) + } +} + // sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRecord, owned map[string]bool) map[string]bool { var stale []string From bbfcaa8dda3c59e4462a71de10f7f26de7bee07a Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:36:08 +0800 Subject: [PATCH 09/53] server: the move endpoint and the peer routes a move's target serves --- sandboxd/main.go | 1 + sandboxd/server/move.go | 205 +++++++++++++++++++++++++++++++++ sandboxd/server/move_test.go | 88 ++++++++++++++ sandboxd/server/server.go | 14 +++ sandboxd/server/server_test.go | 15 +++ 5 files changed, 323 insertions(+) create mode 100644 sandboxd/server/move.go create mode 100644 sandboxd/server/move_test.go diff --git a/sandboxd/main.go b/sandboxd/main.go index 3c74db4d..370eafc5 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -134,6 +134,7 @@ func main() { mgr.WithPeerDelete(broadcaster.Delete) adopter := &peer.Adopter{Peers: msh.PeerAddrs} mgr.WithPeerAdopt(adopter.Adopt) + mgr.WithMovePeer(&server.MoveClient{Token: cfg.APIToken}) go gossipNodeState(ctx, msh, mgr) logger.Infof(ctx, "mesh %s joined (%d seeds)", cfg.Mesh.ID(), len(cfg.Mesh.Join)) } diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go new file mode 100644 index 00000000..3ce6e1fb --- /dev/null +++ b/sandboxd/server/move.go @@ -0,0 +1,205 @@ +package server + +import ( + "bytes" + "cmp" + "context" + "encoding/json/jsontext" + "encoding/json/v2" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const moveCallTimeout = 10 * time.Minute + +var _ pool.MovePeer = (*MoveClient)(nil) + +// MoveClient reaches a move's target on its control port with the root token. +type MoveClient struct { + Token string +} + +func (c *MoveClient) Admit(ctx context.Context, addr, id string, a pool.MoveAdmission) (string, error) { + var out moveAdmitResponse + err := c.call(ctx, http.MethodPost, addr, movePath(id, ""), moveAdmitRequest{Tenant: a.Tenant, Key: a.Key, From: a.From}, &out) + return out.Node, err +} + +func (c *MoveClient) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error { + if err := peer.SendRound(ctx, addr, c.Token, movePath(id, "/rounds/"+strconv.Itoa(round)), dir, files); err != nil { + return fmt.Errorf("%w: %w", pool.ErrMoveTarget, err) + } + return nil +} + +func (c *MoveClient) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) (int64, error) { + var out moveCommitResponse + err := c.call(ctx, http.MethodPost, addr, movePath(id, "/commit"), moveCommitRequest{Claim: claim, Epoch: epoch}, &out) + return out.Epoch, err +} + +func (c *MoveClient) Drop(ctx context.Context, addr, id string) error { + return c.call(ctx, http.MethodDelete, addr, movePath(id, ""), nil, nil) +} + +// call maps the target's refusals onto the pool errors the source answers with, passing the target's reason through. +func (c *MoveClient) call(ctx context.Context, method, addr, path string, in, out any) error { + ctx, cancel := context.WithTimeout(ctx, moveCallTimeout) + defer cancel() + var body io.Reader + if in != nil { + raw, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(raw) + } + target := addr + path + if u, err := url.Parse(addr); err != nil || u.Scheme == "" { + target = "http://" + target + } + req, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.Token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("%w: %s: %w", pool.ErrMoveTarget, addr, err) + } + defer func() { _ = resp.Body.Close() }() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent: + if out == nil { + return nil + } + return json.Unmarshal(raw, out) + case http.StatusTooManyRequests: + return fmt.Errorf("%w: target %s: %s", pool.ErrQuota, addr, targetReason(raw)) + case http.StatusConflict: + return fmt.Errorf("%w: target %s: %s", pool.ErrNotMovable, addr, targetReason(raw)) + } + return fmt.Errorf("%w: target %s answered %d: %s", pool.ErrMoveTarget, addr, resp.StatusCode, targetReason(raw)) +} + +// MoveRequest is the wire body of POST /v1/sandboxes/{id}/move. +type MoveRequest struct { + To string `json:"to"` + MaxDowntimeMS int `json:"max_downtime_ms,omitzero"` +} + +// MoveResponse is the wire reply of POST /v1/sandboxes/{id}/move. +type MoveResponse struct { + OwnerAddr string `json:"owner_addr"` + Rounds int `json:"rounds"` + DowntimeMS int64 `json:"downtime_ms"` +} + +type moveAdmitRequest struct { + Tenant string `json:"tenant,omitempty"` + Key types.PoolKey `json:"key"` + From string `json:"from"` +} + +type moveAdmitResponse struct { + Node string `json:"node"` +} + +type moveCommitRequest struct { + Claim jsontext.Value `json:"claim"` + Epoch int64 `json:"epoch"` +} + +type moveCommitResponse struct { + Epoch int64 `json:"epoch"` +} + +func (s *Server) handleMove(w http.ResponseWriter, r *http.Request) { + req, ok := decodeBodyStrict[MoveRequest](w, r) + if !ok { + return + } + if req.MaxDowntimeMS < 0 { + writeErr(w, http.StatusBadRequest, "max_downtime_ms must not be negative") + return + } + addr, ok := s.movePeerAddr(req.To) + if !ok { + writeErr(w, http.StatusBadRequest, "to must name a current mesh peer as GET /v1/peers lists it") + return + } + id := r.PathValue("id") + budget := time.Duration(req.MaxDowntimeMS) * time.Millisecond + res, err := s.mgr.Move(r.Context(), id, addr, budget) + writeResult(w, r, "move", id, "move failed", err, func() { + writeJSON(w, http.StatusOK, MoveResponse{OwnerAddr: s.placer.ClientAddr(addr), Rounds: res.Rounds, DowntimeMS: res.Downtime.Milliseconds()}) + }) +} + +func (s *Server) handleAdmitMove(w http.ResponseWriter, r *http.Request) { + req, ok := decodeBodyStrict[moveAdmitRequest](w, r) + if !ok { + return + } + id := r.PathValue("id") + node, err := s.mgr.AdmitMove(id, pool.MoveAdmission{Tenant: req.Tenant, Key: req.Key.Defaulted(), From: req.From}) + writeResult(w, r, "admit move", id, "admit move failed", err, func() { + writeJSON(w, http.StatusOK, moveAdmitResponse{Node: node}) + }) +} + +func (s *Server) handleMoveRound(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + err := s.mgr.ReceiveMoveRound(id, r.Body) + writeResult(w, r, "move round", id, "move round failed", err, noContent(w)) +} + +func (s *Server) handleCommitMove(w http.ResponseWriter, r *http.Request) { + req, ok := decodeBodyStrict[moveCommitRequest](w, r) + if !ok { + return + } + id := r.PathValue("id") + epoch, err := s.mgr.CommitMove(r.Context(), id, req.Claim, req.Epoch) + writeResult(w, r, "commit move", id, "commit move failed", err, func() { + writeJSON(w, http.StatusOK, moveCommitResponse{Epoch: epoch}) + }) +} + +func (s *Server) handleDropMove(w http.ResponseWriter, r *http.Request) { + s.mgr.DropMove(r.PathValue("id")) + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) movePeerAddr(to string) (string, bool) { + if s.placer == nil || to == "" { + return "", false + } + for _, addr := range s.placer.PeerAddrs() { + if addr == to || s.placer.ClientAddr(addr) == to { + return addr, true + } + } + return "", false +} + +func movePath(id, suffix string) string { + return "/v1/moves/" + url.PathEscape(id) + suffix +} + +func targetReason(raw []byte) string { + var body struct { + Error string `json:"error"` + } + _ = json.Unmarshal(raw, &body) + return cmp.Or(body.Error, "no reason given") +} diff --git a/sandboxd/server/move_test.go b/sandboxd/server/move_test.go new file mode 100644 index 00000000..ace7c19e --- /dev/null +++ b/sandboxd/server/move_test.go @@ -0,0 +1,88 @@ +package server + +import ( + "encoding/json/v2" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/pool" +) + +func TestMoveRouteResolvesThePeerAndMapsTheOutcome(t *testing.T) { + var gotAddr string + var gotBudget time.Duration + outcome := error(nil) + mgr := &fakeManager{move: func(_, addr string, budget time.Duration) (pool.MoveResult, error) { + gotAddr, gotBudget = addr, budget + return pool.MoveResult{Rounds: 3, Downtime: 180 * time.Millisecond}, outcome + }} + srv := newMeshTestServer(t, "root", mgr, &fakePlacer{addrs: []string{"10.0.0.2:7777"}}, nil) + + resp := doReq(t, http.MethodPost, srv.URL+"/v1/sandboxes/sb_1/move", "root", `{"to":"10.0.0.2:7777","max_downtime_ms":1500}`) + var body MoveResponse + if err := json.UnmarshalRead(resp.Body, &body); err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("move: %d %v", resp.StatusCode, err) + } + if gotAddr != "10.0.0.2:7777" || gotBudget != 1500*time.Millisecond || body != (MoveResponse{OwnerAddr: "10.0.0.2:7777", Rounds: 3, DowntimeMS: 180}) { + t.Errorf("move to %s within %s answered %+v", gotAddr, gotBudget, body) + } + for _, tt := range []struct { + token, body string + err error + want int + }{ + {"root", `{"to":"10.0.0.9:7777"}`, nil, http.StatusBadRequest}, + {"root", `{"to":"10.0.0.2:7777","max_downtime_ms":-1}`, nil, http.StatusBadRequest}, + {"root", `{"to":"10.0.0.2:7777"}`, pool.ErrUnknownSandbox, http.StatusNotFound}, + {"root", `{"to":"10.0.0.2:7777"}`, fmt.Errorf("%w: it has volumes", pool.ErrNotMovable), http.StatusConflict}, + {"root", `{"to":"10.0.0.2:7777"}`, fmt.Errorf("%w: target: 3 moves in flight", pool.ErrQuota), http.StatusTooManyRequests}, + {"root", `{"to":"10.0.0.2:7777"}`, pool.ErrMoveTarget, http.StatusBadGateway}, + {"root", `{"to":"10.0.0.2:7777"}`, fmt.Errorf("%w: %w", pool.ErrMoveStoreDown, pool.ErrMoveTarget), http.StatusServiceUnavailable}, + {"root", `{"to":"10.0.0.2:7777"}`, pool.ErrMoveDowntime, http.StatusServiceUnavailable}, + {"root", `{"to":"10.0.0.2:7777"}`, pool.ErrLeaseLost, http.StatusServiceUnavailable}, + } { + outcome = tt.err + resp := doReq(t, http.MethodPost, srv.URL+"/v1/sandboxes/sb_1/move", tt.token, tt.body) + if resp.StatusCode != tt.want { + t.Errorf("move %s with %v: %d, want %d", tt.body, tt.err, resp.StatusCode, tt.want) + } + } +} + +func TestMoveClientPassesTheTargetsRefusalThrough(t *testing.T) { + status, reason := http.StatusOK, "" + var gotAuth, gotPath string + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth, gotPath = r.Header.Get("Authorization"), r.URL.Path + if status != http.StatusOK { + writeErr(w, status, reason) + return + } + writeJSON(w, http.StatusOK, moveAdmitResponse{Node: "b"}) + })) + defer target.Close() + c := &MoveClient{Token: "root"} + node, err := c.Admit(t.Context(), strings.TrimPrefix(target.URL, "http://"), "sb_1", pool.MoveAdmission{From: "a"}) + if err != nil || node != "b" || gotAuth != "Bearer root" || gotPath != "/v1/moves/sb_1" { + t.Fatalf("admit: %q %v auth %q path %q", node, err, gotAuth, gotPath) + } + for _, tt := range []struct { + status int + want error + }{ + {http.StatusTooManyRequests, pool.ErrQuota}, + {http.StatusConflict, pool.ErrNotMovable}, + {http.StatusInternalServerError, pool.ErrMoveTarget}, + } { + status, reason = tt.status, "node claim quota reached: 4 live claims, cap 4" + _, err := c.Admit(t.Context(), target.URL, "sb_1", pool.MoveAdmission{From: "a"}) + if !errors.Is(err, tt.want) || !strings.Contains(err.Error(), reason) { + t.Errorf("admit answered %d: %v, want %v with the target's reason", tt.status, err, tt.want) + } + } +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 5b2baa23..1c71ded6 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -60,6 +60,10 @@ var poolErrHTTP = []struct { {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrLeaseLost, http.StatusServiceUnavailable, "lease lost; retry"}, {pool.ErrEvacuating, http.StatusServiceUnavailable, "sandbox moving to another node; retry"}, + {pool.ErrMoveStoreDown, http.StatusServiceUnavailable, ""}, + {pool.ErrMoveDowntime, http.StatusServiceUnavailable, ""}, + {pool.ErrMoveTarget, http.StatusBadGateway, ""}, + {pool.ErrNotMovable, http.StatusConflict, ""}, {pool.ErrNoEgress, http.StatusConflict, ""}, {pool.ErrNoEgressHibernate, http.StatusConflict, ""}, {pool.ErrNoEgressFork, http.StatusConflict, ""}, @@ -142,6 +146,11 @@ type Manager interface { Uncordon(ctx context.Context) Info() ([]pool.PoolInfo, pool.Gauges) Templates() []pool.TemplateInfo + Move(ctx context.Context, id, addr string, maxDowntime time.Duration) (pool.MoveResult, error) + AdmitMove(id string, a pool.MoveAdmission) (string, error) + ReceiveMoveRound(id string, body io.Reader) error + CommitMove(ctx context.Context, id string, claim []byte, epoch int64) (int64, error) + DropMove(id string) } // Dialer opens the hybrid-vsock connection to a VM's silkd. @@ -332,6 +341,11 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("PUT /v1/tenants", s.requireRoot(s.handlePutTenants)) mux.HandleFunc("PUT /v1/tenants/{name}", s.requireRoot(s.handlePutTenant)) mux.HandleFunc("DELETE /v1/tenants/{name}", s.requireRoot(s.handleDeleteTenant)) + mux.HandleFunc("POST /v1/sandboxes/{id}/move", s.requireRoot(s.handleMove)) + mux.HandleFunc("POST /v1/moves/{id}", s.requireRoot(s.handleAdmitMove)) + mux.HandleFunc("PUT /v1/moves/{id}/rounds/{n}", s.requireRoot(s.handleMoveRound)) + mux.HandleFunc("POST /v1/moves/{id}/commit", s.requireRoot(s.handleCommitMove)) + mux.HandleFunc("DELETE /v1/moves/{id}", s.requireRoot(s.handleDropMove)) mux.HandleFunc("POST /v1/drain", s.requireRoot(s.handleDrain)) mux.HandleFunc("DELETE /v1/drain", s.requireRoot(s.handleUncordon)) mux.HandleFunc("GET /v1/usage", s.requireRoot(s.handleUsage)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index ad0693f5..db553fc5 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2246,6 +2246,7 @@ type fakeManager struct { envErr error env types.Env claimDeadline func(id, token string) (time.Time, error) + move func(id, addr string, maxDowntime time.Duration) (pool.MoveResult, error) gotTenant string gotDigest string @@ -2598,6 +2599,20 @@ func (f *fakeManager) Info() ([]pool.PoolInfo, pool.Gauges) { func (f *fakeManager) Templates() []pool.TemplateInfo { return f.infoTemplates } +func (f *fakeManager) Move(_ context.Context, id, addr string, maxDowntime time.Duration) (pool.MoveResult, error) { + return f.move(id, addr, maxDowntime) +} + +func (f *fakeManager) AdmitMove(string, pool.MoveAdmission) (string, error) { return "", nil } + +func (f *fakeManager) ReceiveMoveRound(string, io.Reader) error { return nil } + +func (f *fakeManager) CommitMove(context.Context, string, []byte, int64) (int64, error) { + return 0, nil +} + +func (f *fakeManager) DropMove(string) {} + func (f *fakeManager) SetTemplateLabels(_ context.Context, key types.PoolKey, labels types.Metadata, tenant, digest string) error { f.labeled = append(f.labeled, fmt.Sprintf("%s %s %v %q", key.Template, key.Size, labels, tenant)) f.gotDigest = digest From 3471d733f4e127c555ef35f93f202123b33fcb0e Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:37:05 +0800 Subject: [PATCH 10/53] docs: the live move endpoint, its record, counting and move_concurrency --- docs/cluster.md | 75 ++++++++++++++++++++++++++++++++++++++++++++ docs/deploy.md | 6 ++-- docs/sandboxd-api.md | 45 ++++++++++++++++++++++---- 3 files changed, 118 insertions(+), 8 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 00e9305d..dcd23314 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -308,6 +308,81 @@ the move stops working until it is minted again, except behind a shared `preview_advertise`, where it answers when it reaches the new owner. `DELETE /v1/drain` stops the evacuation: claims not yet started stay as they are, and one mid-archive finishes its archive but keeps its row. +### Moving a running claim + +`POST /v1/sandboxes/{id}/move` (root only) hands one running claim to a named +peer with cocoon's live migration, so its guest keeps its processes and +memory. Evacuation stays the cold path: + +| | Cold: `POST /v1/drain?evacuate=1` | Hot: `POST /v1/sandboxes/{id}/move` | +|---|---|---| +| What moves | every claim of a draining node | one running claim | +| How | hibernate, archive to the checkpoint store, release the row, a peer adopts | memory and disk rounds straight to the target, which restores and starts the guest | +| Guest after the move | stopped until the next call wakes it | running, with its processes and memory | +| Target | any peer that answers | the one the caller names | +| When it cannot move | the claim is skipped and keeps serving | the call fails and the claim keeps running here | + +Only a none-lane claim without volumes moves; a hibernated, archived, failed or +waking claim, an egress-lane claim and a claim with volumes answer 409. Open +exec and pty streams, port forwards and egress connections end; the client +meets a 404 on the source and runs `Lookup`. + +**The record.** Each move has a record of its own in the meta store (table +`sandboxd_moves`), separate from the archive rows: `id`, `from`, `to`, `state` +and `epoch`, each change a compare-and-set at a new epoch. + +| From | To | Who | When | +|---|---|---|---| +| (none) | `pending` | source | after the target admitted the move, before the baseline round | +| `pending` | `final` | source | the final round paused the guest | +| `final` | `committed` | target | after the receive; the point of no return | +| `pending` or `final` | `aborted` | source | before it resumes the guest | + +The target commits only from `final`, so it never starts a guest restored from +a round the source kept running after. The source resumes its guest only after +its `aborted` write lands; a write that finds `committed` means the claim moved, +and the source drops it. While the meta store is down, a guest paused in the +final round stays paused and the call answers 503; the abort is retried every 2 +seconds. The source deletes the record when the move ends. Each node's hourly +sweep aborts and deletes its records older than an hour that no move in flight +holds. + +**Rounds.** The source sends a baseline round, then up to 4 deltas, each over +the control port with the root token. Once the last delta's size at the rate +the rounds reached, plus 400 ms for the receive and the start, fits +`max_downtime_ms` (default 2000), the source pauses the guest for the final +round and the target receives the series and starts the guest. A move whose +deltas never fit answers 503 and the guest runs on. `downtime_ms` in the answer +is the pause the source measured, from the final capture to the target's +commit. + +**Counting.** The target admits the move like a new claim: node `max_claims`, +`max_running`, the tenant's `max_claims`, `memory_limit` and +`move_concurrency`, and holds a reservation from admission to its commit. The +reservation counts in every cap but is no claim: listings, reaps and gossip +never see it. At the commit the reservation becomes the claim under one lock, +so the target never counts it twice. The source counts the claim until it drops +it after the commit, so the cluster counts a moving claim twice, which can make +a node refuse earlier but never over-admit. `memory_limit` is checked again +before the receive. A reservation with no round or commit for 2 minutes is +dropped with its staged files. + +**Replicated claims.** Before the final round the source deletes the claim's +registry row and its replica, so no third node can adopt a stale replica while +the target runs the claim. The target's first replica round puts a new row one +interval after the start, and an aborted move lets the source's next round put +it again. + +**Failures.** Any failure before the commit aborts: the source resumes the +guest, the target drops its reservation, files and any received VM, and the +call answers the error. A target answer lost after its commit is resolved +through the record, and the call answers 200. A source restarted with a move +in its journal reads the record: `committed` drops the claim, anything else is +aborted and the guest resumes. A target restarted with a received claim not yet +started starts it when the record committed to it and removes it otherwise. A +guest the target cannot start fails there through the +[failed path](#running-claims-across-node-loss); it never goes back. + ### Node lease With `archive_failover` every node renews one row of `sandboxd_nodes (cell, diff --git a/docs/deploy.md b/docs/deploy.md index 2a250501..21afca81 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -160,6 +160,7 @@ the intersection of the two. | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim first tries a redirect to a warm peer, else to a peer that serves the pool; a volume claim answers the 429 with no redirect) | | `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was; a claim takes the same redirect as `max_claims`. Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A wake cannot move to another node, so its 429 stays until a running claim sleeps or goes. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | +| `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `archive_failover` and `mesh.cluster_key`: a node that sets this without them refuses to load, and one that leaves it unset cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or a master build), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM keeps the VMM it started with), a run dir that supports reflink such as XFS, and nodes of the same CPU model. Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token; a config reload records as op `config_reload` with `changed`, and a runtime volume change as op `volumes` with `added` or `removed`. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no warm VMs on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | | `memory_limit` | unset | `{"anon": "300G", "psi_some_avg10": 40}`, either key alone: while `AnonPages` in `/proc/meminfo` is above `anon`, or the `some avg10` of `/proc/pressure/memory` is above `psi_some_avg10`, checked every 2 s, the node refuses new claims, forks and checkpoint claims with 429 and parks refill; `/v1/info` reports it as at capacity and the mesh sees no pools on it, so a warm or pool redirect skips it; a template or volume held only there still redirects to it and answers 429. Wakes of its own claims, hibernates, checkpoints and archives go on, and running sandboxes keep running. Guest memory is a private file mapping, so page cache and clean guest pages are reclaimable and only anon is what a hibernate has to free. Both signals are node-wide; `AnonPages` leaves out swapped pages and explicit hugepages. A signal the node cannot read, such as PSI on a kernel without it, logs an error and is ignored. PSI also counts a task stalled by its own cgroup limit and weighs by busy CPUs, so on a mostly idle node a few throttled tasks in one small cgroup can cross it: 16 tasks under a 64M `MemoryHigh` reached 75 on an idle 384-CPU host with memory to spare. Treat `anon` as the main limit, and set `psi_some_avg10` only where no workload runs under a tight memory cgroup. Size `anon` so the running guests' further growth still fits under physical memory | @@ -984,7 +985,7 @@ all at once or not at all. Every field falls into one of three classes: - **Reloaded**: - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `reader`, `egress_upstream_env`, for existing keys and for keys new in the file; - `egress_classes`: each class's `egress` and `egress_upstream_env`; - - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_conn_rate`, `disk_reserve`, `memory_limit`, `max_claims`, `max_running`, `max_fork_count`, `release_delay_seconds`, `vmm_restart`, `sync_claims`, `checkpoint_ttl_hours`, and the unpooled defaults `idle_hibernate_seconds`, `archive_after_seconds` and `archive_delete_after_seconds`. + - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_conn_rate`, `disk_reserve`, `memory_limit`, `max_claims`, `max_running`, `move_concurrency`, `max_fork_count`, `release_delay_seconds`, `vmm_restart`, `sync_claims`, `checkpoint_ttl_hours`, and the unpooled defaults `idle_hibernate_seconds`, `archive_after_seconds` and `archive_delete_after_seconds`. - **Left to their API**: a pool's targets (`warm`, `warm_max`, idle and archive durations) belong to `PUT /v1/pools`. A reload never changes them and lists them as `ignored`. A pool key new in the file gets its settings at once and stays @@ -1007,7 +1008,8 @@ Effects on what already runs: `egress_usage_bytes` apply to claims armed after the reload. - **Claim caps**: a lower `max_claims` or `max_running` refuses new claims, and a lower `max_running` refuses wakes, until the count falls under it. It stops - nothing that runs. + nothing that runs. A lower `move_concurrency` refuses new moves the same way + and stops no move in flight. - **Later events only**: `max_fork_count` applies to the next fork, `release_delay_seconds` to the next release, `vmm_restart` to the next VMM exit and `sync_claims` to the next commit. A new `archive_delete_after_seconds` diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 5af58479..b6909da5 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -34,7 +34,8 @@ they reach its guest, so they share these answers: - 503 `lease lost; retry` for a replicated claim while its node's lease is overdue ([running claims](cluster.md#running-claims-across-node-loss)) - 503 `sandbox moving to another node; retry` while an - [evacuation](#post-v1drain) has yet to move it + [evacuation](#post-v1drain) has yet to move it, or while a + [live move](#post-v1sandboxesidmove) pauses it for its final round - 503 `archive registry down; retry` for an archived claim under `archive_failover` while the meta store is down - 503 `tenant store down; retry` for an archived claim whose tenant the node has @@ -845,6 +846,36 @@ Auth: root only (tenant tokens get 403). Lifts the drain, stops an evacuation, a fresh info payload. A claim an evacuation is archiving finishes its archive and stays here archived; the next call wakes it. +## POST /v1/sandboxes/{id}/move + +Auth: root only (tenant tokens get 403). Moves one running claim to a peer with +its guest's processes and memory (see +[moving a running claim](cluster.md#moving-a-running-claim)). Body: +`{"to": "", "max_downtime_ms": 2000}`, `to` as `GET /v1/peers` lists it +and never this node (else 400), `max_downtime_ms` optional with a default of +2000. Answers `{"owner_addr": "", "rounds": n, "downtime_ms": d}` once the +peer runs the guest and this node dropped the claim; `downtime_ms` is the pause +from the final capture to the peer's commit. While the move runs, release of +the claim answers 503 `sandbox moving to another node; retry`, and fork, +checkpoint, promote and hibernate answer the same during the rounds and wait +out the final round. Env writes, renew and data-plane calls keep working until +the final round. Every error leaves the claim running here: + +- 404 unknown sandbox +- 409 the claim cannot move (hibernated, archived, failed, waking, with + volumes, egress lane, a move already running), cocoon or its run dir cannot + migrate, the target refused (`target : `), or the node lacks + `archive_failover` or `mesh.cluster_key` +- 429 `move_concurrency` reached here or on the target, or the target's + admission refused (`target : `) +- 502 the target is unreachable or failed +- 503 the move or archive registry is down, the node lease is overdue, or the + final round would not fit `max_downtime_ms` after 4 deltas + +The target serves root-only peer routes for a move, `POST /v1/moves/{id}`, +`PUT /v1/moves/{id}/rounds/{n}`, `POST /v1/moves/{id}/commit` and +`DELETE /v1/moves/{id}`; they are no public API. + ## POST /v1/sandboxes/{id}/preview Auth: node `api_token` in the header, the sandbox's own token in the body. @@ -1088,7 +1119,7 @@ truth is the usage journal below. Always on: every lifecycle transition appends one JSONL event to `/usage.jsonl` — `{"t": , "ev": -"claim|renew|hibernate|wake|fork|checkpoint|adopt|promote|release|reap|archive|unarchive|archive_delete|egress|egress_bytes|vmm_exit|vmm_restart|vmm_failed", +"claim|renew|hibernate|wake|fork|checkpoint|adopt|promote|release|reap|archive|unarchive|archive_delete|egress|egress_bytes|vmm_exit|vmm_restart|vmm_failed|move_in|move_out", "id": "sb_…"}` plus `vm` (the VM's name; absent on `renew`, `adopt`, `archive`, `unarchive`, `archive_delete`, `egress`, `egress_bytes`, and the `reap` or `release` of an archived claim), `key` (the pool key's stable hash, claim @@ -1096,7 +1127,7 @@ events), `template`, `net` and `size` (the claim's template, lane and size tier, claim events, so a consumer can price a sandbox by its shape without the claim record, which is gone after release; `size` sets the CPU and memory, and `template` is a label, since promote lets the caller name it), `deadline` (the deadline in force after this event: `claim`, -`renew`, `wake` and `unarchive` carry the lease, `archive` and `adopt` the retention +`renew`, `wake`, `unarchive` and `move_in` carry the lease, `archive` and `adopt` the retention end, or nothing for an archive kept forever, and `reap` the deadline it ended under), `tenant` (the owning tenant, on every event of a tenant's claim), `children` (fork) and `ref` (the promoted template / checkpoint id, the egress host, a `vmm_failed` reason, or why reconcile ended a claim). An egress event that injected credentials names @@ -1108,11 +1139,13 @@ the subset of those names claimed `rw`, so billing can discriminate write access (mounts and host paths are not billing dimensions). The file rotates at 64 MiB keeping one `.1` backup, so a tailing collector never loses a window silently. A last line a crash cut short is ended with a newline when the node -reopens the journal, so it never swallows the next event. Folding rules: fold a sandbox's events of all nodes in `t` order. +reopens the journal, so it never swallows the next event. A [live move](#post-v1sandboxesidmove) +records `move_in` on the target when it starts the guest and `move_out` on the +source when it drops the claim, each with its own node's `vm`. Folding rules: fold a sandbox's events of all nodes in `t` order. An event ends the current span at its `t` and enters a state: `claim`, `wake`, -`unarchive` and `vmm_restart` enter `running`, `hibernate` enters `hibernated`, +`unarchive`, `vmm_restart` and `move_in` enter `running`, `hibernate` enters `hibernated`, `archive` enters `archived`, and `vmm_exit`, `archive_delete`, `release` and -`reap` enter `ended`. Other events, `vmm_failed` included, change no state, and +`reap` enter `ended`. Other events, `vmm_failed` and `move_out` included, change no state, and `reap`, `adopt` and `archive_delete` have the exceptions below. Billable compute, hibernated storage and archived storage seconds are the time in `running`, `hibernated` and `archived`, a cheaper store tier than a hibernated From 063f8c1be27008106ff2693c84a2415c86afe0ae Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:39:48 +0800 Subject: [PATCH 11/53] e2e: the fake engine refuses the migrate calls --- e2e/fakeengine_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 96a22e07..de87f1e0 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -134,6 +134,20 @@ func (f *fakeEngine) WriteGuestEnv(_ context.Context, vsockSocket string, doc [] return nil } +func (f *fakeEngine) CheckMigrateSupport(context.Context) error { + return errors.New("the e2e engine does not migrate") +} + +func (f *fakeEngine) MigrateCapture(context.Context, string, string, bool) (types.MigrateRound, error) { + return types.MigrateRound{}, errors.New("the e2e engine does not migrate") +} + +func (f *fakeEngine) MigrateReceive(context.Context, string, string, types.PoolKey) (types.VMRecord, error) { + return types.VMRecord{}, errors.New("the e2e engine does not migrate") +} + +func (f *fakeEngine) MigrateAbort(context.Context, string, string) error { return nil } + func (f *fakeEngine) Hibernate(ctx context.Context, name, _ string) error { return f.Remove(ctx, name) } From 9dd4a869217672e7f7fe1778eaa8a9d03095c03f Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:44:09 +0800 Subject: [PATCH 12/53] review: lint findings and type order in the move code --- sandboxd/pool/move.go | 22 +++++++++++----------- sandboxd/pool/move_out.go | 5 +++-- sandboxd/pool/move_test.go | 8 ++++---- sandboxd/server/move_test.go | 2 ++ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 07217bdc..50dec5fa 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -26,17 +26,6 @@ const ( moveStaleAge = time.Hour ) -// outMove is a source's move in flight; final is guarded by m.mu, the rest belongs to the goroutine driving the move. -type outMove struct { - addr string - to string - epoch int64 - series string - rounds int - final bool - locked bool -} - // MoveAdmission is what a source asks a target to reserve for a live move. type MoveAdmission struct { Tenant string @@ -60,6 +49,17 @@ type MovePeer interface { Drop(ctx context.Context, addr, id string) error } +// outMove is a source's move in flight; final is guarded by m.mu, the rest belongs to the goroutine driving the move. +type outMove struct { + addr string + to string + epoch int64 + series string + rounds int + final bool + locked bool +} + // WithMovePeer wires the transport a move uses to reach its target; set it before serving. func (m *Manager) WithMovePeer(p MovePeer) { m.movePeer = p diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 8b1f481f..acfb7687 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -252,7 +252,7 @@ func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM // failMoveOut resolves a failed move through its record: committed finishes it, anything else aborts and resumes the guest. func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, cause error) (MoveResult, error) { if mv.epoch == 0 { - m.abortMoveOut(ctx, sb, mv) + _ = m.abortMoveOut(ctx, sb, mv) return MoveResult{}, cause } committed, err := m.settleMoveOut(ctx, sb, mv) @@ -291,7 +291,8 @@ func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM m.finishMoveOut(ctx, sb, mv, rec.Epoch) return true, nil case ours && (rec.State == moves.Pending || rec.State == moves.Final): - aborted, err := m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted) + var aborted int64 + aborted, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted) if errors.Is(err, moves.ErrLost) { continue } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 59ca323c..654d9e0d 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -241,18 +241,18 @@ func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { mc.s.mu.Lock() replica := sb.ReplicaCk mc.s.mu.Unlock() - if _, ok, _ := mc.failoverCluster.reg.Get(t.Context(), sb.ID); !ok || replica == "" { + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || replica == "" { t.Fatal("setup: the claim has no replica row") } mc.p.commitHook = func(int) error { - if _, ok, _ := mc.failoverCluster.reg.Get(t.Context(), sb.ID); ok { + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); ok { t.Error("the replica row is still adoptable at the commit") } - mc.failoverCluster.reg.Expire("a") + mc.reg.Expire("a") if adopted, err := third.AdoptArchived(t.Context(), sb.ID, sb.Token); adopted || err != nil { t.Errorf("a third node adopts the moving claim: %v %v", adopted, err) } - _ = mc.failoverCluster.reg.Renew(t.Context(), "a", time.Hour) + _ = mc.reg.Renew(t.Context(), "a", time.Hour) return nil } if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { diff --git a/sandboxd/server/move_test.go b/sandboxd/server/move_test.go index ace7c19e..8ccd480d 100644 --- a/sandboxd/server/move_test.go +++ b/sandboxd/server/move_test.go @@ -24,6 +24,7 @@ func TestMoveRouteResolvesThePeerAndMapsTheOutcome(t *testing.T) { srv := newMeshTestServer(t, "root", mgr, &fakePlacer{addrs: []string{"10.0.0.2:7777"}}, nil) resp := doReq(t, http.MethodPost, srv.URL+"/v1/sandboxes/sb_1/move", "root", `{"to":"10.0.0.2:7777","max_downtime_ms":1500}`) + defer func() { _ = resp.Body.Close() }() var body MoveResponse if err := json.UnmarshalRead(resp.Body, &body); err != nil || resp.StatusCode != http.StatusOK { t.Fatalf("move: %d %v", resp.StatusCode, err) @@ -48,6 +49,7 @@ func TestMoveRouteResolvesThePeerAndMapsTheOutcome(t *testing.T) { } { outcome = tt.err resp := doReq(t, http.MethodPost, srv.URL+"/v1/sandboxes/sb_1/move", tt.token, tt.body) + _ = resp.Body.Close() if resp.StatusCode != tt.want { t.Errorf("move %s with %v: %d, want %d", tt.body, tt.err, resp.StatusCode, tt.want) } From 94a63656b309f1a8db245805dc3c987e8298b244 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:47:34 +0800 Subject: [PATCH 13/53] pool: only a move's source deletes its record, and the final phase has two minutes A target's hourly sweep deleted a committed record older than an hour. A source that came back after that read the missing record as an aborted move and resumed its guest while the target ran the claim. Now a target only aborts stale pending or final records. The final capture, its transfer and the commit ran with no deadline, so a hung target kept the source's guest paused until TCP gave up. They now share a two minute budget, after which the source resolves the move through its record. --- docs/cluster.md | 9 ++++++--- sandboxd/pool/move.go | 10 +++++++--- sandboxd/pool/move_out.go | 2 ++ sandboxd/pool/move_test.go | 29 +++++++++++++++++++++++++++++ sandboxd/pool/pool.go | 2 +- 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index dcd23314..96986c71 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -344,8 +344,9 @@ its `aborted` write lands; a write that finds `committed` means the claim moved, and the source drops it. While the meta store is down, a guest paused in the final round stays paused and the call answers 503; the abort is retried every 2 seconds. The source deletes the record when the move ends. Each node's hourly -sweep aborts and deletes its records older than an hour that no move in flight -holds. +sweep aborts its records older than an hour that no move in flight holds, and +deletes only those it sourced: a source reads a missing record as an aborted +move, so a target never deletes one. **Rounds.** The source sends a baseline round, then up to 4 deltas, each over the control port with the root token. Once the last delta's size at the rate @@ -354,7 +355,9 @@ the rounds reached, plus 400 ms for the receive and the start, fits round and the target receives the series and starts the guest. A move whose deltas never fit answers 503 and the guest runs on. `downtime_ms` in the answer is the pause the source measured, from the final capture to the target's -commit. +commit. The final capture, its transfer and the target's commit have 2 minutes +together; past that the source resolves the move through the record, and the +guest resumes on the source unless the target committed. **Counting.** The target admits the move like a new claim: node `max_claims`, `max_running`, the tenant's `max_claims`, `memory_limit` and diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 50dec5fa..ef728c7d 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -24,6 +24,7 @@ const ( // Receive and vm start on the target add this to the final round's transfer. moveStartAllowance = 400 * time.Millisecond moveStaleAge = time.Hour + moveFinalTimeout = 2 * time.Minute ) // MoveAdmission is what a source asks a target to reserve for a live move. @@ -78,10 +79,10 @@ func (m *Manager) moveDir(id string) string { return filepath.Join(m.dataDir, movesDir, id) } -// sweepStaleMoves ends this node's move records no move in flight holds. -func (m *Manager) sweepStaleMoves(ctx context.Context) { +// sweepStaleMoves aborts this node's stale moves and deletes only those it sourced, since a source reads a missing record as aborted. +func (m *Manager) sweepStaleMoves(ctx context.Context, now time.Time) { logger := log.WithFunc("pool.sweepStaleMoves") - recs, err := m.moves.Stale(ctx, time.Now().Add(-moveStaleAge)) + recs, err := m.moves.Stale(ctx, now.Add(-moveStaleAge)) if err != nil { logger.Warnf(ctx, "list stale moves: %v", err) return @@ -97,6 +98,9 @@ func (m *Manager) sweepStaleMoves(ctx context.Context) { continue } } + if rec.From != m.nodeID { + continue + } if err = m.moves.Delete(ctx, rec.ID, epoch); err != nil { logger.Warnf(ctx, "delete stale move %s: %v", rec.ID, err) } diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index acfb7687..061636b3 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -107,6 +107,8 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b m.mu.Unlock() sb.Transition.Lock() mv.locked = true + ctx, cancel := context.WithTimeout(ctx, moveFinalTimeout) + defer cancel() if err = m.unrowForMove(ctx, sb); err != nil { return movedAt{}, err } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 654d9e0d..0e062a95 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -415,6 +415,35 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { } } +func TestStaleMoveRecordsAreDeletedOnlyByTheirSource(t *testing.T) { + mc := newMoveCluster(t) + ctx, reg := t.Context(), mc.moveReg + done, _ := reg.Begin(ctx, "sb_done", "a", "b") + done, _ = reg.Advance(ctx, "sb_done", done, moves.Pending, moves.Final) + if _, err := reg.Advance(ctx, "sb_done", done, moves.Final, moves.Committed); err != nil { + t.Fatalf("commit: %v", err) + } + if _, err := reg.Begin(ctx, "sb_open", "a", "b"); err != nil { + t.Fatalf("begin: %v", err) + } + later := time.Now().Add(2 * moveStaleAge) + + mc.t.sweepStaleMoves(ctx, later) + if rec, ok, _ := reg.Get(ctx, "sb_done"); !ok || rec.State != moves.Committed { + t.Errorf("committed record after the target's sweep: %+v %v, want it kept for its source", rec, ok) + } + if rec, ok, _ := reg.Get(ctx, "sb_open"); !ok || rec.State != moves.Aborted { + t.Errorf("open record after the target's sweep: %+v %v, want it aborted and kept", rec, ok) + } + + mc.s.sweepStaleMoves(ctx, later) + for _, id := range []string{"sb_done", "sb_open"} { + if rec, ok, _ := reg.Get(ctx, id); ok { + t.Errorf("record %s after the source's sweep: %+v, want it deleted", id, rec) + } + } +} + func BenchmarkQuotaErr(b *testing.B) { for _, incoming := range []int{0, 2} { b.Run(map[int]string{0: "no reservations", 2: "two reservations"}[incoming], func(b *testing.B) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 38bbcb1a..2448434f 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -520,7 +520,7 @@ func (m *Manager) Run(ctx context.Context) { go m.sweepOrphanArchives(ctx) } if m.moves != nil { - go m.sweepStaleMoves(ctx) + go m.sweepStaleMoves(ctx, time.Now()) } } } From 603c49010fa4b31e279db585daae66ba1ab3b2ac Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 19:48:35 +0800 Subject: [PATCH 14/53] pool: a commit without a reservation answers 409, since the move ended and no cap refused it --- sandboxd/pool/move_in.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index e1abdfdc..49e8d130 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -67,7 +67,7 @@ func (m *Manager) ReceiveMoveRound(id string, body io.Reader) error { func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch int64) (int64, error) { ctx = context.WithoutCancel(ctx) if !m.touchReservation(id) { - return 0, fmt.Errorf("%w: no reservation for the move of %s", ErrQuota, id) + return 0, fmt.Errorf("%w: no reservation for the move of %s", ErrNotMovable, id) } if m.memoryLow.Load() { return 0, fmt.Errorf("%w: %s", ErrQuota, memoryLowReason) From a6366c78104cc7a752e345c2708c1eac87b4e722 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:01:05 +0800 Subject: [PATCH 15/53] moves: a started state, and Begin no longer overwrites a committed record A committed record is the proof a target needs to start its guest and a source needs to drop its claim. Overwriting it let a later move erase that proof while either side still depended on it. --- sandboxd/moves/moves.go | 14 ++++++++------ sandboxd/moves/movestest/movestest.go | 11 ++++++----- sandboxd/moves/pg/pg.go | 2 +- sandboxd/moves/pg/schema.sql | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/sandboxd/moves/moves.go b/sandboxd/moves/moves.go index 3ba42d77..ca2fb229 100644 --- a/sandboxd/moves/moves.go +++ b/sandboxd/moves/moves.go @@ -12,6 +12,7 @@ const ( Pending State = "pending" Final State = "final" Committed State = "committed" + Started State = "started" Aborted State = "aborted" ) @@ -22,14 +23,15 @@ var ( ErrLost = errors.New("move record moved on") advances = map[[2]State]bool{ - {Pending, Final}: true, - {Final, Committed}: true, - {Pending, Aborted}: true, - {Final, Aborted}: true, + {Pending, Final}: true, + {Final, Committed}: true, + {Committed, Started}: true, + {Pending, Aborted}: true, + {Final, Aborted}: true, } ) -// State is where a move stands; Committed and Aborted end it. +// State is where a move stands; Started and Aborted end it. type State string // Record is one claim's move. @@ -44,7 +46,7 @@ type Record struct { // Registry keeps the records; every write is a compare-and-set, and every epoch it hands out is new. type Registry interface { - // Begin writes a Pending record for id at a new epoch and returns it; ErrLost when a Pending or Final record holds id. + // Begin writes a Pending record for id at a new epoch and returns it; ErrLost when a Pending, Final or Committed record holds id. Begin(ctx context.Context, id, from, to string) (int64, error) // Advance moves the record at epoch expect in state from to state to at a new epoch and returns it. Advance(ctx context.Context, id string, expect int64, from, to State) (int64, error) diff --git a/sandboxd/moves/movestest/movestest.go b/sandboxd/moves/movestest/movestest.go index 5a69e10c..03c1311c 100644 --- a/sandboxd/moves/movestest/movestest.go +++ b/sandboxd/moves/movestest/movestest.go @@ -14,7 +14,7 @@ import ( const racers = 8 -var states = []moves.State{moves.Pending, moves.Final, moves.Committed, moves.Aborted} +var states = []moves.State{moves.Pending, moves.Final, moves.Committed, moves.Started, moves.Aborted} // Opener returns an empty registry; each call is a fresh table. type Opener func(t *testing.T) moves.Registry @@ -40,7 +40,7 @@ func (m *Mem) Begin(_ context.Context, id, from, to string) (int64, error) { if m.down { return 0, moves.ErrUnavailable } - if old, ok := m.recs[id]; ok && old.State != moves.Committed && old.State != moves.Aborted { + if old, ok := m.recs[id]; ok && old.State != moves.Started && old.State != moves.Aborted { return 0, moves.ErrLost } return m.write(moves.Record{ID: id, From: from, To: to, State: moves.Pending}), nil @@ -147,7 +147,7 @@ func testBeginGet(t *testing.T, r moves.Registry) { func testAdvance(t *testing.T, r moves.Registry) { ctx := t.Context() for i, path := range [][]moves.State{ - {moves.Final, moves.Committed}, + {moves.Final, moves.Committed, moves.Started}, {moves.Aborted}, {moves.Final, moves.Aborted}, } { @@ -255,7 +255,7 @@ func testAdvanceRace(t *testing.T, r moves.Registry) { func testBeginOverEnded(t *testing.T, r moves.Registry) { ctx := t.Context() - for _, end := range []moves.State{moves.Committed, moves.Aborted} { + for _, end := range []moves.State{moves.Started, moves.Aborted} { id := "sb_" + string(end) ended := mustReach(t, r, id, end) again, err := r.Begin(ctx, id, "t", "u") @@ -270,7 +270,7 @@ func testBeginOverEnded(t *testing.T, r moves.Registry) { func testBeginOverLive(t *testing.T, r moves.Registry) { ctx := t.Context() - for _, live := range []moves.State{moves.Pending, moves.Final} { + for _, live := range []moves.State{moves.Pending, moves.Final, moves.Committed} { id := "sb_" + string(live) epoch := mustReach(t, r, id, live) if _, err := r.Begin(ctx, id, "t", "u"); !errors.Is(err, moves.ErrLost) { @@ -355,6 +355,7 @@ func mustReach(t *testing.T, r moves.Registry, id string, state moves.State) int path := map[moves.State][]moves.State{ moves.Final: {moves.Final}, moves.Committed: {moves.Final, moves.Committed}, + moves.Started: {moves.Final, moves.Committed, moves.Started}, moves.Aborted: {moves.Aborted}, }[state] from := moves.Pending diff --git a/sandboxd/moves/pg/pg.go b/sandboxd/moves/pg/pg.go index 0ffdf787..beebc9b1 100644 --- a/sandboxd/moves/pg/pg.go +++ b/sandboxd/moves/pg/pg.go @@ -45,7 +45,7 @@ func (r *Registry) Begin(ctx context.Context, id, from, to string) (int64, error VALUES ($1, $2, $3, $4, 'pending', nextval('sandboxd_moves_epoch')) ON CONFLICT (cell, id) DO UPDATE SET from_node = EXCLUDED.from_node, to_node = EXCLUDED.to_node, state = EXCLUDED.state, epoch = EXCLUDED.epoch, updated_at = now() - WHERE sandboxd_moves.state IN ('committed', 'aborted') + WHERE sandboxd_moves.state IN ('started', 'aborted') RETURNING epoch`, r.cell, id, from, to) } diff --git a/sandboxd/moves/pg/schema.sql b/sandboxd/moves/pg/schema.sql index bde23c02..5d93e466 100644 --- a/sandboxd/moves/pg/schema.sql +++ b/sandboxd/moves/pg/schema.sql @@ -4,7 +4,7 @@ CREATE TABLE IF NOT EXISTS sandboxd_moves ( id text NOT NULL, from_node text NOT NULL, to_node text NOT NULL, - state text NOT NULL CHECK (state IN ('pending', 'final', 'committed', 'aborted')), + state text NOT NULL CHECK (state IN ('pending', 'final', 'committed', 'started', 'aborted')), epoch bigint NOT NULL CHECK (epoch > 0), updated_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (cell, id) From f02213135585a0eef51ac33843c6c6bb32e6bdda Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:03:08 +0800 Subject: [PATCH 16/53] pool: a move's record outlives each side's need for it, and a source resumes only on proof A source deleted the committed record as soon as it dropped the claim, and a target holding MoveIn read the missing record as an abort and removed the only guest, after a lost commit reply or a crash before its journal landed. A source also read a missing record, or another node's, as an abort, so one back from an outage resumed a claim that had moved on. The target now clears MoveIn before it writes started, and the source deletes only started and aborted records, so no record goes while the target still needs it. An abort writes aborted, resumes, drops the target, clears the journal and only then deletes, so a source holding a journal never finds its own abort missing and may read a missing record as a move. A target holding a claim on a pending or final record writes aborted instead of leaving it, because a commit whose reply was lost could still land after its read. --- docs/cluster.md | 41 ++-- sandboxd/pool/move.go | 14 +- sandboxd/pool/move_in.go | 57 ++++-- sandboxd/pool/move_out.go | 113 +++++------ sandboxd/pool/move_test.go | 334 ++++++++++++++++++++++++++------- sandboxd/pool/pool_test.go | 5 + sandboxd/server/move.go | 16 +- sandboxd/server/server.go | 2 +- sandboxd/server/server_test.go | 4 +- 9 files changed, 411 insertions(+), 175 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 96986c71..8d574c6c 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -336,17 +336,32 @@ and `epoch`, each change a compare-and-set at a new epoch. | (none) | `pending` | source | after the target admitted the move, before the baseline round | | `pending` | `final` | source | the final round paused the guest | | `final` | `committed` | target | after the receive; the point of no return | -| `pending` or `final` | `aborted` | source | before it resumes the guest | +| `committed` | `started` | target | after it journaled that it runs the claim | +| `pending` or `final` | `aborted` | source, or a target that holds a received claim or finds the record stale | before the guest resumes | + +A new move of the claim overwrites only a `started` or `aborted` record; a +`committed` one refuses it with 409 until its target marks it `started`. The target commits only from `final`, so it never starts a guest restored from -a round the source kept running after. The source resumes its guest only after -its `aborted` write lands; a write that finds `committed` means the claim moved, -and the source drops it. While the meta store is down, a guest paused in the -final round stays paused and the call answers 503; the abort is retried every 2 -seconds. The source deletes the record when the move ends. Each node's hourly -sweep aborts its records older than an hour that no move in flight holds, and -deletes only those it sourced: a source reads a missing record as an aborted -move, so a target never deletes one. +a round the source kept running after. The source resumes its guest only on +proof the move stayed here: a record it wrote, at or past the epoch in its +journal, in `pending` or `final` (it writes `aborted` first) or `aborted`. A +`committed` or `started` record, a missing one, or another node's means the +claim moved on, and the source drops it and its VM. While the meta store is +down, a guest paused in the final round stays paused and the call answers 503; +the abort is retried every 2 seconds. + +The record outlives each side's need for it. A target with a received claim +starts it on `committed` or `started` and drops it on `aborted` or a missing +record, which is safe because no record is deleted while the target still +journals that claim: it clears its journal first and only then writes `started`. +An abort runs in the order `aborted`, resume, target drop, journal clear, record +delete, so a source holding a journal never finds its own abort missing. The +source deletes a record only in `started` or `aborted`, never `committed`. Each +node's hourly sweep handles its records older than an hour that no move in +flight holds: a target writes `started` over a `committed` record, either side +writes `aborted` over a `pending` or `final` one, and the source deletes the +ended ones. **Rounds.** The source sends a baseline round, then up to 4 deltas, each over the control port with the root token. Once the last delta's size at the rate @@ -379,10 +394,10 @@ it again. **Failures.** Any failure before the commit aborts: the source resumes the guest, the target drops its reservation, files and any received VM, and the call answers the error. A target answer lost after its commit is resolved -through the record, and the call answers 200. A source restarted with a move -in its journal reads the record: `committed` drops the claim, anything else is -aborted and the guest resumes. A target restarted with a received claim not yet -started starts it when the record committed to it and removes it otherwise. A +through the record, and the call answers 200. A restarted node resolves a move +in its journal by the same rules, so a source back from an outage never resumes +a claim that moved on, and a target that crashed after its commit starts the +guest. A guest the target cannot start fails there through the [failed path](#running-claims-across-node-loss); it never goes back. diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index ef728c7d..3c2372e9 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -45,8 +45,8 @@ type MovePeer interface { // Admit reserves a claim slot on the target and returns its node id. Admit(ctx context.Context, addr, id string, a MoveAdmission) (string, error) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error - // Commit hands the target the claim record and the final epoch and returns the committed epoch. - Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) (int64, error) + // Commit hands the target the claim record and the final epoch. + Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) error Drop(ctx context.Context, addr, id string) error } @@ -79,7 +79,7 @@ func (m *Manager) moveDir(id string) string { return filepath.Join(m.dataDir, movesDir, id) } -// sweepStaleMoves aborts this node's stale moves and deletes only those it sourced, since a source reads a missing record as aborted. +// sweepStaleMoves ends this node's stale moves no move in flight holds: a target marks a committed one started, either side aborts an open one, and only the source deletes an ended one. func (m *Manager) sweepStaleMoves(ctx context.Context, now time.Time) { logger := log.WithFunc("pool.sweepStaleMoves") recs, err := m.moves.Stale(ctx, now.Add(-moveStaleAge)) @@ -92,7 +92,13 @@ func (m *Manager) sweepStaleMoves(ctx context.Context, now time.Time) { continue } epoch := rec.Epoch - if rec.State == moves.Pending || rec.State == moves.Final { + switch rec.State { + case moves.Committed: + if rec.To == m.nodeID { + m.markMoveStarted(ctx, rec.ID, rec.Epoch, nil) + } + continue + case moves.Pending, moves.Final: if epoch, err = m.moves.Advance(ctx, rec.ID, rec.Epoch, rec.State, moves.Aborted); err != nil { logger.Warnf(ctx, "abort stale move %s: %v", rec.ID, err) continue diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index 49e8d130..4e0dfdfb 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -63,24 +63,24 @@ func (m *Manager) ReceiveMoveRound(id string, body io.Reader) error { return nil } -// CommitMove receives the staged series as the claim, commits the move record and starts the guest; it returns the committed epoch. -func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch int64) (int64, error) { +// CommitMove receives the staged series as the claim, commits the move record and starts the guest. +func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch int64) error { ctx = context.WithoutCancel(ctx) if !m.touchReservation(id) { - return 0, fmt.Errorf("%w: no reservation for the move of %s", ErrNotMovable, id) + return fmt.Errorf("%w: no reservation for the move of %s", ErrNotMovable, id) } if m.memoryLow.Load() { - return 0, fmt.Errorf("%w: %s", ErrQuota, memoryLowReason) + return fmt.Errorf("%w: %s", ErrQuota, memoryLowReason) } sb := &types.Sandbox{} if err := json.Unmarshal(claim, sb); err != nil || sb.ID != id { - return 0, fmt.Errorf("%w: the claim record does not decode as %s", ErrNotMovable, id) + return fmt.Errorf("%w: the claim record does not decode as %s", ErrNotMovable, id) } name := vmName(sb.Key) rec, err := m.eng.MigrateReceive(ctx, m.moveDir(id), name, sb.Key) if err != nil { m.removeStaleVM(ctx, name, types.VMRecord{State: vmStateCreating}) - return 0, fmt.Errorf("move %s: receive: %w", id, err) + return fmt.Errorf("move %s: receive: %w", id, err) } sb.VMName, sb.VsockSocket, sb.TAP, sb.HibernateSnap, sb.PendingSnap = name, rec.VsockSocket, "", "", "" sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk, sb.ReplicaPending = "", 0, "", "", "" @@ -93,7 +93,7 @@ func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch if in == nil || m.claimed[id] != nil { m.mu.Unlock() m.destroy(ctx, name) - return 0, fmt.Errorf("%w: the reservation for %s ended", ErrNotMovable, id) + return fmt.Errorf("%w: the reservation for %s ended", ErrNotMovable, id) } sb.MoveIn = &types.MoveIn{From: in.from, Epoch: epoch} delete(m.incoming, id) @@ -102,19 +102,19 @@ func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch m.mu.Unlock() if err = m.store.commit(js); err != nil { m.dropMoveIn(ctx, sb) - return 0, fmt.Errorf("move %s: persist claims: %w", id, err) + return fmt.Errorf("move %s: persist claims: %w", id, err) } committed, err := m.moves.Advance(ctx, id, epoch, moves.Final, moves.Committed) switch { case errors.Is(err, moves.ErrLost): m.dropMoveIn(ctx, sb) - return 0, fmt.Errorf("%w: the source ended the move of %s", ErrNotMovable, id) + return fmt.Errorf("%w: the source ended the move of %s", ErrNotMovable, id) case err != nil: go m.settleMoveIn(ctx, sb) - return 0, fmt.Errorf("move %s: %w", id, err) + return fmt.Errorf("move %s: %w", id, err) } - m.startMoveIn(ctx, sb) - return committed, nil + m.startMoveIn(ctx, sb, committed) + return nil } // DropMove ends a reservation and its staging dir; a claim a commit already journaled is left to its record. @@ -128,8 +128,8 @@ func (m *Manager) DropMove(id string) { } } -// startMoveIn starts a committed claim's guest and serves it; a guest that cannot start fails the claim here, never back to the source. The caller holds sb.Transition. -func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox) { +// startMoveIn starts a committed claim's guest and serves it, then marks the record started at committed when that is its epoch; a guest that cannot start fails the claim here, never back to the source. The caller holds sb.Transition. +func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, committed int64) { logger := log.WithFunc("pool.startMoveIn") from := sb.MoveIn.From var err error @@ -140,11 +140,12 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox) { logger.Warnf(ctx, "start moved claim %s: %v", sb.ID, err) } if err != nil { - _, _ = m.commitIfLive(ctx, sb, func() { sb.MoveIn = nil }) + _, persistErr := m.commitIfLive(ctx, sb, func() { sb.MoveIn = nil }) sb.ExitRecorded = true m.recordUsage(ctx, usage.Event{Event: "vmm_exit", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) m.failLocked(ctx, sb, reasonMoveStartFailed) _ = os.RemoveAll(m.moveDir(sb.ID)) + m.markMoveStarted(ctx, sb.ID, committed, persistErr) return } if sock, probeErr := m.probeReady(ctx, sb.VMName, sb.VsockSocket, claimProbeTimeout); probeErr != nil { @@ -168,9 +169,19 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox) { sb.Touch() m.recordUsage(ctx, usage.Event{Event: "move_in", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant, Deadline: m.deadlineOf(sb)}) _ = os.RemoveAll(m.moveDir(sb.ID)) + m.markMoveStarted(ctx, sb.ID, committed, err) logger.Infof(ctx, "sandbox %s moved here from %s", sb.ID, from) } +func (m *Manager) markMoveStarted(ctx context.Context, id string, committed int64, persistErr error) { + if committed == 0 || persistErr != nil { + return + } + if _, err := m.moves.Advance(ctx, id, committed, moves.Committed, moves.Started); err != nil { + log.WithFunc("pool.markMoveStarted").Warnf(ctx, "mark the move of %s started, leaving it to the sweep: %v", id, err) + } +} + // dropMoveIn forgets a received claim whose move did not commit and removes its VM. func (m *Manager) dropMoveIn(ctx context.Context, sb *types.Sandbox) { if holds, ok := m.forgetClaim(ctx, sb); ok { @@ -187,7 +198,7 @@ func (m *Manager) settleMoveIn(ctx context.Context, sb *types.Sandbox) { } } -// resolveMoveIn starts a received claim whose record committed to this node and drops any other; false while the registry is down. +// resolveMoveIn starts a received claim whose record committed to this node, ends a move its source left open and drops any other; false while the registry is down. func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -202,9 +213,19 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "resolve moved claim %s, retrying: %v", sb.ID, err) return false } - if ok && rec.State == moves.Committed && rec.To == m.nodeID { - m.startMoveIn(ctx, sb) + toHere := ok && rec.To == m.nodeID + switch { + case toHere && rec.State == moves.Committed: + m.startMoveIn(ctx, sb, rec.Epoch) return true + case toHere && rec.State == moves.Started: + m.startMoveIn(ctx, sb, 0) + return true + case toHere && (rec.State == moves.Pending || rec.State == moves.Final): + if _, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted); err != nil { + log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) + return false + } } m.dropMoveIn(ctx, sb) return true diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 061636b3..522f5588 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -16,11 +16,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/usage" ) -type movedAt struct { - epoch int64 - downtime time.Duration -} - // Move hands a running claim to the peer at addr with cocoon's live migration, pausing its guest for about maxDowntime at most. func (m *Manager) Move(ctx context.Context, id, addr string, maxDowntime time.Duration) (MoveResult, error) { sb, ok := m.byID(id) @@ -41,42 +36,42 @@ func (m *Manager) Move(ctx context.Context, id, addr string, maxDowntime time.Du return MoveResult{}, err } ctx = context.WithoutCancel(ctx) - paused, err := m.moveOut(ctx, sb, mv, cmp.Or(maxDowntime, DefaultMoveDowntime)) + downtime, err := m.moveOut(ctx, sb, mv, cmp.Or(maxDowntime, DefaultMoveDowntime)) if err == nil { - m.finishMoveOut(ctx, sb, mv, paused.epoch) - return MoveResult{Rounds: mv.rounds, Downtime: paused.downtime}, nil + m.finishMoveOut(ctx, sb, mv) + return MoveResult{Rounds: mv.rounds, Downtime: downtime}, nil } return m.failMoveOut(ctx, sb, mv, err) } // moveOut runs the rounds up to the target's commit; on any error the caller resolves the move through its record. -func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, budget time.Duration) (movedAt, error) { +func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, budget time.Duration) (time.Duration, error) { m.mu.Lock() admission, vm := MoveAdmission{Tenant: sb.Tenant, Key: sb.Key, From: m.nodeID}, sb.VMName m.mu.Unlock() to, err := m.movePeer.Admit(ctx, mv.addr, sb.ID, admission) if err != nil { - return movedAt{}, err + return 0, err } mv.to = to epoch, err := m.moves.Begin(ctx, sb.ID, m.nodeID, to) if errors.Is(err, moves.ErrLost) { - return movedAt{}, fmt.Errorf("%w: another move of %s is on record", ErrNotMovable, sb.ID) + return 0, fmt.Errorf("%w: another move of %s is on record", ErrNotMovable, sb.ID) } if err != nil { - return movedAt{}, err + return 0, err } mv.epoch = epoch if err = m.journalMoveOut(ctx, sb, mv); err != nil { - return movedAt{}, err + return 0, err } round, err := m.eng.MigrateCapture(ctx, vm, "", false) if err != nil { - return movedAt{}, fmt.Errorf("%w: baseline capture: %w", ErrNotMovable, err) + return 0, fmt.Errorf("%w: baseline capture: %w", ErrNotMovable, err) } mv.series = round.Series if err = m.journalMoveOut(ctx, sb, mv); err != nil { - return movedAt{}, err + return 0, err } var sent int64 var sending time.Duration @@ -84,7 +79,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b for i := 0; ; i++ { start := time.Now() if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { - return movedAt{}, err + return 0, err } sending += time.Since(start) size := roundBytes(round) @@ -96,11 +91,11 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b break } if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, false); err != nil { - return movedAt{}, fmt.Errorf("delta capture: %w", err) + return 0, fmt.Errorf("delta capture: %w", err) } } if !fits { - return movedAt{}, fmt.Errorf("%w: %d deltas did not shrink under %s", ErrMoveDowntime, moveDeltaLimit, budget) + return 0, fmt.Errorf("%w: %d deltas did not shrink under %s", ErrMoveDowntime, moveDeltaLimit, budget) } m.mu.Lock() mv.final = true @@ -110,18 +105,18 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b ctx, cancel := context.WithTimeout(ctx, moveFinalTimeout) defer cancel() if err = m.unrowForMove(ctx, sb); err != nil { - return movedAt{}, err + return 0, err } pausedAt := time.Now() if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, true); err != nil { - return movedAt{}, fmt.Errorf("final capture: %w", err) + return 0, fmt.Errorf("final capture: %w", err) } final, err := m.moves.Advance(ctx, sb.ID, mv.epoch, moves.Pending, moves.Final) if err != nil { - return movedAt{}, err + return 0, err } if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { - return movedAt{}, err + return 0, err } m.mu.Lock() dto := dtoOf(sb) @@ -129,13 +124,12 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b dto.MoveOut = nil claim, err := json.Marshal(&dto) if err != nil { - return movedAt{}, fmt.Errorf("encode claim: %w", err) + return 0, fmt.Errorf("encode claim: %w", err) } - committed, err := m.movePeer.Commit(ctx, mv.addr, sb.ID, claim, final) - if err != nil { - return movedAt{}, err + if err = m.movePeer.Commit(ctx, mv.addr, sb.ID, claim, final); err != nil { + return 0, err } - return movedAt{epoch: committed, downtime: time.Since(pausedAt)}, nil + return time.Since(pausedAt), nil } func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round types.MigrateRound) error { @@ -236,8 +230,8 @@ func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox) error { return err } -// finishMoveOut drops a claim the target committed, like a claim another node adopted. -func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, committed int64) { +// finishMoveOut drops a claim that moved on, like a claim another node adopted, and deletes the record once its target marked it started. +func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) { m.mu.Lock() vm := sb.VMName m.mu.Unlock() @@ -245,25 +239,33 @@ func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM teardown() m.recordUsage(ctx, usage.Event{Event: "move_out", ID: sb.ID, VMName: vm, Tenant: sb.Tenant}) } - if err := m.moves.Delete(ctx, sb.ID, committed); err != nil { - log.WithFunc("pool.finishMoveOut").Warnf(ctx, "delete the committed move of %s: %v", sb.ID, err) - } m.endMoveOut(sb, mv) + logger := log.WithFunc("pool.finishMoveOut") + rec, ok, err := m.moves.Get(ctx, sb.ID) + if err != nil { + logger.Warnf(ctx, "read the move of %s, leaving it to the sweep: %v", sb.ID, err) + return + } + if ok && rec.From == m.nodeID && rec.Epoch >= mv.epoch && rec.State == moves.Started { + if err = m.moves.Delete(ctx, sb.ID, rec.Epoch); err != nil { + logger.Warnf(ctx, "delete the move of %s, leaving it to the sweep: %v", sb.ID, err) + } + } } -// failMoveOut resolves a failed move through its record: committed finishes it, anything else aborts and resumes the guest. +// failMoveOut resolves a failed move through its record: only proof that the move stayed here resumes the guest, anything else drops the claim. func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, cause error) (MoveResult, error) { if mv.epoch == 0 { - _ = m.abortMoveOut(ctx, sb, mv) + _ = m.abortMoveOut(ctx, sb, mv, 0) return MoveResult{}, cause } - committed, err := m.settleMoveOut(ctx, sb, mv) + moved, err := m.settleMoveOut(ctx, sb, mv) switch { case err != nil: log.WithFunc("pool.failMoveOut").Warnf(ctx, "move of %s failed (%v) and its abort waits for the registry: %v", sb.ID, cause, err) go m.retryMoveOut(ctx, sb, mv) return MoveResult{}, fmt.Errorf("%w: %w", ErrMoveStoreDown, cause) - case committed: + case moved: return MoveResult{Rounds: mv.rounds}, nil } return MoveResult{}, cause @@ -280,7 +282,7 @@ func (m *Manager) retryMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo } } -// settleMoveOut reads the record: committed drops the claim, pending or final is aborted before the guest resumes; an error leaves the move as it is. +// settleMoveOut resumes the guest only on a record of this move still open or aborted; a missing record, another node's, or a committed or started one means the claim moved on. An error leaves the move as it is. func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) (bool, error) { for range 2 { rec, ok, err := m.moves.Get(ctx, sb.ID) @@ -289,32 +291,26 @@ func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM } ours := ok && rec.From == m.nodeID && rec.Epoch >= mv.epoch switch { - case ours && rec.State == moves.Committed: - m.finishMoveOut(ctx, sb, mv, rec.Epoch) + case !ours || rec.State == moves.Committed || rec.State == moves.Started: + m.finishMoveOut(ctx, sb, mv) return true, nil - case ours && (rec.State == moves.Pending || rec.State == moves.Final): - var aborted int64 - aborted, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted) + case rec.State == moves.Pending || rec.State == moves.Final: + rec.Epoch, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted) if errors.Is(err, moves.ErrLost) { continue } if err != nil { return false, err } - rec.Epoch, rec.State = aborted, moves.Aborted } - if ours && rec.State == moves.Aborted { - if err = m.moves.Delete(ctx, sb.ID, rec.Epoch); err != nil && !errors.Is(err, moves.ErrLost) { - return false, err - } - } - return false, m.abortMoveOut(ctx, sb, mv) + return false, m.abortMoveOut(ctx, sb, mv, rec.Epoch) } return false, fmt.Errorf("the move record of %s kept changing", sb.ID) } -// abortMoveOut ends the series, which resumes a paused guest, and releases the target and the budget slot. -func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) error { +// abortMoveOut ends the series, which resumes a paused guest, releases the target and the budget slot, and deletes the aborted record only once the journal no longer needs it. +func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, aborted int64) error { + logger := log.WithFunc("pool.abortMoveOut") if mv.series != "" { m.mu.Lock() vm := sb.VMName @@ -325,13 +321,20 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo } if mv.addr != "" && mv.to != "" { if err := m.movePeer.Drop(ctx, mv.addr, sb.ID); err != nil { - log.WithFunc("pool.abortMoveOut").Warnf(ctx, "drop the reservation for %s on %s: %v", sb.ID, mv.addr, err) + logger.Warnf(ctx, "drop the reservation for %s on %s: %v", sb.ID, mv.addr, err) } } - if _, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = nil }); err != nil { - log.WithFunc("pool.abortMoveOut").Warnf(ctx, "clear the move journal of %s: %v", sb.ID, err) - } + _, journalErr := m.commitIfLive(ctx, sb, func() { sb.MoveOut = nil }) m.endMoveOut(sb, mv) + if journalErr != nil { + logger.Warnf(ctx, "clear the move journal of %s, leaving its record to the sweep: %v", sb.ID, journalErr) + return nil + } + if aborted != 0 { + if err := m.moves.Delete(ctx, sb.ID, aborted); err != nil { + logger.Warnf(ctx, "delete the aborted move of %s, leaving it to the sweep: %v", sb.ID, err) + } + } return nil } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 0e062a95..365c7c42 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -330,77 +331,80 @@ func TestMoveRefusesClaimsThatCannotMove(t *testing.T) { } func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { + const vm, series = "sbx-moved", "s1" + aborted := func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.aborts, vm+":"+series) || slices.Contains(eng.removes, vm) { + t.Errorf("aborts %v removes %v, want the series aborted and the VM kept", eng.aborts, eng.removes) + } + } + movedOn := func(t *testing.T, eng *fakeEngine) { + if len(eng.aborts) != 0 || !slices.Contains(eng.removes, vm) { + t.Errorf("aborts %v removes %v, want the VM removed and never resumed", eng.aborts, eng.removes) + } + } + started := func(t *testing.T, eng *fakeEngine) { + if !slices.Contains(eng.starts, vm) || slices.Contains(eng.removes, vm) { + t.Errorf("starts %v removes %v, want the received VM started", eng.starts, eng.removes) + } + } + unstarted := func(t *testing.T, eng *fakeEngine) { + if slices.Contains(eng.starts, vm) || !slices.Contains(eng.removes, vm) { + t.Errorf("starts %v removes %v, want the received VM removed unstarted", eng.starts, eng.removes) + } + } for name, tt := range map[string]struct { - node string - reach moves.State - journal func(epoch int64) *types.Sandbox - wantClaim bool - check func(t *testing.T, eng *fakeEngine) + node string + from, to string + reach moves.State + wantClaim bool + wantRecord moves.State + check func(t *testing.T, eng *fakeEngine) }{ - "source at pending": {node: "a", reach: moves.Pending, wantClaim: true, journal: func(epoch int64) *types.Sandbox { - return &types.Sandbox{MoveOut: &types.MoveOut{To: "b", Epoch: epoch, Series: "s1"}} - }, check: func(t *testing.T, eng *fakeEngine) { - if !slices.Contains(eng.aborts, "sbx-moved:s1") { - t.Errorf("aborts %v, want the series aborted", eng.aborts) - } - }}, - "source at final": {node: "a", reach: moves.Final, wantClaim: true, journal: func(epoch int64) *types.Sandbox { - return &types.Sandbox{MoveOut: &types.MoveOut{To: "b", Epoch: epoch, Series: "s1"}} - }, check: func(t *testing.T, eng *fakeEngine) { - if !slices.Contains(eng.aborts, "sbx-moved:s1") { - t.Errorf("aborts %v, want the series aborted", eng.aborts) - } - }}, - "source at committed": {node: "a", reach: moves.Committed, journal: func(epoch int64) *types.Sandbox { - return &types.Sandbox{MoveOut: &types.MoveOut{To: "b", Epoch: epoch, Series: "s1"}} - }, check: func(t *testing.T, eng *fakeEngine) { - if !slices.Contains(eng.removes, "sbx-moved") { - t.Errorf("removes %v, want the moved-away VM removed", eng.removes) - } - }}, - "target at committed": {node: "b", reach: moves.Committed, wantClaim: true, journal: func(epoch int64) *types.Sandbox { - return &types.Sandbox{MoveIn: &types.MoveIn{From: "a", Epoch: epoch}} - }, check: func(t *testing.T, eng *fakeEngine) { - if !slices.Contains(eng.starts, "sbx-moved") { - t.Errorf("starts %v, want the received VM started", eng.starts) - } - }}, - "target at aborted": {node: "b", reach: moves.Aborted, journal: func(epoch int64) *types.Sandbox { - return &types.Sandbox{MoveIn: &types.MoveIn{From: "a", Epoch: epoch}} - }, check: func(t *testing.T, eng *fakeEngine) { - if !slices.Contains(eng.removes, "sbx-moved") || slices.Contains(eng.starts, "sbx-moved") { - t.Errorf("removes %v starts %v, want the received VM removed unstarted", eng.removes, eng.starts) - } - }}, + "source at pending": {node: "a", from: "a", to: "b", reach: moves.Pending, wantClaim: true, check: aborted}, + "source at final": {node: "a", from: "a", to: "b", reach: moves.Final, wantClaim: true, check: aborted}, + "source at aborted": {node: "a", from: "a", to: "b", reach: moves.Aborted, wantClaim: true, check: aborted}, + "source at committed": {node: "a", from: "a", to: "b", reach: moves.Committed, wantRecord: moves.Committed, check: movedOn}, + "source at started": {node: "a", from: "a", to: "b", reach: moves.Started, check: movedOn}, + "source, record missing": {node: "a", check: movedOn}, + "source, another node moved": {node: "a", from: "b", to: "c", reach: moves.Pending, wantRecord: moves.Pending, check: movedOn}, + "target at final": {node: "b", from: "a", to: "b", reach: moves.Final, wantRecord: moves.Aborted, check: unstarted}, + "target at committed": {node: "b", from: "a", to: "b", reach: moves.Committed, wantClaim: true, wantRecord: moves.Started, check: started}, + "target at started": {node: "b", from: "a", to: "b", reach: moves.Started, wantClaim: true, wantRecord: moves.Started, check: started}, + "target at aborted": {node: "b", from: "a", to: "b", reach: moves.Aborted, wantRecord: moves.Aborted, check: unstarted}, + "target, record missing": {node: "b", check: unstarted}, } { t.Run(name, func(t *testing.T) { c := newFailoverCluster(t) reg := movestest.NewMem() c.moves = reg ctx := t.Context() - epoch, _ := reg.Begin(ctx, "sb_moved", "a", "b") - path := map[moves.State][]moves.State{moves.Final: {moves.Final}, moves.Committed: {moves.Final, moves.Committed}, moves.Aborted: {moves.Aborted}}[tt.reach] - from, at := moves.Pending, epoch - for _, to := range path { - at, _ = reg.Advance(ctx, "sb_moved", at, from, to) - from = to + journalEpoch := int64(1) + if tt.from != "" { + journalEpoch = seedMoveRecord(t, reg, "sb_moved", tt.from, tt.to, tt.reach) + } + sb := &types.Sandbox{ID: "sb_moved", VMName: vm, Key: testKey, Token: "tok", Deadline: time.Now().Add(time.Hour)} + if tt.node == "a" { + sb.MoveOut = &types.MoveOut{To: "b", Epoch: journalEpoch, Series: series} + } else { + sb.MoveIn = &types.MoveIn{From: "a", Epoch: journalEpoch} } - sb := tt.journal(epoch) - sb.ID, sb.VMName, sb.Key, sb.Token, sb.Deadline = "sb_moved", "sbx-moved", testKey, "tok", time.Now().Add(time.Hour) dataDir := t.TempDir() seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) if err := os.MkdirAll(filepath.Join(dataDir, movesDir, sb.ID), 0o700); err != nil { t.Fatal(err) } eng := newFakeEngine() - eng.vms["sbx-moved"], eng.stopped["sbx-moved"] = "/vsock/sbx-moved", sb.MoveIn != nil + eng.vms[vm], eng.stopped[vm] = "/vsock/"+vm, sb.MoveIn != nil + eng.migrateSeries[vm] = series m := c.nodeOn(t, eng, tt.node, dataDir, keepAnHour) m.liftFence() if err := m.Reconcile(ctx); err != nil { t.Fatalf("reconcile: %v", err) } - waitFor(t, func() bool { return !m.moveHeld(sb.ID) }) - waitFor(t, func() bool { _, ok, _ := reg.Get(ctx, sb.ID); return !ok || tt.node == "b" }) + waitFor(t, func() bool { + rec, ok, _ := reg.Get(ctx, sb.ID) + return !m.moveHeld(sb.ID) && (ok && rec.State == tt.wantRecord || !ok && tt.wantRecord == "") + }) held, ok := m.byID(sb.ID) if ok != tt.wantClaim { t.Fatalf("claim held %v, want %v", ok, tt.wantClaim) @@ -418,32 +422,156 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { func TestStaleMoveRecordsAreDeletedOnlyByTheirSource(t *testing.T) { mc := newMoveCluster(t) ctx, reg := t.Context(), mc.moveReg - done, _ := reg.Begin(ctx, "sb_done", "a", "b") - done, _ = reg.Advance(ctx, "sb_done", done, moves.Pending, moves.Final) - if _, err := reg.Advance(ctx, "sb_done", done, moves.Final, moves.Committed); err != nil { - t.Fatalf("commit: %v", err) + seedMoveRecord(t, reg, "sb_done", "a", "b", moves.Committed) + seedMoveRecord(t, reg, "sb_open", "a", "b", moves.Pending) + later := time.Now().Add(2 * moveStaleAge) + stateOf := func(id string) moves.State { + rec, _, _ := reg.Get(ctx, id) + return rec.State + } + + mc.s.sweepStaleMoves(ctx, later) + if got := stateOf("sb_done"); got != moves.Committed { + t.Errorf("committed record after the source's sweep: %q, want it kept until the target advances it", got) } - if _, err := reg.Begin(ctx, "sb_open", "a", "b"); err != nil { - t.Fatalf("begin: %v", err) + if got := stateOf("sb_open"); got != "" { + t.Errorf("open record after the source's sweep: %q, want it aborted and deleted", got) } - later := time.Now().Add(2 * moveStaleAge) + seedMoveRecord(t, reg, "sb_open", "a", "b", moves.Pending) mc.t.sweepStaleMoves(ctx, later) - if rec, ok, _ := reg.Get(ctx, "sb_done"); !ok || rec.State != moves.Committed { - t.Errorf("committed record after the target's sweep: %+v %v, want it kept for its source", rec, ok) + if got := stateOf("sb_done"); got != moves.Started { + t.Errorf("committed record after the target's sweep: %q, want it started and kept", got) } - if rec, ok, _ := reg.Get(ctx, "sb_open"); !ok || rec.State != moves.Aborted { - t.Errorf("open record after the target's sweep: %+v %v, want it aborted and kept", rec, ok) + if got := stateOf("sb_open"); got != moves.Aborted { + t.Errorf("open record after the target's sweep: %q, want it aborted and kept", got) } mc.s.sweepStaleMoves(ctx, later) for _, id := range []string{"sb_done", "sb_open"} { - if rec, ok, _ := reg.Get(ctx, id); ok { - t.Errorf("record %s after the source's sweep: %+v, want it deleted", id, rec) + if got := stateOf(id); got != "" { + t.Errorf("record %s after the source's second sweep: %q, want it deleted", id, got) } } } +func TestMoveStaleSweepLeavesACommittedRecordItsTargetStillHoldsMoveInFor(t *testing.T) { + mc := newMoveCluster(t) + ctx, reg := t.Context(), mc.moveReg + epoch := seedMoveRecord(t, reg, "sb_held", "a", "b", moves.Committed) + sb := &types.Sandbox{ID: "sb_held", VMName: "sbx-held", Key: testKey, Deadline: time.Now().Add(time.Hour), MoveIn: &types.MoveIn{From: "a", Epoch: epoch}} + mc.t.mu.Lock() + mc.t.addClaimLocked(sb) + mc.t.store.set(sb) + mc.t.mu.Unlock() + mc.t.sweepStaleMoves(ctx, time.Now().Add(2*moveStaleAge)) + if rec, _, _ := reg.Get(ctx, "sb_held"); rec.State != moves.Committed { + t.Errorf("record %+v, want committed until the target persists its journal", rec) + } +} + +func TestMoveCommitWhoseRegistryReplyIsLostKeepsTheOnlyGuest(t *testing.T) { + mc := newMoveCluster(t) + mc.t.moves = &lossyMoves{Registry: mc.moveReg, loseCommit: true} + sb := mustClaim(t, mc.s, testKey) + vm := sb.VMName + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v, want it resolved through the committed record", err) + } + if _, held := mc.s.byID(sb.ID); held || !fakeOf(mc.s).removed(vm) { + t.Fatal("the source kept a claim the target committed") + } + waitFor(t, func() bool { + moved, ok := mc.t.byID(sb.ID) + return ok && !mc.t.moveHeld(sb.ID) && slices.Contains(startsOf(mc.t), moved.VMName) + }) + waitFor(t, func() bool { + rec, ok, _ := mc.moveReg.Get(t.Context(), sb.ID) + return !ok || rec.State == moves.Started + }) +} + +func TestMoveTargetMarksTheRecordStartedOnlyAfterItsJournalPersists(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + ctx := t.Context() + fakeOf(mc.t).startHook = func() { breakStore(t, mc.t) } + if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + if rec, ok, _ := mc.moveReg.Get(ctx, sb.ID); !ok || rec.State != moves.Committed { + t.Fatalf("record %+v %v with the target's journal unpersisted, want committed", rec, ok) + } + healStore(t, mc.t) + later := time.Now().Add(2 * moveStaleAge) + mc.t.sweepStaleMoves(ctx, later) + if rec, ok, _ := mc.moveReg.Get(ctx, sb.ID); !ok || rec.State != moves.Started { + t.Fatalf("record %+v %v after the target's sweep, want started", rec, ok) + } + mc.s.sweepStaleMoves(ctx, later) + if _, ok, _ := mc.moveReg.Get(ctx, sb.ID); ok { + t.Error("the source's sweep kept a started record") + } +} + +func TestMoveAbortKeepsTheRecordUntilTheJournalIsCleared(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + ctx := t.Context() + mc.p.commitHook = func(int) error { return ErrMoveTarget } + mc.p.dropHook = func() { breakStore(t, mc.s) } + if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveTarget) { + t.Fatalf("move: %v, want ErrMoveTarget", err) + } + if rec, ok, _ := mc.moveReg.Get(ctx, sb.ID); !ok || rec.State != moves.Aborted { + t.Fatalf("record %+v %v with the journal unpersisted, want it kept aborted", rec, ok) + } + assertStillOnSource(t, mc, sb) + healStore(t, mc.s) + mc.s.sweepStaleMoves(ctx, time.Now().Add(2*moveStaleAge)) + if _, ok, _ := mc.moveReg.Get(ctx, sb.ID); ok { + t.Error("the sweep kept the aborted record") + } +} + +func TestMoveSourceBackFromAnOutageNeverResumesAClaimThatMovedOn(t *testing.T) { + mc := newMoveCluster(t) + third := mc.moveNode(t, "c") + sb := mustClaim(t, mc.s, testKey) + vm := sb.VMName + down := &lossyMoves{Registry: mc.moveReg} + mc.s.moves = down + mc.p.commitHook = func(int) error { + down.setDown(true) + return nil + } + mc.p.loseCommitReply = true + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveStoreDown) { + t.Fatalf("move with the source cut off: %v, want ErrMoveStoreDown", err) + } + mc.p.commitHook, mc.p.loseCommitReply = nil, false + if !pausedOn(mc.s, vm) { + t.Fatal("the source resumed while it could not read the record") + } + if _, err := mc.t.Move(t.Context(), sb.ID, "c", time.Minute); err != nil { + t.Fatalf("move on to the third node: %v", err) + } + if _, held := third.byID(sb.ID); !held { + t.Fatal("the third node does not hold the claim") + } + down.setDown(false) + waitFor(t, func() bool { + _, held := mc.s.byID(sb.ID) + return !held && !mc.s.movingNow(sb.ID) + }) + eng := fakeOf(mc.s) + eng.mu.Lock() + defer eng.mu.Unlock() + if len(eng.aborts) != 0 || !slices.Contains(eng.removes, vm) { + t.Errorf("source aborts %v removes %v, want the VM removed and never resumed", eng.aborts, eng.removes) + } +} + func BenchmarkQuotaErr(b *testing.B) { for _, incoming := range []int{0, 2} { b.Run(map[int]string{0: "no reservations", 2: "two reservations"}[incoming], func(b *testing.B) { @@ -505,6 +633,7 @@ type fakeMovePeer struct { roundHook func(round int) error commitHook func(round int) error loseCommitReply bool + dropHook func() } func newFakeMovePeer(t *testing.T, c *failoverCluster) *fakeMovePeer { @@ -541,20 +670,23 @@ func (p *fakeMovePeer) SendRound(ctx context.Context, addr, id string, round int return nil } -func (p *fakeMovePeer) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) (int64, error) { +func (p *fakeMovePeer) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) error { if hook := p.commitHook; hook != nil { if err := hook(0); err != nil { - return 0, err + return err } } - committed, err := p.target(addr).CommitMove(ctx, id, claim, epoch) + err := p.target(addr).CommitMove(ctx, id, claim, epoch) if err == nil && p.loseCommitReply { - return 0, ErrMoveTarget + return ErrMoveTarget } - return committed, err + return err } func (p *fakeMovePeer) Drop(_ context.Context, addr, id string) error { + if hook := p.dropHook; hook != nil { + hook() + } p.target(addr).DropMove(id) return nil } @@ -566,6 +698,70 @@ func (p *fakeMovePeer) target(addr string) *Manager { return p.c.nodes[i] } +type lossyMoves struct { + moves.Registry + loseCommit bool + down atomic.Bool +} + +func (r *lossyMoves) setDown(down bool) { r.down.Store(down) } + +func (r *lossyMoves) Begin(ctx context.Context, id, from, to string) (int64, error) { + if r.down.Load() { + return 0, moves.ErrUnavailable + } + return r.Registry.Begin(ctx, id, from, to) +} + +func (r *lossyMoves) Advance(ctx context.Context, id string, expect int64, from, to moves.State) (int64, error) { + if r.down.Load() { + return 0, moves.ErrUnavailable + } + epoch, err := r.Registry.Advance(ctx, id, expect, from, to) + if err == nil && to == moves.Committed && r.loseCommit { + r.loseCommit = false + return 0, moves.ErrUnavailable + } + return epoch, err +} + +func (r *lossyMoves) Get(ctx context.Context, id string) (moves.Record, bool, error) { + if r.down.Load() { + return moves.Record{}, false, moves.ErrUnavailable + } + return r.Registry.Get(ctx, id) +} + +func seedMoveRecord(t *testing.T, reg moves.Registry, id, from, to string, state moves.State) int64 { + t.Helper() + ctx := t.Context() + epoch, err := reg.Begin(ctx, id, from, to) + if err != nil { + t.Fatalf("begin %s: %v", id, err) + } + path := map[moves.State][]moves.State{ + moves.Final: {moves.Final}, + moves.Committed: {moves.Final, moves.Committed}, + moves.Started: {moves.Final, moves.Committed, moves.Started}, + moves.Aborted: {moves.Aborted}, + }[state] + at, reached := epoch, moves.Pending + for _, next := range path { + if at, err = reg.Advance(ctx, id, at, reached, next); err != nil { + t.Fatalf("advance %s to %s: %v", id, next, err) + } + reached = next + } + return epoch +} + +func startsOf(m *Manager) []string { + eng := fakeOf(m) + eng.mu.Lock() + defer eng.mu.Unlock() + return slices.Clone(eng.starts) +} + func moveCounts(m *Manager, tenant string) [3]int { m.mu.Lock() defer m.mu.Unlock() diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index dc75e63f..8a5eb22d 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1270,6 +1270,7 @@ type fakeEngine struct { snapListErr error snapSaveErr error startErr error + startHook func() syncErr error trimErr error installCAErr error @@ -1410,6 +1411,10 @@ func (f *fakeEngine) Remove(ctx context.Context, name string) error { func (f *fakeEngine) Start(_ context.Context, name string) error { f.mu.Lock() defer f.mu.Unlock() + if hook := f.startHook; hook != nil { + f.startHook = nil + hook() + } f.starts = append(f.starts, name) if f.startErr != nil { return f.startErr diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go index 3ce6e1fb..682ed1e0 100644 --- a/sandboxd/server/move.go +++ b/sandboxd/server/move.go @@ -40,10 +40,8 @@ func (c *MoveClient) SendRound(ctx context.Context, addr, id string, round int, return nil } -func (c *MoveClient) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) (int64, error) { - var out moveCommitResponse - err := c.call(ctx, http.MethodPost, addr, movePath(id, "/commit"), moveCommitRequest{Claim: claim, Epoch: epoch}, &out) - return out.Epoch, err +func (c *MoveClient) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) error { + return c.call(ctx, http.MethodPost, addr, movePath(id, "/commit"), moveCommitRequest{Claim: claim, Epoch: epoch}, nil) } func (c *MoveClient) Drop(ctx context.Context, addr, id string) error { @@ -119,10 +117,6 @@ type moveCommitRequest struct { Epoch int64 `json:"epoch"` } -type moveCommitResponse struct { - Epoch int64 `json:"epoch"` -} - func (s *Server) handleMove(w http.ResponseWriter, r *http.Request) { req, ok := decodeBodyStrict[MoveRequest](w, r) if !ok { @@ -169,10 +163,8 @@ func (s *Server) handleCommitMove(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - epoch, err := s.mgr.CommitMove(r.Context(), id, req.Claim, req.Epoch) - writeResult(w, r, "commit move", id, "commit move failed", err, func() { - writeJSON(w, http.StatusOK, moveCommitResponse{Epoch: epoch}) - }) + err := s.mgr.CommitMove(r.Context(), id, req.Claim, req.Epoch) + writeResult(w, r, "commit move", id, "commit move failed", err, noContent(w)) } func (s *Server) handleDropMove(w http.ResponseWriter, r *http.Request) { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 1c71ded6..80560e82 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -149,7 +149,7 @@ type Manager interface { Move(ctx context.Context, id, addr string, maxDowntime time.Duration) (pool.MoveResult, error) AdmitMove(id string, a pool.MoveAdmission) (string, error) ReceiveMoveRound(id string, body io.Reader) error - CommitMove(ctx context.Context, id string, claim []byte, epoch int64) (int64, error) + CommitMove(ctx context.Context, id string, claim []byte, epoch int64) error DropMove(id string) } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index db553fc5..d7edd338 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2607,9 +2607,7 @@ func (f *fakeManager) AdmitMove(string, pool.MoveAdmission) (string, error) { re func (f *fakeManager) ReceiveMoveRound(string, io.Reader) error { return nil } -func (f *fakeManager) CommitMove(context.Context, string, []byte, int64) (int64, error) { - return 0, nil -} +func (f *fakeManager) CommitMove(context.Context, string, []byte, int64) error { return nil } func (f *fakeManager) DropMove(string) {} From 154a4002678ab9b732f3dc07aaef234f4ae6c7e2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:03:34 +0800 Subject: [PATCH 17/53] pool: renew and env writes answer 503 during a move's final round Both succeeded on the source while the guest was paused for the final round, after or beside the record the target received, so the write vanished with the source's claim. The final flag and the claim snapshot are ordered by m.mu, and a write that holds Transition lands before the move can take it. --- docs/sandboxd-api.md | 3 ++- sandboxd/pool/claim.go | 4 ++++ sandboxd/pool/env.go | 4 ++++ sandboxd/pool/move_out.go | 4 ++++ sandboxd/pool/move_test.go | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index b6909da5..05b5210b 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -859,7 +859,8 @@ from the final capture to the peer's commit. While the move runs, release of the claim answers 503 `sandbox moving to another node; retry`, and fork, checkpoint, promote and hibernate answer the same during the rounds and wait out the final round. Env writes, renew and data-plane calls keep working until -the final round. Every error leaves the claim running here: +the final round and answer the same 503 during it, because the peer would miss +them. Every error leaves the claim running here: - 404 unknown sandbox - 409 the claim cannot move (hibernated, archived, failed, waking, with diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 37cc9970..6bf51a3d 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -167,6 +167,10 @@ func (m *Manager) Renew(ctx context.Context, id string, cred Cred, ttl time.Dura m.mu.Unlock() return time.Time{}, ErrUnknownSandbox } + if m.movingFinalLocked(id) { + m.mu.Unlock() + return time.Time{}, ErrEvacuating + } // An archived claim's Deadline is a retention window, not a lease: a TTL over it schedules a purge if _, archiving := m.archiving[id]; sb.ArchiveCk != "" || archiving { m.mu.Unlock() diff --git a/sandboxd/pool/env.go b/sandboxd/pool/env.go index 7c9a656b..7e66fd8c 100644 --- a/sandboxd/pool/env.go +++ b/sandboxd/pool/env.go @@ -50,6 +50,10 @@ func (m *Manager) writeEnv(ctx context.Context, id, tenant, verb string, next fu m.mu.Unlock() return ErrUnknownSandbox } + if !locked && m.movingFinalLocked(id) { + m.mu.Unlock() + return ErrEvacuating + } prev, sock, failed := sb.Env, sb.VsockSocket, sb.Failed != "" env := next(prev) if len(env) == 0 { diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 522f5588..695ce332 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -354,6 +354,10 @@ func (m *Manager) endMoveOut(sb *types.Sandbox, mv *outMove) { func (m *Manager) movingFinal(id string) bool { m.mu.Lock() defer m.mu.Unlock() + return m.movingFinalLocked(id) +} + +func (m *Manager) movingFinalLocked(id string) bool { mv := m.moving[id] return mv != nil && mv.final } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 365c7c42..dd2eed05 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -310,6 +310,39 @@ func TestMoveHoldsOffCapturesReleasesAndEvacuationWhileItRuns(t *testing.T) { } } +func TestMoveRefusesWritesThatWouldMissTheRecordTheTargetReceives(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + var deadline time.Time + mc.p.roundHook = func(round int) error { + if round != 1 { + return nil + } + var err error + if deadline, err = mc.s.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Hour, ""); err != nil { + t.Errorf("renew during the rounds: %v", err) + } + patchHostOnly(t, mc.s, sb, "rounds") + return nil + } + mc.p.commitHook = func(int) error { + if _, err := mc.s.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, 2*time.Hour, ""); !errors.Is(err, ErrEvacuating) { + t.Errorf("renew during the final round: %v, want ErrEvacuating", err) + } + if err := mc.s.PatchEnv(t.Context(), sb.ID, types.EnvPatch{"GH_TOKEN": {Value: "final", Guest: hostOnly}}, ""); !errors.Is(err, ErrEvacuating) { + t.Errorf("env write during the final round: %v, want ErrEvacuating", err) + } + return nil + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + moved, ok := mc.t.byID(sb.ID) + if !ok || !moved.Deadline.Equal(deadline) || moved.Env["GH_TOKEN"].Value != "rounds" { + t.Errorf("target claim deadline %v env %v, want the deadline %v and the env of the rounds", moved.Deadline, moved.Env, deadline) + } +} + func TestMoveRefusesClaimsThatCannotMove(t *testing.T) { mc := newMoveCluster(t) hibernatedSb := mustClaim(t, mc.s, testKey) From 708516cc562f246a1b4fb3ae012e1cd7560d5f5e Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:03:54 +0800 Subject: [PATCH 18/53] pool: a reservation with a round arriving is not idle, and a round that outlives its reservation fails The reservation was touched only around a round, so a round over two minutes lost it mid-stream. The second touch ignored the loss, so the round answered 204 and left files in a staging dir nothing owned, and the commit failed after the guest paused. --- docs/cluster.md | 4 +- sandboxd/pool/move_in.go | 38 ++++++++++++++---- sandboxd/pool/move_test.go | 82 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 8d574c6c..333d9247 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -382,8 +382,8 @@ never see it. At the commit the reservation becomes the claim under one lock, so the target never counts it twice. The source counts the claim until it drops it after the commit, so the cluster counts a moving claim twice, which can make a node refuse earlier but never over-admit. `memory_limit` is checked again -before the receive. A reservation with no round or commit for 2 minutes is -dropped with its staged files. +before the receive. A reservation with no round arriving and none finished or +commit asked for 2 minutes is dropped with its staged files. **Replicated claims.** Before the final round the source deletes the claim's registry row and its replica, so no third node can adopt a stale replica while diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index 4e0dfdfb..65e52e55 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -21,9 +21,10 @@ const reasonMoveStartFailed = "vmm did not start after a live move" // inMove is a target's reservation for a move in flight; guarded by m.mu. type inMove struct { - tenant string - from string - touched time.Time + tenant string + from string + touched time.Time + receiving int } // AdmitMove reserves a claim slot for a live move of id to this node and returns this node's id. @@ -53,13 +54,18 @@ func (m *Manager) AdmitMove(id string, a MoveAdmission) (string, error) { // ReceiveMoveRound writes one round of a reserved move into its staging dir. func (m *Manager) ReceiveMoveRound(id string, body io.Reader) error { - if !m.touchReservation(id) { + in := m.beginReceive(id) + if in == nil { + return ErrUnknownSandbox + } + err := peer.ReceiveRound(body, m.moveDir(id)) + if !m.endReceive(id, in) { + _ = os.RemoveAll(m.moveDir(id)) return ErrUnknownSandbox } - if err := peer.ReceiveRound(body, m.moveDir(id)); err != nil { + if err != nil { return fmt.Errorf("move %s: %w", id, err) } - m.touchReservation(id) return nil } @@ -231,6 +237,24 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { return true } +func (m *Manager) beginReceive(id string) *inMove { + m.mu.Lock() + defer m.mu.Unlock() + in := m.incoming[id] + if in != nil { + in.receiving++ + } + return in +} + +func (m *Manager) endReceive(id string, in *inMove) bool { + m.mu.Lock() + defer m.mu.Unlock() + in.receiving-- + in.touched = time.Now() + return m.incoming[id] == in +} + func (m *Manager) touchReservation(id string) bool { m.mu.Lock() defer m.mu.Unlock() @@ -247,7 +271,7 @@ func (m *Manager) dropIdleReservations() { var idle []string m.mu.Lock() for id, in := range m.incoming { - if now.Sub(in.touched) > moveReservationIdle { + if in.receiving == 0 && now.Sub(in.touched) > moveReservationIdle { idle = append(idle, id) } } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index dd2eed05..45568299 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -1,9 +1,11 @@ package pool import ( + "bytes" "cmp" "context" "errors" + "io" "net" "net/http" "net/http/httptest" @@ -227,13 +229,74 @@ func TestMoveTargetCountsAReservationLikeAClaim(t *testing.T) { func TestMoveTargetDropsAReservationItsSourceAbandoned(t *testing.T) { mc := newMoveCluster(t) reserveMove(t, mc.t, "sb_r", "acme") - mc.t.mu.Lock() - mc.t.incoming["sb_r"].touched = time.Now().Add(-2 * moveReservationIdle) - mc.t.mu.Unlock() + backdateReservation(mc.t, "sb_r") + mc.t.dropIdleReservations() + assertNoMoveResidue(t, mc.t, "sb_r") +} + +func TestMoveTargetKeepsTheReservationOfARoundStillArriving(t *testing.T) { + mc := newMoveCluster(t) + reserveMove(t, mc.t, "sb_r", "acme") + body, stream := io.Pipe() + done := make(chan error, 1) + go func() { done <- mc.t.ReceiveMoveRound("sb_r", body) }() + waitFor(t, func() bool { + mc.t.mu.Lock() + defer mc.t.mu.Unlock() + return mc.t.incoming["sb_r"].receiving == 1 + }) + backdateReservation(mc.t, "sb_r") + mc.t.dropIdleReservations() + if _, reserved := reservationOf(mc.t, "sb_r"); !reserved { + t.Fatal("the idle drop took a reservation with a round still arriving") + } + _ = stream.CloseWithError(errors.New("source hung up")) + if err := <-done; err == nil { + t.Fatal("a cut round was accepted") + } + if _, reserved := reservationOf(mc.t, "sb_r"); !reserved { + t.Fatal("a cut round took the reservation") + } + backdateReservation(mc.t, "sb_r") mc.t.dropIdleReservations() assertNoMoveResidue(t, mc.t, "sb_r") } +func TestMoveTargetRefusesARoundThatOutlivedItsReservation(t *testing.T) { + mc := newMoveCluster(t) + reserveMove(t, mc.t, "sb_r", "acme") + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "state.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + var captured bytes.Buffer + capture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(&captured, r.Body) + w.WriteHeader(http.StatusNoContent) + })) + defer capture.Close() + if err := peer.SendRound(t.Context(), capture.URL, "", "/", src, []string{"state.json"}); err != nil { + t.Fatalf("capture a round: %v", err) + } + body, stream := io.Pipe() + done := make(chan error, 1) + go func() { done <- mc.t.ReceiveMoveRound("sb_r", body) }() + waitFor(t, func() bool { + mc.t.mu.Lock() + defer mc.t.mu.Unlock() + return mc.t.incoming["sb_r"].receiving == 1 + }) + mc.t.DropMove("sb_r") + go func() { + _, _ = stream.Write(captured.Bytes()) + _ = stream.Close() + }() + if err := <-done; !errors.Is(err, ErrUnknownSandbox) { + t.Fatalf("round over a dropped reservation: %v, want ErrUnknownSandbox", err) + } + assertNoMoveResidue(t, mc.t, "sb_r") +} + func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) third := mc.moveNode(t, "c") @@ -788,6 +851,19 @@ func seedMoveRecord(t *testing.T, reg moves.Registry, id, from, to string, state return epoch } +func backdateReservation(m *Manager, id string) { + m.mu.Lock() + defer m.mu.Unlock() + m.incoming[id].touched = time.Now().Add(-2 * moveReservationIdle) +} + +func reservationOf(m *Manager, id string) (*inMove, bool) { + m.mu.Lock() + defer m.mu.Unlock() + in, ok := m.incoming[id] + return in, ok +} + func startsOf(m *Manager) []string { eng := fakeOf(m) eng.mu.Lock() From 39695df8ca5780936467ee50a99e4dd4383c0d86 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:04:22 +0800 Subject: [PATCH 19/53] pool: a move whose claim a failover took ends without an abort of the removed VM MigrateAbort failed on the VM the failover removed, so the retry loop never ended and kept a budget slot and Transition. With the claim gone there is no guest to resume. --- sandboxd/pool/move_out.go | 8 ++++---- sandboxd/pool/move_test.go | 25 +++++++++++++++++++++++++ sandboxd/pool/pool_test.go | 3 +++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 695ce332..ab5b8319 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -311,10 +311,10 @@ func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM // abortMoveOut ends the series, which resumes a paused guest, releases the target and the budget slot, and deletes the aborted record only once the journal no longer needs it. func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, aborted int64) error { logger := log.WithFunc("pool.abortMoveOut") - if mv.series != "" { - m.mu.Lock() - vm := sb.VMName - m.mu.Unlock() + m.mu.Lock() + vm, live := sb.VMName, m.claimed[sb.ID] == sb + m.mu.Unlock() + if live && mv.series != "" { if err := m.eng.MigrateAbort(ctx, vm, mv.series); err != nil { return fmt.Errorf("abort series %s: %w", mv.series, err) } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 45568299..fd08c67d 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -373,6 +373,31 @@ func TestMoveHoldsOffCapturesReleasesAndEvacuationWhileItRuns(t *testing.T) { } } +func TestMoveEndsWhenAFailoverTookTheClaimFromTheSource(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + mc.p.commitHook = func(int) error { + mc.s.dropAdopted(t.Context(), sb) + return ErrMoveTarget + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveTarget) { + t.Fatalf("move of a claim another node took: %v, want ErrMoveTarget", err) + } + if mc.s.movingNow(sb.ID) { + t.Fatal("the move still holds its slot") + } + if !sb.Transition.TryLock() { + t.Fatal("the move still holds Transition") + } + sb.Transition.Unlock() + eng := fakeOf(mc.s) + eng.mu.Lock() + defer eng.mu.Unlock() + if len(eng.aborts) != 0 { + t.Errorf("aborts %v on a removed VM", eng.aborts) + } +} + func TestMoveRefusesWritesThatWouldMissTheRecordTheTargetReceives(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 8a5eb22d..d6590938 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1907,6 +1907,9 @@ func (f *fakeEngine) MigrateAbort(_ context.Context, vmName, series string) erro if f.abortErr != nil { return f.abortErr } + if _, ok := f.vms[vmName]; !ok { + return fmt.Errorf("vm %s not found", vmName) + } if f.migrateSeries[vmName] == series { delete(f.migrateSeries, vmName) f.paused[vmName] = false From 317d4fb924636620bca368499e425c72744d457c Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:04:50 +0800 Subject: [PATCH 20/53] pool: each round of a move has a deadline from its size A target that stopped reading kept a non-final round waiting until TCP gave up, with the source's slot held. A round now has one minute plus its size at 10 MB/s, and the final round stays under the two minute budget of the final phase. --- docs/cluster.md | 3 ++- sandboxd/pool/move.go | 2 ++ sandboxd/pool/move_out.go | 6 ++++++ sandboxd/pool/move_test.go | 25 +++++++++++++++++++++++++ sandboxd/store/peer/move_test.go | 19 +++++++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/cluster.md b/docs/cluster.md index 333d9247..46b88868 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -370,7 +370,8 @@ the rounds reached, plus 400 ms for the receive and the start, fits round and the target receives the series and starts the guest. A move whose deltas never fit answers 503 and the guest runs on. `downtime_ms` in the answer is the pause the source measured, from the final capture to the target's -commit. The final capture, its transfer and the target's commit have 2 minutes +commit. Each round has one minute plus its size at 10 MB/s to arrive, so a +target that stops reading fails the move. The final capture, its transfer and the target's commit have 2 minutes together; past that the source resolves the move through the record, and the guest resumes on the source unless the target committed. diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 3c2372e9..75e2e1ee 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -25,6 +25,8 @@ const ( moveStartAllowance = 400 * time.Millisecond moveStaleAge = time.Hour moveFinalTimeout = 2 * time.Minute + moveRoundBase = time.Minute + moveRoundMinRate = 10_000_000 ) // MoveAdmission is what a source asks a target to reserve for a live move. diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index ab5b8319..ecd1e594 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -138,6 +138,8 @@ func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round t for i, f := range round.Files { names[i] = f.Name } + ctx, cancel := context.WithTimeout(ctx, roundTimeout(roundBytes(round))) + defer cancel() return m.movePeer.SendRound(ctx, mv.addr, id, mv.rounds, round.Dir, names) } @@ -377,6 +379,10 @@ func roundBytes(round types.MigrateRound) int64 { return n } +func roundTimeout(size int64) time.Duration { + return moveRoundBase + time.Duration(size/moveRoundMinRate)*time.Second +} + // finalPause estimates the final round's pause from the last delta's size at the rate the rounds so far reached. func finalPause(last, sent int64, sending time.Duration) time.Duration { if sent <= 0 { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index fd08c67d..eb6b4d26 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -373,6 +373,27 @@ func TestMoveHoldsOffCapturesReleasesAndEvacuationWhileItRuns(t *testing.T) { } } +func TestMoveGivesEachRoundADeadlineFromItsSize(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + fakeOf(mc.s).migrateSizes = []int64{1 << 30, 1, 1 << 40} + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Hour); err != nil { + t.Fatalf("move: %v", err) + } + if len(mc.p.roundBudgets) != 3 { + t.Fatalf("round budgets %v, want three rounds", mc.p.roundBudgets) + } + for i, within := range [][2]time.Duration{ + {moveRoundBase + 100*time.Second, moveRoundBase + 107*time.Second}, + {moveRoundBase - time.Second, moveRoundBase}, + {moveFinalTimeout - 5*time.Second, moveFinalTimeout}, + } { + if got := mc.p.roundBudgets[i]; got <= within[0] || got > within[1] { + t.Errorf("round %d budget %s, want within %v", i+1, got, within) + } + } +} + func TestMoveEndsWhenAFailoverTookTheClaimFromTheSource(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) @@ -755,6 +776,7 @@ type fakeMovePeer struct { commitHook func(round int) error loseCommitReply bool dropHook func() + roundBudgets []time.Duration } func newFakeMovePeer(t *testing.T, c *failoverCluster) *fakeMovePeer { @@ -780,6 +802,9 @@ func (p *fakeMovePeer) Admit(_ context.Context, addr, id string, a MoveAdmission } func (p *fakeMovePeer) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error { + if deadline, ok := ctx.Deadline(); ok { + p.roundBudgets = append(p.roundBudgets, time.Until(deadline)) + } if hook := p.roundHook; hook != nil { if err := hook(round); err != nil { return err diff --git a/sandboxd/store/peer/move_test.go b/sandboxd/store/peer/move_test.go index ffd87b01..4452507d 100644 --- a/sandboxd/store/peer/move_test.go +++ b/sandboxd/store/peer/move_test.go @@ -3,6 +3,7 @@ package peer import ( "archive/tar" "bytes" + "context" "errors" "io" "net/http" @@ -13,6 +14,7 @@ import ( "strings" "syscall" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/store/storetest" @@ -216,6 +218,23 @@ func TestSendRoundFailsOnARefusedPutOrAMissingFile(t *testing.T) { } } +func TestSendRoundFailsWhenATargetStopsReading(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "memory-ranges"), bytes.Repeat([]byte("guest-memory-"), 4<<20), 0o600); err != nil { + t.Fatal(err) + } + stop := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-stop })) + defer srv.Close() + defer close(stop) + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + if err := SendRound(ctx, srv.URL, "", "/r", src, []string{"memory-ranges"}); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("round to a target that stopped reading: %v, want the deadline", err) + } +} + func roundStream(t *testing.T, dir string, names ...string) []byte { t.Helper() var buf bytes.Buffer From e3f8dc5ccb493b67df67ca36a70db4c81c2d2a7b Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:26:26 +0800 Subject: [PATCH 21/53] pool: resolve journaled moves only after Reconcile's last read of the claims resumeMoves started goroutines that drop claims while Reconcile still read m.claimed without its lock, which the race detector caught under load. Resuming last keeps Reconcile's pass single threaded. The restart test now waits for the VM removal that follows the claim's drop, and lossyMoves keeps its unexported method below the exported set. --- sandboxd/pool/move_test.go | 7 ++++--- sandboxd/pool/reconcile.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index eb6b4d26..8113465a 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -545,7 +545,8 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { } waitFor(t, func() bool { rec, ok, _ := reg.Get(ctx, sb.ID) - return !m.moveHeld(sb.ID) && (ok && rec.State == tt.wantRecord || !ok && tt.wantRecord == "") + settled := !m.moveHeld(sb.ID) && (tt.wantClaim || eng.removed(vm)) + return settled && (ok && rec.State == tt.wantRecord || !ok && tt.wantRecord == "") }) held, ok := m.byID(sb.ID) if ok != tt.wantClaim { @@ -850,8 +851,6 @@ type lossyMoves struct { down atomic.Bool } -func (r *lossyMoves) setDown(down bool) { r.down.Store(down) } - func (r *lossyMoves) Begin(ctx context.Context, id, from, to string) (int64, error) { if r.down.Load() { return 0, moves.ErrUnavailable @@ -878,6 +877,8 @@ func (r *lossyMoves) Get(ctx context.Context, id string) (moves.Record, bool, er return r.Registry.Get(ctx, id) } +func (r *lossyMoves) setDown(down bool) { r.down.Store(down) } + func seedMoveRecord(t *testing.T, reg moves.Registry, id, from, to string, state moves.State) int64 { t.Helper() ctx := t.Context() diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 7189f9d3..b7a49b60 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -135,9 +135,9 @@ func (m *Manager) Reconcile(ctx context.Context) error { }).Wait() } m.resyncEgress(ctx, live, removed) - m.resumeMoves(context.WithoutCancel(ctx), outs, ins) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) + m.resumeMoves(context.WithoutCancel(ctx), outs, ins) return saveErr } From 59ee5c5f0179d070ed1cfe7459869a1118cba144 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:35:14 +0800 Subject: [PATCH 22/53] server: a max_downtime_ms at or under the start allowance answers 400 at once Such a budget can never fit, so an idle guest copied for 21 s on hardware before the 503. --- sandboxd/pool/move.go | 6 +++--- sandboxd/pool/move_out.go | 4 ++-- sandboxd/server/move.go | 6 +++--- sandboxd/server/move_test.go | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 75e2e1ee..18f1d3ef 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -15,15 +15,15 @@ import ( const ( // DefaultMoveDowntime is the final round's pause a move allows when the caller names none. DefaultMoveDowntime = 2 * time.Second + // MoveStartAllowance is what the target's receive and start add to the final round's transfer, so a smaller downtime budget never fits. + MoveStartAllowance = 400 * time.Millisecond movesDir = "moves" // A reservation with no round or commit for this long belongs to a source that gave up. moveReservationIdle = 2 * time.Minute moveRetryInterval = 2 * time.Second moveDeltaLimit = 4 - // Receive and vm start on the target add this to the final round's transfer. - moveStartAllowance = 400 * time.Millisecond - moveStaleAge = time.Hour + moveStaleAge = time.Hour moveFinalTimeout = 2 * time.Minute moveRoundBase = time.Minute moveRoundMinRate = 10_000_000 diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index ecd1e594..3f8b49cf 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -386,7 +386,7 @@ func roundTimeout(size int64) time.Duration { // finalPause estimates the final round's pause from the last delta's size at the rate the rounds so far reached. func finalPause(last, sent int64, sending time.Duration) time.Duration { if sent <= 0 { - return moveStartAllowance + return MoveStartAllowance } - return moveStartAllowance + time.Duration(float64(sending)*float64(last)/float64(sent)) + return MoveStartAllowance + time.Duration(float64(sending)*float64(last)/float64(sent)) } diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go index 682ed1e0..2ef6ab83 100644 --- a/sandboxd/server/move.go +++ b/sandboxd/server/move.go @@ -122,8 +122,9 @@ func (s *Server) handleMove(w http.ResponseWriter, r *http.Request) { if !ok { return } - if req.MaxDowntimeMS < 0 { - writeErr(w, http.StatusBadRequest, "max_downtime_ms must not be negative") + budget := time.Duration(req.MaxDowntimeMS) * time.Millisecond + if req.MaxDowntimeMS != 0 && budget <= pool.MoveStartAllowance { + writeErr(w, http.StatusBadRequest, fmt.Sprintf("max_downtime_ms must be above %d: the target's receive and start alone take that long", pool.MoveStartAllowance.Milliseconds())) return } addr, ok := s.movePeerAddr(req.To) @@ -132,7 +133,6 @@ func (s *Server) handleMove(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - budget := time.Duration(req.MaxDowntimeMS) * time.Millisecond res, err := s.mgr.Move(r.Context(), id, addr, budget) writeResult(w, r, "move", id, "move failed", err, func() { writeJSON(w, http.StatusOK, MoveResponse{OwnerAddr: s.placer.ClientAddr(addr), Rounds: res.Rounds, DowntimeMS: res.Downtime.Milliseconds()}) diff --git a/sandboxd/server/move_test.go b/sandboxd/server/move_test.go index 8ccd480d..15237d4c 100644 --- a/sandboxd/server/move_test.go +++ b/sandboxd/server/move_test.go @@ -4,6 +4,7 @@ import ( "encoding/json/v2" "errors" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -39,6 +40,8 @@ func TestMoveRouteResolvesThePeerAndMapsTheOutcome(t *testing.T) { }{ {"root", `{"to":"10.0.0.9:7777"}`, nil, http.StatusBadRequest}, {"root", `{"to":"10.0.0.2:7777","max_downtime_ms":-1}`, nil, http.StatusBadRequest}, + {"root", `{"to":"10.0.0.2:7777","max_downtime_ms":400}`, nil, http.StatusBadRequest}, + {"root", `{"to":"10.0.0.2:7777","max_downtime_ms":401}`, nil, http.StatusOK}, {"root", `{"to":"10.0.0.2:7777"}`, pool.ErrUnknownSandbox, http.StatusNotFound}, {"root", `{"to":"10.0.0.2:7777"}`, fmt.Errorf("%w: it has volumes", pool.ErrNotMovable), http.StatusConflict}, {"root", `{"to":"10.0.0.2:7777"}`, fmt.Errorf("%w: target: 3 moves in flight", pool.ErrQuota), http.StatusTooManyRequests}, @@ -56,6 +59,24 @@ func TestMoveRouteResolvesThePeerAndMapsTheOutcome(t *testing.T) { } } +func TestMoveRouteRefusesABudgetTheStartAllowanceAloneExceedsBeforeAnyMoveStarts(t *testing.T) { + calls := 0 + mgr := &fakeManager{move: func(string, string, time.Duration) (pool.MoveResult, error) { + calls++ + return pool.MoveResult{}, nil + }} + srv := newMeshTestServer(t, "root", mgr, &fakePlacer{addrs: []string{"10.0.0.2:7777"}}, nil) + resp := doReq(t, http.MethodPost, srv.URL+"/v1/sandboxes/sb_1/move", "root", `{"to":"10.0.0.2:7777","max_downtime_ms":400}`) + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil || resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(body), "above 400") { + t.Errorf("answer %d %s %v, want a 400 naming the floor", resp.StatusCode, body, err) + } + if calls != 0 { + t.Errorf("the manager saw %d moves, want none", calls) + } +} + func TestMoveClientPassesTheTargetsRefusalThrough(t *testing.T) { status, reason := http.StatusOK, "" var gotAuth, gotPath string From 1b9bc10970b31eaa4a9d9977c7404898733664de Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:36:13 +0800 Subject: [PATCH 23/53] pool: a retried admission from the same source replaces its reservation After a source crash the move answered 409 until the reservation idled out: 409 at +13 s and 200 at +183 s on hardware. A source runs one move of a claim at a time, so an admission with the same from replaces the reservation and resets the staging dir. A different from still answers 409. Replacement lets an old stream or commit outlive its reservation, so a late round no longer removes the retry's staging dir, and a commit swaps only the reservation it started from. --- sandboxd/pool/move_in.go | 43 +++++++++---------- sandboxd/pool/move_test.go | 86 ++++++++++++++++++++++++++++++++++++++ sandboxd/pool/pool_test.go | 6 +++ 3 files changed, 112 insertions(+), 23 deletions(-) diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index 65e52e55..830e95eb 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -59,8 +59,10 @@ func (m *Manager) ReceiveMoveRound(id string, body io.Reader) error { return ErrUnknownSandbox } err := peer.ReceiveRound(body, m.moveDir(id)) - if !m.endReceive(id, in) { - _ = os.RemoveAll(m.moveDir(id)) + if current, reserved := m.endReceive(id, in); !current { + if !reserved { + _ = os.RemoveAll(m.moveDir(id)) + } return ErrUnknownSandbox } if err != nil { @@ -72,9 +74,11 @@ func (m *Manager) ReceiveMoveRound(id string, body io.Reader) error { // CommitMove receives the staged series as the claim, commits the move record and starts the guest. func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch int64) error { ctx = context.WithoutCancel(ctx) - if !m.touchReservation(id) { + admitted := m.beginReceive(id) + if admitted == nil { return fmt.Errorf("%w: no reservation for the move of %s", ErrNotMovable, id) } + defer m.endReceive(id, admitted) if m.memoryLow.Load() { return fmt.Errorf("%w: %s", ErrQuota, memoryLowReason) } @@ -95,13 +99,12 @@ func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - in := m.incoming[id] - if in == nil || m.claimed[id] != nil { + if m.incoming[id] != admitted || m.claimed[id] != nil { m.mu.Unlock() m.destroy(ctx, name) return fmt.Errorf("%w: the reservation for %s ended", ErrNotMovable, id) } - sb.MoveIn = &types.MoveIn{From: in.from, Epoch: epoch} + sb.MoveIn = &types.MoveIn{From: admitted.from, Epoch: epoch} delete(m.incoming, id) m.addClaimLocked(sb) js := m.store.set(sb) @@ -247,22 +250,13 @@ func (m *Manager) beginReceive(id string) *inMove { return in } -func (m *Manager) endReceive(id string, in *inMove) bool { +func (m *Manager) endReceive(id string, in *inMove) (current, reserved bool) { m.mu.Lock() defer m.mu.Unlock() in.receiving-- in.touched = time.Now() - return m.incoming[id] == in -} - -func (m *Manager) touchReservation(id string) bool { - m.mu.Lock() - defer m.mu.Unlock() - in := m.incoming[id] - if in != nil { - in.touched = time.Now() - } - return in != nil + held := m.incoming[id] + return held == in, held != nil } // dropIdleReservations ends the reservations of sources that went quiet. @@ -297,13 +291,16 @@ func (m *Manager) admitMoveLocked(id string, a MoveAdmission) error { if _, held := m.claimed[id]; held { return fmt.Errorf("%w: this node already holds %s", ErrNotMovable, id) } - if _, held := m.incoming[id]; held { - return fmt.Errorf("%w: a move of %s to this node is already running", ErrNotMovable, id) - } - if err := m.moveBudgetErr(); err != nil { + extra := 1 + if in, held := m.incoming[id]; held { + if in.from != a.From { + return fmt.Errorf("%w: a move of %s to this node is already running", ErrNotMovable, id) + } + extra = 0 + } else if err := m.moveBudgetErr(); err != nil { return err } - if err := m.quotaErr(1, a.Tenant); err != nil { + if err := m.quotaErr(extra, a.Tenant); err != nil { return err } m.incoming[id] = &inMove{tenant: a.Tenant, from: a.From, touched: time.Now()} diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 8113465a..15cea0dc 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -297,6 +297,92 @@ func TestMoveTargetRefusesARoundThatOutlivedItsReservation(t *testing.T) { assertNoMoveResidue(t, mc.t, "sb_r") } +func TestMoveTargetLetsTheSameSourceRetryAndRefusesAnother(t *testing.T) { + mc := newMoveCluster(t) + tuneView(mc.t, func(v *configView) { v.moveConcurrency, v.maxClaims = 1, 1 }) + reserveMove(t, mc.t, "sb_r", "acme") + first, _ := reservationOf(mc.t, "sb_r") + stale := filepath.Join(mc.t.moveDir("sb_r"), "state.json") + if err := os.WriteFile(stale, []byte("a crashed source's round"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := mc.t.AdmitMove("sb_r", MoveAdmission{Tenant: "acme", Key: testKey, From: "other"}); !errors.Is(err, ErrNotMovable) { + t.Errorf("admission from another source: %v, want ErrNotMovable", err) + } + if _, err := os.Stat(stale); err != nil { + t.Errorf("a refused admission touched the staging dir: %v", err) + } + reserveMove(t, mc.t, "sb_r", "acme") + if again, _ := reservationOf(mc.t, "sb_r"); again == first { + t.Error("the retry kept the crashed source's reservation") + } + if _, err := os.Stat(stale); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the retry kept the crashed source's files: %v", err) + } + if got := moveCounts(mc.t, "acme"); got != [3]int{1, 1, 1} { + t.Errorf("target counts %v after the retry, want one reservation", got) + } +} + +func TestMoveTargetKeepsARetrysStagingDirWhenTheCrashedSourcesRoundEnds(t *testing.T) { + mc := newMoveCluster(t) + reserveMove(t, mc.t, "sb_r", "acme") + body, stream := io.Pipe() + done := make(chan error, 1) + go func() { done <- mc.t.ReceiveMoveRound("sb_r", body) }() + waitFor(t, func() bool { + in, _ := reservationOf(mc.t, "sb_r") + mc.t.mu.Lock() + defer mc.t.mu.Unlock() + return in.receiving == 1 + }) + reserveMove(t, mc.t, "sb_r", "acme") + retried := filepath.Join(mc.t.moveDir("sb_r"), "state.json") + if err := os.WriteFile(retried, []byte("the retry's round"), 0o600); err != nil { + t.Fatal(err) + } + _ = stream.CloseWithError(errors.New("the crashed source's connection ended")) + if err := <-done; !errors.Is(err, ErrUnknownSandbox) { + t.Fatalf("the old round: %v, want ErrUnknownSandbox", err) + } + if _, err := os.Stat(retried); err != nil { + t.Errorf("the old round removed the retry's files: %v", err) + } +} + +func TestMoveCommitOfACrashedSourceDoesNotTakeTheRetrysReservation(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + eng := fakeOf(mc.t) + eng.receiveEntered, eng.receiveStall = make(chan struct{}), make(chan struct{}) + reserveMove(t, mc.t, sb.ID, "") + done := make(chan error, 1) + go func() { done <- mc.t.CommitMove(context.WithoutCancel(t.Context()), sb.ID, []byte(`{"id":"`+sb.ID+`","key":{"template":"rt:24.04","net":"none","size":"small"}}`), 1) }() + <-eng.receiveEntered + reserveMove(t, mc.t, sb.ID, "") + retry, _ := reservationOf(mc.t, sb.ID) + if err := os.WriteFile(filepath.Join(mc.t.moveDir(sb.ID), "state.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + close(eng.receiveStall) + if err := <-done; !errors.Is(err, ErrNotMovable) { + t.Fatalf("the crashed source's commit: %v, want ErrNotMovable", err) + } + if held, ok := reservationOf(mc.t, sb.ID); !ok || held != retry { + t.Error("the crashed source's commit took the retry's reservation") + } + if _, claimed := mc.t.byID(sb.ID); claimed { + t.Error("the crashed source's commit made a claim") + } + eng.mu.Lock() + defer eng.mu.Unlock() + for _, name := range eng.receives { + if _, live := eng.vms[name]; live { + t.Errorf("the crashed source's received VM %s survived", name) + } + } +} + func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) third := mc.moveNode(t, "c") diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index d6590938..bf00c0ab 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1349,6 +1349,8 @@ type fakeEngine struct { captureErr error finalCaptureErr error receiveErr error + receiveStall chan struct{} + receiveEntered chan struct{} abortErr error } @@ -1886,6 +1888,10 @@ func (f *fakeEngine) MigrateCapture(_ context.Context, vmName, series string, fi } func (f *fakeEngine) MigrateReceive(_ context.Context, fromDir, name string, _ types.PoolKey) (types.VMRecord, error) { + if f.receiveEntered != nil { + close(f.receiveEntered) + <-f.receiveStall + } f.mu.Lock() defer f.mu.Unlock() f.receives = append(f.receives, name) From 8270cbb313392bd820610d0a1bcb4ae75b2551e4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:36:43 +0800 Subject: [PATCH 24/53] pool: a move resolved through its record reports the guest's pause When the commit answer was lost the 200 carried downtime_ms 0 on hardware. The pause start now lives on the move, and both paths read it at the moment the target's answer came back or was lost. --- sandboxd/pool/move.go | 31 +++++++++++++++-------- sandboxd/pool/move_out.go | 51 +++++++++++++++++++------------------- sandboxd/pool/move_test.go | 6 ++++- 3 files changed, 50 insertions(+), 38 deletions(-) diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 18f1d3ef..0d09ddcb 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -23,10 +23,10 @@ const ( moveReservationIdle = 2 * time.Minute moveRetryInterval = 2 * time.Second moveDeltaLimit = 4 - moveStaleAge = time.Hour - moveFinalTimeout = 2 * time.Minute - moveRoundBase = time.Minute - moveRoundMinRate = 10_000_000 + moveStaleAge = time.Hour + moveFinalTimeout = 2 * time.Minute + moveRoundBase = time.Minute + moveRoundMinRate = 10_000_000 ) // MoveAdmission is what a source asks a target to reserve for a live move. @@ -54,13 +54,22 @@ type MovePeer interface { // outMove is a source's move in flight; final is guarded by m.mu, the rest belongs to the goroutine driving the move. type outMove struct { - addr string - to string - epoch int64 - series string - rounds int - final bool - locked bool + addr string + to string + epoch int64 + series string + rounds int + final bool + locked bool + pausedAt time.Time +} + +func (mv *outMove) result() MoveResult { + res := MoveResult{Rounds: mv.rounds} + if !mv.pausedAt.IsZero() { + res.Downtime = time.Since(mv.pausedAt) + } + return res } // WithMovePeer wires the transport a move uses to reach its target; set it before serving. diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 3f8b49cf..9c5f7147 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -36,42 +36,43 @@ func (m *Manager) Move(ctx context.Context, id, addr string, maxDowntime time.Du return MoveResult{}, err } ctx = context.WithoutCancel(ctx) - downtime, err := m.moveOut(ctx, sb, mv, cmp.Or(maxDowntime, DefaultMoveDowntime)) - if err == nil { - m.finishMoveOut(ctx, sb, mv) - return MoveResult{Rounds: mv.rounds, Downtime: downtime}, nil + err := m.moveOut(ctx, sb, mv, cmp.Or(maxDowntime, DefaultMoveDowntime)) + if err != nil { + return m.failMoveOut(ctx, sb, mv, err) } - return m.failMoveOut(ctx, sb, mv, err) + result := mv.result() + m.finishMoveOut(ctx, sb, mv) + return result, nil } // moveOut runs the rounds up to the target's commit; on any error the caller resolves the move through its record. -func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, budget time.Duration) (time.Duration, error) { +func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, budget time.Duration) error { m.mu.Lock() admission, vm := MoveAdmission{Tenant: sb.Tenant, Key: sb.Key, From: m.nodeID}, sb.VMName m.mu.Unlock() to, err := m.movePeer.Admit(ctx, mv.addr, sb.ID, admission) if err != nil { - return 0, err + return err } mv.to = to epoch, err := m.moves.Begin(ctx, sb.ID, m.nodeID, to) if errors.Is(err, moves.ErrLost) { - return 0, fmt.Errorf("%w: another move of %s is on record", ErrNotMovable, sb.ID) + return fmt.Errorf("%w: another move of %s is on record", ErrNotMovable, sb.ID) } if err != nil { - return 0, err + return err } mv.epoch = epoch if err = m.journalMoveOut(ctx, sb, mv); err != nil { - return 0, err + return err } round, err := m.eng.MigrateCapture(ctx, vm, "", false) if err != nil { - return 0, fmt.Errorf("%w: baseline capture: %w", ErrNotMovable, err) + return fmt.Errorf("%w: baseline capture: %w", ErrNotMovable, err) } mv.series = round.Series if err = m.journalMoveOut(ctx, sb, mv); err != nil { - return 0, err + return err } var sent int64 var sending time.Duration @@ -79,7 +80,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b for i := 0; ; i++ { start := time.Now() if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { - return 0, err + return err } sending += time.Since(start) size := roundBytes(round) @@ -91,11 +92,11 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b break } if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, false); err != nil { - return 0, fmt.Errorf("delta capture: %w", err) + return fmt.Errorf("delta capture: %w", err) } } if !fits { - return 0, fmt.Errorf("%w: %d deltas did not shrink under %s", ErrMoveDowntime, moveDeltaLimit, budget) + return fmt.Errorf("%w: %d deltas did not shrink under %s", ErrMoveDowntime, moveDeltaLimit, budget) } m.mu.Lock() mv.final = true @@ -105,18 +106,18 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b ctx, cancel := context.WithTimeout(ctx, moveFinalTimeout) defer cancel() if err = m.unrowForMove(ctx, sb); err != nil { - return 0, err + return err } - pausedAt := time.Now() + mv.pausedAt = time.Now() if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, true); err != nil { - return 0, fmt.Errorf("final capture: %w", err) + return fmt.Errorf("final capture: %w", err) } final, err := m.moves.Advance(ctx, sb.ID, mv.epoch, moves.Pending, moves.Final) if err != nil { - return 0, err + return err } if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { - return 0, err + return err } m.mu.Lock() dto := dtoOf(sb) @@ -124,12 +125,9 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b dto.MoveOut = nil claim, err := json.Marshal(&dto) if err != nil { - return 0, fmt.Errorf("encode claim: %w", err) - } - if err = m.movePeer.Commit(ctx, mv.addr, sb.ID, claim, final); err != nil { - return 0, err + return fmt.Errorf("encode claim: %w", err) } - return time.Since(pausedAt), nil + return m.movePeer.Commit(ctx, mv.addr, sb.ID, claim, final) } func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round types.MigrateRound) error { @@ -261,6 +259,7 @@ func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMov _ = m.abortMoveOut(ctx, sb, mv, 0) return MoveResult{}, cause } + result := mv.result() moved, err := m.settleMoveOut(ctx, sb, mv) switch { case err != nil: @@ -268,7 +267,7 @@ func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMov go m.retryMoveOut(ctx, sb, mv) return MoveResult{}, fmt.Errorf("%w: %w", ErrMoveStoreDown, cause) case moved: - return MoveResult{Rounds: mv.rounds}, nil + return result, nil } return MoveResult{}, cause } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 15cea0dc..3cb7090c 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -170,9 +170,13 @@ func TestMoveWhoseCommitAnswerIsLostResolvesThroughTheRecord(t *testing.T) { mc := newMoveCluster(t) sb := claimTenant(t, mc.s, "acme") mc.p.loseCommitReply = true - if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + res, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute) + if err != nil { t.Fatalf("move whose commit answer was lost: %v, want the committed move", err) } + if res.Rounds != 3 || res.Downtime <= 0 { + t.Errorf("result %+v, want the three rounds and the pause", res) + } if _, held := mc.s.byID(sb.ID); held { t.Error("the source kept a claim the target committed") } From 8e6765148c6ea4363ce19f0f4913f4df0b95c3a7 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:37:13 +0800 Subject: [PATCH 25/53] pool: a test for a target killed between its commit and its start On hardware a SIGKILL of the target right after its commit lost the claim 2 of 2 while the call answered 200. The test covers that sequence: the source finishes, the record stays committed, and the restarted target starts the guest. --- sandboxd/pool/move_test.go | 61 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 3cb7090c..6840ca18 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -361,7 +362,9 @@ func TestMoveCommitOfACrashedSourceDoesNotTakeTheRetrysReservation(t *testing.T) eng.receiveEntered, eng.receiveStall = make(chan struct{}), make(chan struct{}) reserveMove(t, mc.t, sb.ID, "") done := make(chan error, 1) - go func() { done <- mc.t.CommitMove(context.WithoutCancel(t.Context()), sb.ID, []byte(`{"id":"`+sb.ID+`","key":{"template":"rt:24.04","net":"none","size":"small"}}`), 1) }() + go func() { + done <- mc.t.CommitMove(context.WithoutCancel(t.Context()), sb.ID, []byte(`{"id":"`+sb.ID+`","key":{"template":"rt:24.04","net":"none","size":"small"}}`), 1) + }() <-eng.receiveEntered reserveMove(t, mc.t, sb.ID, "") retry, _ := reservationOf(mc.t, sb.ID) @@ -724,6 +727,45 @@ func TestMoveCommitWhoseRegistryReplyIsLostKeepsTheOnlyGuest(t *testing.T) { }) } +func TestMoveTargetKilledBetweenCommitAndStartStartsTheGuestAfterItsRestart(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + ctx := t.Context() + killed := make(chan struct{}) + mc.t.moves = &lossyMoves{Registry: mc.moveReg, killAfterCommit: killed, processEnd: ctx.Done()} + mc.p.commitKilled = killed + + if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move whose target died after its commit: %v, want the 200 the record proves", err) + } + if _, held := mc.s.byID(sb.ID); held { + t.Fatal("the source kept a claim the target committed") + } + if got := startsOf(mc.t); len(got) != 0 { + t.Fatalf("the killed target started %v", got) + } + if rec, ok, _ := mc.moveReg.Get(ctx, sb.ID); !ok || rec.State != moves.Committed { + t.Fatalf("record %+v %v after the source finished, want it kept committed", rec, ok) + } + + restarted := mc.restart(t, mc.t) + restarted.liftFence() + if err := restarted.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + waitFor(t, func() bool { + rec, ok, _ := mc.moveReg.Get(ctx, sb.ID) + return !restarted.moveHeld(sb.ID) && ok && rec.State == moves.Started + }) + moved, ok := restarted.byID(sb.ID) + if !ok || moved.Token != sb.Token || !slices.Contains(startsOf(restarted), moved.VMName) || fakeOf(restarted).removed(moved.VMName) { + t.Fatalf("restarted target claim %+v %v starts %v, want the only guest started", moved, ok, startsOf(restarted)) + } + if got := usageEventsOf(t, restarted, sb.ID); got != "move_in" { + t.Errorf("target usage %s, want move_in", got) + } +} + func TestMoveTargetMarksTheRecordStartedOnlyAfterItsJournalPersists(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) @@ -868,6 +910,7 @@ type fakeMovePeer struct { loseCommitReply bool dropHook func() roundBudgets []time.Duration + commitKilled <-chan struct{} } func newFakeMovePeer(t *testing.T, c *failoverCluster) *fakeMovePeer { @@ -913,6 +956,11 @@ func (p *fakeMovePeer) Commit(ctx context.Context, addr, id string, claim []byte return err } } + if p.commitKilled != nil { + go func() { _ = p.target(addr).CommitMove(ctx, id, claim, epoch) }() + <-p.commitKilled + return ErrMoveTarget + } err := p.target(addr).CommitMove(ctx, id, claim, epoch) if err == nil && p.loseCommitReply { return ErrMoveTarget @@ -937,8 +985,10 @@ func (p *fakeMovePeer) target(addr string) *Manager { type lossyMoves struct { moves.Registry - loseCommit bool - down atomic.Bool + loseCommit bool + down atomic.Bool + killAfterCommit chan struct{} + processEnd <-chan struct{} } func (r *lossyMoves) Begin(ctx context.Context, id, from, to string) (int64, error) { @@ -953,6 +1003,11 @@ func (r *lossyMoves) Advance(ctx context.Context, id string, expect int64, from, return 0, moves.ErrUnavailable } epoch, err := r.Registry.Advance(ctx, id, expect, from, to) + if err == nil && to == moves.Committed && r.killAfterCommit != nil { + close(r.killAfterCommit) + <-r.processEnd + runtime.Goexit() + } if err == nil && to == moves.Committed && r.loseCommit { r.loseCommit = false return 0, moves.ErrUnavailable From 9a7f8c548192cf3e982eaf4ddb7bee4396c6c108 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:37:13 +0800 Subject: [PATCH 26/53] docs: a move needs the same cocoon path strings, its downtime floor, and a retried admission --- docs/cluster.md | 16 ++++++++++++++-- docs/deploy.md | 2 +- docs/sandboxd-api.md | 3 ++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 46b88868..f5dc1b6e 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -366,7 +366,8 @@ ended ones. **Rounds.** The source sends a baseline round, then up to 4 deltas, each over the control port with the root token. Once the last delta's size at the rate the rounds reached, plus 400 ms for the receive and the start, fits -`max_downtime_ms` (default 2000), the source pauses the guest for the final +`max_downtime_ms` (default 2000, and above 400 since those 400 ms alone would +never fit), the source pauses the guest for the final round and the target receives the series and starts the guest. A move whose deltas never fit answers 503 and the guest runs on. `downtime_ms` in the answer is the pause the source measured, from the final capture to the target's @@ -383,9 +384,20 @@ never see it. At the commit the reservation becomes the claim under one lock, so the target never counts it twice. The source counts the claim until it drops it after the commit, so the cluster counts a moving claim twice, which can make a node refuse earlier but never over-admit. `memory_limit` is checked again -before the receive. A reservation with no round arriving and none finished or +before the receive. A new admission from the source that holds the reservation +replaces it and resets its staged files, so a move retried after a source crash +does not wait out the idle drop; one from another source answers 409. A +reservation with no round arriving and none finished or commit asked for 2 minutes is dropped with its staged files. +**Same paths.** Both nodes need the same path strings in cocoon's `root_dir` and +`run_dir`. The requirement is on the string, not the real directory: a symlink +of one name can lead to different disks on each node, and `data_dir` may differ. +The string has to be set before the VM starts, because the VM's record keeps it. +A move between nodes with different strings fails at the receive, after the +guest paused: cocoon refuses the series with `untrusted storage path in snapshot +metadata`, the call answers 502 and the guest resumes on the source. + **Replicated claims.** Before the final round the source deletes the claim's registry row and its replica, so no third node can adopt a stale replica while the target runs the claim. The target's first replica round puts a new row one diff --git a/docs/deploy.md b/docs/deploy.md index 21afca81..f6f9e1c0 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -160,7 +160,7 @@ the intersection of the two. | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim first tries a redirect to a warm peer, else to a peer that serves the pool; a volume claim answers the 429 with no redirect) | | `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was; a claim takes the same redirect as `max_claims`. Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A wake cannot move to another node, so its 429 stays until a running claim sleeps or goes. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | -| `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `archive_failover` and `mesh.cluster_key`: a node that sets this without them refuses to load, and one that leaves it unset cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or a master build), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM keeps the VMM it started with), a run dir that supports reflink such as XFS, and nodes of the same CPU model. Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | +| `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `archive_failover` and `mesh.cluster_key`: a node that sets this without them refuses to load, and one that leaves it unset cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or a master build), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM keeps the VMM it started with), a run dir that supports reflink such as XFS, nodes of the same CPU model, and the same path strings in cocoon's `root_dir` and `run_dir` on both nodes ([same paths](cluster.md#moving-a-running-claim)). Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token; a config reload records as op `config_reload` with `changed`, and a runtime volume change as op `volumes` with `added` or `removed`. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no warm VMs on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | | `memory_limit` | unset | `{"anon": "300G", "psi_some_avg10": 40}`, either key alone: while `AnonPages` in `/proc/meminfo` is above `anon`, or the `some avg10` of `/proc/pressure/memory` is above `psi_some_avg10`, checked every 2 s, the node refuses new claims, forks and checkpoint claims with 429 and parks refill; `/v1/info` reports it as at capacity and the mesh sees no pools on it, so a warm or pool redirect skips it; a template or volume held only there still redirects to it and answers 429. Wakes of its own claims, hibernates, checkpoints and archives go on, and running sandboxes keep running. Guest memory is a private file mapping, so page cache and clean guest pages are reclaimable and only anon is what a hibernate has to free. Both signals are node-wide; `AnonPages` leaves out swapped pages and explicit hugepages. A signal the node cannot read, such as PSI on a kernel without it, logs an error and is ignored. PSI also counts a task stalled by its own cgroup limit and weighs by busy CPUs, so on a mostly idle node a few throttled tasks in one small cgroup can cross it: 16 tasks under a 64M `MemoryHigh` reached 75 on an idle 384-CPU host with memory to spare. Treat `anon` as the main limit, and set `psi_some_avg10` only where no workload runs under a tight memory cgroup. Size `anon` so the running guests' further growth still fits under physical memory | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 05b5210b..8fd70108 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -853,7 +853,8 @@ its guest's processes and memory (see [moving a running claim](cluster.md#moving-a-running-claim)). Body: `{"to": "", "max_downtime_ms": 2000}`, `to` as `GET /v1/peers` lists it and never this node (else 400), `max_downtime_ms` optional with a default of -2000. Answers `{"owner_addr": "", "rounds": n, "downtime_ms": d}` once the +2000 and a floor: 400 or less answers 400 at once, because the target's receive +and start alone are allowed that long. Answers `{"owner_addr": "", "rounds": n, "downtime_ms": d}` once the peer runs the guest and this node dropped the claim; `downtime_ms` is the pause from the final capture to the peer's commit. While the move runs, release of the claim answers 503 `sandbox moving to another node; retry`, and fork, From fa229dbad568fb2c16d31a58674c56123c5357c0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:56:50 +0800 Subject: [PATCH 27/53] pool: a received claim is gated like a claim in a move until it starts While the registry was down settleMoveIn slept with Transition free and nothing gated a received claim, since the gates read the moving map, which is empty on a target. The idle sweep hibernated the paused guest, and startMoveIn then cold-booted it with HibernateSnap still set. Captures, hibernates, reaps, evacuation, release, renew, env writes, dials and wakes now treat MoveIn like a move in flight; operator dials and instance metadata answer ErrPaused as they do for any paused guest. --- sandboxd/pool/capture.go | 2 +- sandboxd/pool/claim.go | 6 +-- sandboxd/pool/drain.go | 3 +- sandboxd/pool/env.go | 2 +- sandboxd/pool/failover.go | 2 +- sandboxd/pool/hibernate.go | 6 +-- sandboxd/pool/move_out.go | 20 ++++++--- sandboxd/pool/move_test.go | 90 ++++++++++++++++++++++++++++++++++++-- sandboxd/pool/operator.go | 2 +- sandboxd/pool/reap.go | 2 +- sandboxd/pool/replica.go | 5 +-- 11 files changed, 115 insertions(+), 25 deletions(-) diff --git a/sandboxd/pool/capture.go b/sandboxd/pool/capture.go index 56ba133b..bfee3b1c 100644 --- a/sandboxd/pool/capture.go +++ b/sandboxd/pool/capture.go @@ -34,7 +34,7 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { if m.claimed[sb.ID] != sb { return ErrUnknownSandbox } - if _, moving := m.moving[sb.ID]; moving { + if m.midMoveLocked(sb) { return ErrEvacuating } if _, err := m.fenceWatch(sb); err != nil { diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 6bf51a3d..6c331f0d 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -118,7 +118,7 @@ func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { if !ok { return ErrUnknownSandbox } - if m.movingNow(id) { + if m.movingNow(sb) { return ErrEvacuating } if m.archives != nil { @@ -167,7 +167,7 @@ func (m *Manager) Renew(ctx context.Context, id string, cred Cred, ttl time.Dura m.mu.Unlock() return time.Time{}, ErrUnknownSandbox } - if m.movingFinalLocked(id) { + if m.pausedForMoveLocked(sb) { m.mu.Unlock() return time.Time{}, ErrEvacuating } @@ -261,7 +261,7 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. return nil, ErrPaused } failed := sb.Failed != "" - paused, sock := sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "", sb.VsockSocket + paused, sock := sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || sb.MoveIn != nil, sb.VsockSocket if !paused && !failed { sb.Hold() } diff --git a/sandboxd/pool/drain.go b/sandboxd/pool/drain.go index e3e35c7b..82b9f922 100644 --- a/sandboxd/pool/drain.go +++ b/sandboxd/pool/drain.go @@ -103,8 +103,7 @@ func (m *Manager) evacuateClaim(ctx context.Context, sb *types.Sandbox) error { } m.mu.Lock() _, archiving := m.archiving[sb.ID] - _, moving := m.moving[sb.ID] - busy := archiving || moving + busy := archiving || m.midMoveLocked(sb) gone := m.claimed[sb.ID] != sb || !m.evacuatingLocked(sb.ID) archived, tenantGone := sb.ArchiveCk != "", m.tenantGone(sb.Tenant) pause := !gone && !tenantGone && !busy && !archived diff --git a/sandboxd/pool/env.go b/sandboxd/pool/env.go index 7e66fd8c..f8dc5168 100644 --- a/sandboxd/pool/env.go +++ b/sandboxd/pool/env.go @@ -50,7 +50,7 @@ func (m *Manager) writeEnv(ctx context.Context, id, tenant, verb string, next fu m.mu.Unlock() return ErrUnknownSandbox } - if !locked && m.movingFinalLocked(id) { + if sb.MoveIn != nil || !locked && m.movingFinalLocked(id) { m.mu.Unlock() return ErrEvacuating } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 2591c292..8d8ffee5 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -325,7 +325,7 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox) { func (m *Manager) releaseRowed(ctx context.Context, id string, sb *types.Sandbox) error { sb.Transition.Lock() defer sb.Transition.Unlock() - if m.movingNow(id) { + if m.movingNow(sb) { return ErrEvacuating } if err := m.resolveIntent(context.WithoutCancel(ctx), sb); err != nil { diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 06588d71..4dd63c0c 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -62,7 +62,7 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error if sb.ArchiveCk != "" { return nil // already off the node } - if m.movingNow(sb.ID) { + if m.movingNow(sb) { return ErrEvacuating } if _, err := m.fenceWatch(sb); err != nil { @@ -154,7 +154,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, } func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bool) (string, error) { - if m.movingFinal(sb.ID) { + if m.pausedForMove(sb) { return "", ErrEvacuating } sb.Transition.Lock() @@ -261,7 +261,7 @@ func (m *Manager) idleOnce(ctx context.Context) { return } for _, sb := range m.claimed { - if skipIdle(sb) { + if skipIdle(sb) || m.midMoveLocked(sb) { continue } idle := idleDefault diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 9c5f7147..512708b4 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -351,11 +351,15 @@ func (m *Manager) endMoveOut(sb *types.Sandbox, mv *outMove) { } } -// movingFinal reports whether sb's guest is in the final round of a move, when dials and wakes answer ErrEvacuating. -func (m *Manager) movingFinal(id string) bool { +// pausedForMove reports whether sb's guest is paused by a move, in a source's final round or received and not started, when dials and wakes answer ErrEvacuating. +func (m *Manager) pausedForMove(sb *types.Sandbox) bool { m.mu.Lock() defer m.mu.Unlock() - return m.movingFinalLocked(id) + return m.pausedForMoveLocked(sb) +} + +func (m *Manager) pausedForMoveLocked(sb *types.Sandbox) bool { + return sb.MoveIn != nil || m.movingFinalLocked(sb.ID) } func (m *Manager) movingFinalLocked(id string) bool { @@ -363,11 +367,15 @@ func (m *Manager) movingFinalLocked(id string) bool { return mv != nil && mv.final } -func (m *Manager) movingNow(id string) bool { +func (m *Manager) movingNow(sb *types.Sandbox) bool { m.mu.Lock() defer m.mu.Unlock() - _, moving := m.moving[id] - return moving + return m.midMoveLocked(sb) +} + +func (m *Manager) midMoveLocked(sb *types.Sandbox) bool { + _, moving := m.moving[sb.ID] + return moving || sb.MoveIn != nil } func roundBytes(round types.MigrateRound) int64 { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 6840ca18..2ab94b4c 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -210,7 +210,7 @@ func TestMoveKeepsThePausedGuestUntilItsAbortLands(t *testing.T) { t.Fatal("the source resumed its guest while the registry was down") } mc.moveReg.SetDown(false) - waitFor(t, func() bool { return !pausedOn(mc.s, sb.VMName) && !mc.s.movingNow(sb.ID) }) + waitFor(t, func() bool { return !pausedOn(mc.s, sb.VMName) && !mc.s.movingNow(sb) }) assertStillOnSource(t, mc, sb) if rec, ok, _ := mc.moveReg.Get(t.Context(), sb.ID); ok { t.Errorf("record %+v after the abort, want it deleted", rec) @@ -497,7 +497,7 @@ func TestMoveEndsWhenAFailoverTookTheClaimFromTheSource(t *testing.T) { if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveTarget) { t.Fatalf("move of a claim another node took: %v, want ErrMoveTarget", err) } - if mc.s.movingNow(sb.ID) { + if mc.s.movingNow(sb) { t.Fatal("the move still holds its slot") } if !sb.Transition.TryLock() { @@ -706,6 +706,85 @@ func TestMoveStaleSweepLeavesACommittedRecordItsTargetStillHoldsMoveInFor(t *tes } } +func TestMoveReceivedClaimWaitingOutAnOutageIsNotHibernatedOrServed(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + ctx := t.Context() + lossy := &lossyMoves{Registry: mc.moveReg, downAfterCommit: true} + mc.t.moves = lossy + if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v, want it resolved through the committed record", err) + } + received, ok := mc.t.byID(sb.ID) + if !ok { + t.Fatal("the target does not hold the received claim") + } + waitFor(t, func() bool { + if !received.Transition.TryLock() { + return false + } + received.Transition.Unlock() + return true + }) + cred := Cred{Token: sb.Token} + backdate(mc.t, received, time.Hour) + sweepIdle(t, mc.t) + expire(mc.t, received) + mc.t.reapOnce(ctx) + if _, held := mc.t.byID(sb.ID); !held { + t.Fatal("the reap took the received claim") + } + mc.t.mu.Lock() + mc.t.evac = &evacuation{pending: map[string]*types.Sandbox{sb.ID: received}, stop: func() {}} + mc.t.mu.Unlock() + if err := mc.t.evacuateClaim(ctx, received); !errors.Is(err, errArchiveBusy) { + t.Errorf("evacuation of a received claim: %v, want errArchiveBusy", err) + } + mc.t.mu.Lock() + mc.t.evac = nil + mc.t.mu.Unlock() + _, forkErr := mc.t.Fork(ctx, sb.ID, cred, 1, time.Minute, "", "") + _, ckErr := mc.t.Checkpoint(ctx, sb.ID, cred, "ck", "") + _, _, promoteErr := mc.t.Promote(ctx, sb.ID, cred, "tpl", "") + _, renewErr := mc.t.Renew(ctx, sb.ID, cred, time.Hour, "") + _, _, _, dialErr := mc.t.WakeAgentSocket(ctx, sb.ID, sb.Token) + for op, err := range map[string]error{ + "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, "renew": renewErr, "dial": dialErr, + "hibernate": mc.t.Hibernate(ctx, sb.ID, cred), + "release": mc.t.Release(ctx, sb.ID, cred), + "env": mc.t.PatchEnv(ctx, sb.ID, types.EnvPatch{"GH_TOKEN": {Value: "x", Guest: hostOnly}}, ""), + "wake": mc.t.Wake(ctx, sb.ID, cred), + } { + if !errors.Is(err, ErrEvacuating) { + t.Errorf("%s of a received claim: %v, want ErrEvacuating", op, err) + } + } + if _, err := mc.t.DialPort(ctx, sb.ID, Cred{Operator: true}, 80); !errors.Is(err, ErrPaused) { + t.Errorf("operator dial of a received claim: %v, want ErrPaused", err) + } + if err := mc.t.SetInstanceMetadata(ctx, sb.ID, []byte("{}")); !errors.Is(err, ErrPaused) { + t.Errorf("instance metadata of a received claim: %v, want ErrPaused", err) + } + eng := fakeOf(mc.t) + if got := eng.hibernateCount(); got != 0 { + t.Fatalf("%d hibernates of a paused received guest", got) + } + if got := startsOf(mc.t); len(got) != 0 { + t.Fatalf("started %v before the record resolved", got) + } + + lossy.setDown(false) + waitFor(t, func() bool { return !mc.t.moveHeld(sb.ID) }) + mc.t.mu.Lock() + snap := received.HibernateSnap + mc.t.mu.Unlock() + eng.mu.Lock() + defer eng.mu.Unlock() + if snap != "" || len(eng.hibernates) != 0 || len(eng.exports) != 0 || len(eng.snapSaves) != 0 || !slices.Contains(eng.starts, received.VMName) || eng.stopped[received.VMName] { + t.Errorf("snapshot %q hibernates %v exports %v saves %v starts %v, want only the start of the paused guest", snap, eng.hibernates, eng.exports, eng.snapSaves, eng.starts) + } +} + func TestMoveCommitWhoseRegistryReplyIsLostKeepsTheOnlyGuest(t *testing.T) { mc := newMoveCluster(t) mc.t.moves = &lossyMoves{Registry: mc.moveReg, loseCommit: true} @@ -837,7 +916,7 @@ func TestMoveSourceBackFromAnOutageNeverResumesAClaimThatMovedOn(t *testing.T) { down.setDown(false) waitFor(t, func() bool { _, held := mc.s.byID(sb.ID) - return !held && !mc.s.movingNow(sb.ID) + return !held && !mc.s.movingNow(sb) }) eng := fakeOf(mc.s) eng.mu.Lock() @@ -985,6 +1064,7 @@ func (p *fakeMovePeer) target(addr string) *Manager { type lossyMoves struct { moves.Registry + downAfterCommit bool loseCommit bool down atomic.Bool killAfterCommit chan struct{} @@ -1008,6 +1088,10 @@ func (r *lossyMoves) Advance(ctx context.Context, id string, expect int64, from, <-r.processEnd runtime.Goexit() } + if err == nil && to == moves.Committed && r.downAfterCommit { + r.down.Store(true) + return 0, moves.ErrUnavailable + } if err == nil && to == moves.Committed && r.loseCommit { r.loseCommit = false return 0, moves.ErrUnavailable diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index bf9c2b42..d9179506 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -57,7 +57,7 @@ func (m *Manager) SetInstanceMetadata(ctx context.Context, id string, doc []byte return ErrArchived case sb.Failed != "": return ErrFailed - case sb.HibernateSnap != "" || sb.PendingSnap != "": + case sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.MoveIn != nil: return ErrPaused } return m.eng.WriteInstanceMetadata(ctx, sb.VsockSocket, doc) diff --git a/sandboxd/pool/reap.go b/sandboxd/pool/reap.go index d53fbd23..756c3418 100644 --- a/sandboxd/pool/reap.go +++ b/sandboxd/pool/reap.go @@ -36,7 +36,7 @@ func (m *Manager) reapOnce(ctx context.Context) { var unrows []*types.Sandbox for id, sb := range m.claimed { // A zero deadline means no expiry (an archived claim kept forever). - if _, moving := m.moving[id]; moving || sb.Deadline.IsZero() || !now.After(sb.Deadline) { + if m.midMoveLocked(sb) || sb.Deadline.IsZero() || !now.After(sb.Deadline) { continue } switch { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 92926146..3f4a6626 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -67,8 +67,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { // replicable excludes a claim that is paused, archived or archiving, failed, or whose tenant was removed; callers hold m.mu. func (m *Manager) replicable(sb *types.Sandbox) bool { _, archiving := m.archiving[sb.ID] - _, moving := m.moving[sb.ID] - return !archiving && !moving && runningShaped(sb) && sb.ArchivePending == "" && !m.tenantGone(sb.Tenant) + return !archiving && !m.midMoveLocked(sb) && runningShaped(sb) && sb.ArchivePending == "" && !m.tenantGone(sb.Tenant) } func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { @@ -109,7 +108,7 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { var intent claimSnapshot var refusal error switch { - case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "" || m.moving[sb.ID] != nil: + case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "" || m.midMoveLocked(sb): refusal = errWokeMeanwhile case m.hasVolumeHolds(sb): refusal = ErrVolumeCapture From ea534510e8244f89913b15c9aa947b2317c07324 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:57:35 +0800 Subject: [PATCH 28/53] pool: a node holding a claim settles a committed record that blocks its next move A committed record whose target was lost, or whose started write failed, answered 409 to every later hot move of the claim, forever, even from a node that adopted it. Holding the claim with no MoveIn and no journal of its own proves the target already started it, so Begin's refusal now advances the record to started and begins again, once. --- sandboxd/pool/move_out.go | 23 ++++++++++- sandboxd/pool/move_test.go | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 512708b4..3f6ae5ea 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -55,7 +55,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b return err } mv.to = to - epoch, err := m.moves.Begin(ctx, sb.ID, m.nodeID, to) + epoch, err := m.beginMove(ctx, sb, to) if errors.Is(err, moves.ErrLost) { return fmt.Errorf("%w: another move of %s is on record", ErrNotMovable, sb.ID) } @@ -130,6 +130,27 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b return m.movePeer.Commit(ctx, mv.addr, sb.ID, claim, final) } +func (m *Manager) beginMove(ctx context.Context, sb *types.Sandbox, to string) (int64, error) { + epoch, err := m.moves.Begin(ctx, sb.ID, m.nodeID, to) + if !errors.Is(err, moves.ErrLost) { + return epoch, err + } + rec, ok, err := m.moves.Get(ctx, sb.ID) + if err != nil { + return 0, err + } + m.mu.Lock() + settled := m.claimed[sb.ID] == sb && sb.MoveIn == nil && sb.MoveOut == nil + m.mu.Unlock() + if !ok || rec.State != moves.Committed || !settled { + return 0, moves.ErrLost + } + if _, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, moves.Committed, moves.Started); err != nil && !errors.Is(err, moves.ErrLost) { + return 0, err + } + return m.moves.Begin(ctx, sb.ID, m.nodeID, to) +} + func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round types.MigrateRound) error { mv.rounds++ names := make([]string, len(round.Files)) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 2ab94b4c..488d7c46 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -785,6 +785,83 @@ func TestMoveReceivedClaimWaitingOutAnOutageIsNotHibernatedOrServed(t *testing.T } } +func TestMoveFromTheHolderOfACommittedRecordMarksItStartedAndProceeds(t *testing.T) { + t.Run("target whose started write was lost", func(t *testing.T) { + mc := newMoveCluster(t) + third := mc.moveNode(t, "c") + mc.t.moves = &lossyMoves{Registry: mc.moveReg, loseStarted: true} + sb := mustClaim(t, mc.s, testKey) + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + if rec, _, _ := mc.moveReg.Get(t.Context(), sb.ID); rec.State != moves.Committed { + t.Fatalf("record %+v, want it committed since the started write was lost", rec) + } + if _, err := mc.t.Move(t.Context(), sb.ID, "c", time.Minute); err != nil { + t.Fatalf("move on from the target: %v", err) + } + if _, held := third.byID(sb.ID); !held { + t.Error("the third node does not hold the claim") + } + }) + t.Run("adopter", func(t *testing.T) { + mc := newMoveCluster(t) + adopter := mc.moveNode(t, "c") + sb := mustClaim(t, adopter, testKey) + seedMoveRecord(t, mc.moveReg, sb.ID, "a", "lost", moves.Committed) + if _, err := adopter.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move from the adopter: %v", err) + } + if _, held := mc.t.byID(sb.ID); !held { + t.Error("the target does not hold the claim") + } + }) +} + +func TestMoveBeginLeavesARecordOnlyItsHolderMaySettle(t *testing.T) { + for name, tt := range map[string]struct { + reach moves.State + edit func(sb *types.Sandbox) + }{ + "a move in flight": {reach: moves.Pending}, + "final": {reach: moves.Final}, + "held under a journal": {reach: moves.Committed, edit: func(sb *types.Sandbox) { sb.MoveOut = &types.MoveOut{To: "b", Epoch: 1} }}, + "received and not started": {reach: moves.Committed, edit: func(sb *types.Sandbox) { sb.MoveIn = &types.MoveIn{From: "a", Epoch: 1} }}, + } { + t.Run(name, func(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + seedMoveRecord(t, mc.moveReg, sb.ID, "z", "y", tt.reach) + if tt.edit != nil { + mc.s.mu.Lock() + tt.edit(sb) + mc.s.mu.Unlock() + } + if _, err := mc.s.beginMove(t.Context(), sb, "b"); !errors.Is(err, moves.ErrLost) { + t.Errorf("begin: %v, want ErrLost", err) + } + if rec, _, _ := mc.moveReg.Get(t.Context(), sb.ID); rec.State != tt.reach || rec.From != "z" { + t.Errorf("record %+v, want %s untouched", rec, tt.reach) + } + mc.s.mu.Lock() + sb.MoveOut, sb.MoveIn = nil, nil + mc.s.mu.Unlock() + }) + } + t.Run("a claim this node no longer holds", func(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + seedMoveRecord(t, mc.moveReg, sb.ID, "z", "y", moves.Committed) + mc.s.forgetClaim(t.Context(), sb) + if _, err := mc.s.beginMove(t.Context(), sb, "b"); !errors.Is(err, moves.ErrLost) { + t.Errorf("begin: %v, want ErrLost", err) + } + if rec, _, _ := mc.moveReg.Get(t.Context(), sb.ID); rec.State != moves.Committed { + t.Errorf("record %+v, want it committed", rec) + } + }) +} + func TestMoveCommitWhoseRegistryReplyIsLostKeepsTheOnlyGuest(t *testing.T) { mc := newMoveCluster(t) mc.t.moves = &lossyMoves{Registry: mc.moveReg, loseCommit: true} @@ -1065,6 +1142,7 @@ func (p *fakeMovePeer) target(addr string) *Manager { type lossyMoves struct { moves.Registry downAfterCommit bool + loseStarted bool loseCommit bool down atomic.Bool killAfterCommit chan struct{} @@ -1082,6 +1160,10 @@ func (r *lossyMoves) Advance(ctx context.Context, id string, expect int64, from, if r.down.Load() { return 0, moves.ErrUnavailable } + if to == moves.Started && r.loseStarted { + r.loseStarted = false + return 0, moves.ErrUnavailable + } epoch, err := r.Registry.Advance(ctx, id, expect, from, to) if err == nil && to == moves.Committed && r.killAfterCommit != nil { close(r.killAfterCommit) From 97a5da6c8a1f8788dd1abf321d7c3873c84da290 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 20:57:35 +0800 Subject: [PATCH 29/53] docs: a 503 with the registry down can leave a move unresolved, and a move's known limits --- docs/cluster.md | 18 ++++++++++++++++-- docs/sandboxd-api.md | 5 ++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index f5dc1b6e..8b9e16d6 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -336,11 +336,15 @@ and `epoch`, each change a compare-and-set at a new epoch. | (none) | `pending` | source | after the target admitted the move, before the baseline round | | `pending` | `final` | source | the final round paused the guest | | `final` | `committed` | target | after the receive; the point of no return | -| `committed` | `started` | target | after it journaled that it runs the claim | +| `committed` | `started` | target, or a later source that holds the claim | after it journaled that it runs the claim | | `pending` or `final` | `aborted` | source, or a target that holds a received claim or finds the record stale | before the guest resumes | A new move of the claim overwrites only a `started` or `aborted` record; a -`committed` one refuses it with 409 until its target marks it `started`. +`committed` one refuses it with 409 unless the node making the new move holds the +claim, which proves the target started it, so that node marks it `started` first. +Until a received claim starts, its target gates it like a claim in a move: sweeps +skip it, calls that reach the guest or change the claim answer 503, and operator +calls answer 409. The target commits only from `final`, so it never starts a guest restored from a round the source kept running after. The source resumes its guest only on @@ -398,6 +402,16 @@ A move between nodes with different strings fails at the receive, after the guest paused: cocoon refuses the series with `untrusted storage path in snapshot metadata`, the call answers 502 and the guest resumes on the source. +**Known limits.** + +- A `committed` record whose target never returns stays as one row until a node + holding the claim moves it. +- Without `sync_claims` a host crash can roll back a node's journal. A source that + rolled back to a journal whose record is gone drops the claim, since a missing + record reads as moved on. +- A move call on a claim that moved further than the peer it named answers 200 + with that peer as `owner_addr`; `Lookup` finds the claim. + **Replicated claims.** Before the final round the source deletes the claim's registry row and its replica, so no third node can adopt a stale replica while the target runs the claim. The target's first replica round puts a new row one diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 8fd70108..2270b3f7 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -861,7 +861,10 @@ the claim answers 503 `sandbox moving to another node; retry`, and fork, checkpoint, promote and hibernate answer the same during the rounds and wait out the final round. Env writes, renew and data-plane calls keep working until the final round and answer the same 503 during it, because the peer would miss -them. Every error leaves the claim running here: +them. Every error leaves the claim running here, except a 503 with the registry +down after the final round paused the guest: the guest stays paused until the +record answers, the guest then resumes or, when the record reads `committed`, the +node drops the claim (see [moving a running claim](cluster.md#moving-a-running-claim)): - 404 unknown sandbox - 409 the claim cannot move (hibernated, archived, failed, waking, with From de34c90930bda206c9237a181c4f7fc6debe7658 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:01:57 +0800 Subject: [PATCH 30/53] pool: pin that the idle sweep skips a received claim The hibernate gate already refused it, so the sweep's own skip showed only as a sweep blocked on Transition. --- sandboxd/pool/move_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 488d7c46..5f95b0e0 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -728,7 +728,9 @@ func TestMoveReceivedClaimWaitingOutAnOutageIsNotHibernatedOrServed(t *testing.T }) cred := Cred{Token: sb.Token} backdate(mc.t, received, time.Hour) + received.Transition.Lock() sweepIdle(t, mc.t) + received.Transition.Unlock() expire(mc.t, received) mc.t.reapOnce(ctx) if _, held := mc.t.byID(sb.ID); !held { From c7188c88dc57a1597ed9704edd9b17fbed853359 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:06:12 +0800 Subject: [PATCH 31/53] review: lint finding in the move test --- sandboxd/pool/move_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 5f95b0e0..23d42df0 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -749,7 +749,7 @@ func TestMoveReceivedClaimWaitingOutAnOutageIsNotHibernatedOrServed(t *testing.T _, ckErr := mc.t.Checkpoint(ctx, sb.ID, cred, "ck", "") _, _, promoteErr := mc.t.Promote(ctx, sb.ID, cred, "tpl", "") _, renewErr := mc.t.Renew(ctx, sb.ID, cred, time.Hour, "") - _, _, _, dialErr := mc.t.WakeAgentSocket(ctx, sb.ID, sb.Token) + _, dialErr := mc.t.DialPort(ctx, sb.ID, cred, 80) for op, err := range map[string]error{ "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, "renew": renewErr, "dial": dialErr, "hibernate": mc.t.Hibernate(ctx, sb.ID, cred), From 9d5b0e9b8204b93d69decca8ed43fe92a1f63eee Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:18:21 +0800 Subject: [PATCH 32/53] server: the move client reaches a target advertised by hostname url.Parse reads node-1 in node-1:8080 as a scheme, so Admit, Commit and Drop skipped the http prefix and answered 502 on a hostname advertise_addr, which docs/deploy.md allows. The client now uses the peer package's URL. --- sandboxd/server/move.go | 6 +----- sandboxd/server/move_test.go | 22 ++++++++++++++++++++++ sandboxd/store/peer/adopt.go | 2 +- sandboxd/store/peer/move.go | 2 +- sandboxd/store/peer/transport.go | 4 ++-- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go index 2ef6ab83..02a7de1b 100644 --- a/sandboxd/server/move.go +++ b/sandboxd/server/move.go @@ -60,11 +60,7 @@ func (c *MoveClient) call(ctx context.Context, method, addr, path string, in, ou } body = bytes.NewReader(raw) } - target := addr + path - if u, err := url.Parse(addr); err != nil || u.Scheme == "" { - target = "http://" + target - } - req, err := http.NewRequestWithContext(ctx, method, target, body) + req, err := http.NewRequestWithContext(ctx, method, peer.URL(addr, path), body) if err != nil { return err } diff --git a/sandboxd/server/move_test.go b/sandboxd/server/move_test.go index 15237d4c..50fadc4b 100644 --- a/sandboxd/server/move_test.go +++ b/sandboxd/server/move_test.go @@ -77,6 +77,28 @@ func TestMoveRouteRefusesABudgetTheStartAllowanceAloneExceedsBeforeAnyMoveStarts } } +func TestMoveClientReachesATargetAdvertisedByHostname(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/commit") || r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, http.StatusOK, moveAdmitResponse{Node: "b"}) + })) + defer target.Close() + hostname := strings.Replace(strings.TrimPrefix(target.URL, "http://"), "127.0.0.1", "localhost", 1) + c := &MoveClient{Token: "root"} + if node, err := c.Admit(t.Context(), hostname, "sb_1", pool.MoveAdmission{From: "a"}); err != nil || node != "b" { + t.Errorf("admit at %s: %q %v, want node b", hostname, node, err) + } + if err := c.Commit(t.Context(), hostname, "sb_1", []byte("{}"), 1); err != nil { + t.Errorf("commit at %s: %v", hostname, err) + } + if err := c.Drop(t.Context(), hostname, "sb_1"); err != nil { + t.Errorf("drop at %s: %v", hostname, err) + } +} + func TestMoveClientPassesTheTargetsRefusalThrough(t *testing.T) { status, reason := http.StatusOK, "" var gotAuth, gotPath string diff --git a/sandboxd/store/peer/adopt.go b/sandboxd/store/peer/adopt.go index 5d1dea0c..e99aae0d 100644 --- a/sandboxd/store/peer/adopt.go +++ b/sandboxd/store/peer/adopt.go @@ -17,7 +17,7 @@ type Adopter struct { func (a *Adopter) Adopt(ctx context.Context, id, token string) bool { logger := log.WithFunc("peer.Adopter.Adopt") for _, addr := range a.Peers() { - err := peerCall(ctx, http.MethodGet, peerURL(addr, "/v1/sandboxes/"+url.PathEscape(id)+"/owner"), token, http.StatusOK) + err := peerCall(ctx, http.MethodGet, URL(addr, "/v1/sandboxes/"+url.PathEscape(id)+"/owner"), token, http.StatusOK) if err == nil { return true } diff --git a/sandboxd/store/peer/move.go b/sandboxd/store/peer/move.go index 53c3ea39..2385ba9f 100644 --- a/sandboxd/store/peer/move.go +++ b/sandboxd/store/peer/move.go @@ -25,7 +25,7 @@ func SendRound(ctx context.Context, addr, token, path, dir string, names []strin _ = pw.CloseWithError(err) written <- err }() - err := putRound(ctx, peerURL(addr, path), token, pr) + err := putRound(ctx, URL(addr, path), token, pr) _ = pr.Close() if writeErr := <-written; writeErr != nil && !errors.Is(writeErr, io.ErrClosedPipe) { return fmt.Errorf("send round to %s: %w", addr, writeErr) diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index b37a537b..ff8322a3 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -202,10 +202,10 @@ func writeFile(target string, r io.Reader, mode os.FileMode) error { } func checkpointURL(addr, id, suffix string) string { - return peerURL(addr, "/v1/checkpoints/"+url.PathEscape(id)+suffix) + return URL(addr, "/v1/checkpoints/"+url.PathEscape(id)+suffix) } -func peerURL(addr, path string) string { +func URL(addr, path string) string { if !strings.Contains(addr, "://") { addr = "http://" + addr } From e9a2a4007310b87507e741798df02220fec8c737 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:18:53 +0800 Subject: [PATCH 33/53] pool: a volume change answers 503 while the claim is mid-move on either side checkVolumeRunLocked had no move gate, so PATCH volumes passed during the rounds and on a received claim, attaching a disk cocoon's capture refuses or reaching a paused guest. --- sandboxd/pool/move_test.go | 36 ++++++++++++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 2 ++ 2 files changed, 38 insertions(+) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 23d42df0..6e20a069 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -545,6 +545,42 @@ func TestMoveRefusesWritesThatWouldMissTheRecordTheTargetReceives(t *testing.T) } } +func TestMoveHoldsOffVolumeChanges(t *testing.T) { + addData := types.VolumePatch{Add: []types.Volume{{Name: "data", AttachOnly: true}}} + t.Run("during the rounds", func(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.volumes = liveCatalog(t) }) + sb := mustClaim(t, mc.s, testKey) + mc.p.roundHook = func(round int) error { + if round == 1 { + if _, err := mc.s.PatchVolumes(t.Context(), sb.ID, "", addData); !errors.Is(err, ErrEvacuating) { + t.Errorf("volume change during the rounds: %v, want ErrEvacuating", err) + } + } + return nil + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + if ops := fakeOf(mc.s).volumeOpsLog(); slices.ContainsFunc(ops, func(op string) bool { return strings.HasPrefix(op, "attach") }) { + t.Errorf("volume ops %v during a move", ops) + } + }) + t.Run("on a received claim", func(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.volumes = liveCatalog(t) }) + sb := mustClaim(t, mc.s, testKey) + lossy := &lossyMoves{Registry: mc.moveReg, downAfterCommit: true} + mc.t.moves = lossy + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } + if _, err := mc.t.PatchVolumes(t.Context(), sb.ID, "", addData); !errors.Is(err, ErrEvacuating) { + t.Errorf("volume change on a received claim: %v, want ErrEvacuating", err) + } + lossy.setDown(false) + waitFor(t, func() bool { return !mc.t.moveHeld(sb.ID) }) + }) +} + func TestMoveRefusesClaimsThatCannotMove(t *testing.T) { mc := newMoveCluster(t) hibernatedSb := mustClaim(t, mc.s, testKey) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 934930af..68a6c81f 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -228,6 +228,8 @@ func (m *Manager) checkVolumeRunLocked(ctx context.Context, sb *types.Sandbox, r return ErrUnknownSandbox } switch { + case m.midMoveLocked(sb): + return ErrEvacuating case sb.Failed != "": return ErrFailed case sb.ArchiveCk != "": From 10ed012cee1516b446ff73e6dd65b8b8bae7615b Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:19:36 +0800 Subject: [PATCH 34/53] pool: liveness leaves a moving claim's VM alone until the move ends A VMM that exited during the rounds was cold-booted while the move went on with the old series, so the move could commit a guest that had lost its memory. Liveness, like the replica round, now reads one predicate: running-shaped, not archiving, not moving. The move fails on its next capture, and liveness handles the exit once the move has ended. --- sandboxd/pool/liveness.go | 12 +++++++++--- sandboxd/pool/move_test.go | 34 ++++++++++++++++++++++++++++++++++ sandboxd/pool/pool_test.go | 2 ++ sandboxd/pool/replica.go | 3 +-- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/sandboxd/pool/liveness.go b/sandboxd/pool/liveness.go index cd9bb224..1a65e691 100644 --- a/sandboxd/pool/liveness.go +++ b/sandboxd/pool/liveness.go @@ -116,10 +116,10 @@ func (m *Manager) watchedClaims(vmName string) []liveSuspect { defer m.mu.Unlock() var out []liveSuspect for _, sb := range m.claimed { - if vmName != "" && sb.VMName != vmName || !runningShaped(sb) { + if vmName != "" && sb.VMName != vmName { continue } - if _, archiving := m.archiving[sb.ID]; !archiving && !m.leaseLapsed(sb, now) { + if m.vmSettledLocked(sb) && !m.leaseLapsed(sb, now) { out = append(out, liveSuspect{sb, sb.VMName}) } } @@ -174,7 +174,7 @@ func (m *Manager) recoverVMM(ctx context.Context, sb *types.Sandbox) { defer sb.Transition.Unlock() now := time.Now() m.mu.Lock() - live := m.claimed[sb.ID] == sb && runningShaped(sb) && !m.leaseLapsed(sb, now) + live := m.claimed[sb.ID] == sb && m.vmSettledLocked(sb) && !m.leaseLapsed(sb, now) volumes := m.hasVolumeHolds(sb) m.mu.Unlock() if !live { @@ -308,6 +308,12 @@ func (m *Manager) leaseLapsed(sb *types.Sandbox, now time.Time) bool { return (sb.OnExpire != types.ExpireArchive || m.tenantGone(sb.Tenant)) && !sb.Deadline.IsZero() && now.After(sb.Deadline) } +func (m *Manager) vmSettledLocked(sb *types.Sandbox) bool { + _, archiving := m.archiving[sb.ID] + _, moving := m.moving[sb.ID] + return runningShaped(sb) && !archiving && !moving +} + // runningShaped reports whether sb's VM should be up; callers hold m.mu. func runningShaped(sb *types.Sandbox) bool { return sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.Failed == "" && sb.MoveIn == nil diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 6e20a069..659ce233 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -581,6 +581,40 @@ func TestMoveHoldsOffVolumeChanges(t *testing.T) { }) } +func TestMoveLeavesAnExitedVMMToLivenessUntilItEnds(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + vm, eng := sb.VMName, fakeOf(mc.s) + mc.p.roundHook = func(round int) error { + if round != 1 { + return nil + } + killVMM(eng, vm) + if watched := mc.s.watchedClaims(""); slices.ContainsFunc(watched, func(s liveSuspect) bool { return s.sb == sb }) { + t.Error("liveness watches the moving claim") + } + mc.s.recoverVMM(t.Context(), sb) + if got := startsOf(mc.s); len(got) != 0 { + t.Errorf("liveness started %v during the move", got) + } + return nil + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err == nil { + t.Fatal("a move whose VMM exited succeeded") + } + if _, held := mc.t.byID(sb.ID); held { + t.Error("the target holds a claim whose VMM exited mid-move") + } + assertNoMoveResidue(t, mc.t, sb.ID) + if got := startsOf(mc.s); len(got) != 0 { + t.Fatalf("liveness started %v before the move ended", got) + } + mc.s.recoverVMM(t.Context(), sb) + if got := startsOf(mc.s); !slices.Contains(got, vm) { + t.Errorf("starts %v after the move ended, want liveness to restart the VM", got) + } +} + func TestMoveRefusesClaimsThatCannotMove(t *testing.T) { mc := newMoveCluster(t) hibernatedSb := mustClaim(t, mc.s, testKey) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index bf00c0ab..2d239618 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1861,6 +1861,8 @@ func (f *fakeEngine) MigrateCapture(_ context.Context, vmName, series string, fi switch { case f.captureErr != nil: return types.MigrateRound{}, f.captureErr + case f.stopped[vmName]: + return types.MigrateRound{}, fmt.Errorf("vm %s is not running", vmName) case final && f.finalCaptureErr != nil: f.paused[vmName] = true return types.MigrateRound{}, f.finalCaptureErr diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 3f4a6626..16667149 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -66,8 +66,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { // replicable excludes a claim that is paused, archived or archiving, failed, or whose tenant was removed; callers hold m.mu. func (m *Manager) replicable(sb *types.Sandbox) bool { - _, archiving := m.archiving[sb.ID] - return !archiving && !m.midMoveLocked(sb) && runningShaped(sb) && sb.ArchivePending == "" && !m.tenantGone(sb.Tenant) + return m.vmSettledLocked(sb) && sb.ArchivePending == "" && !m.tenantGone(sb.Tenant) } func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { From 44f53a52a7ac64663da1f21343271c2d947d6bf2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:20:20 +0800 Subject: [PATCH 35/53] pool: a move whose deltas stop shrinking answers at once A hot writer on hardware took 66 s to reach its 503. From the second delta, one not smaller than the delta before it and still too large cannot converge, so the move ends there. --- docs/cluster.md | 3 ++- sandboxd/pool/move_out.go | 21 ++++++++++----------- sandboxd/pool/move_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 8b9e16d6..592e162b 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -373,7 +373,8 @@ the rounds reached, plus 400 ms for the receive and the start, fits `max_downtime_ms` (default 2000, and above 400 since those 400 ms alone would never fit), the source pauses the guest for the final round and the target receives the series and starts the guest. A move whose -deltas never fit answers 503 and the guest runs on. `downtime_ms` in the answer +deltas never fit answers 503 and the guest runs on; from the second delta on, one +not smaller than the delta before it and still too large answers at once. `downtime_ms` in the answer is the pause the source measured, from the final capture to the target's commit. Each round has one minute plus its size at 10 MB/s to arrive, so a target that stops reading fails the move. The final capture, its transfer and the target's commit have 2 minutes diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 3f6ae5ea..96c69ba5 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -74,10 +74,9 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b if err = m.journalMoveOut(ctx, sb, mv); err != nil { return err } - var sent int64 + var sent, previous int64 var sending time.Duration - fits := false - for i := 0; ; i++ { + for delta := 0; ; delta++ { start := time.Now() if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { return err @@ -85,19 +84,19 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b sending += time.Since(start) size := roundBytes(round) sent += size - if i > 0 && finalPause(size, sent, sending) <= budget { - fits = true - } - if fits || i == moveDeltaLimit { - break + if delta > 0 { + if finalPause(size, sent, sending) <= budget { + break + } + if delta == moveDeltaLimit || delta > 1 && size >= previous { + return fmt.Errorf("%w: delta %d of %d bytes does not fit %s", ErrMoveDowntime, delta, size, budget) + } } + previous = size if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, false); err != nil { return fmt.Errorf("delta capture: %w", err) } } - if !fits { - return fmt.Errorf("%w: %d deltas did not shrink under %s", ErrMoveDowntime, moveDeltaLimit, budget) - } m.mu.Lock() mv.final = true m.mu.Unlock() diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 659ce233..e4dcd25c 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -581,6 +581,34 @@ func TestMoveHoldsOffVolumeChanges(t *testing.T) { }) } +func TestMoveGivesUpOnDeltasThatStopShrinking(t *testing.T) { + for name, tt := range map[string]struct { + sizes []int64 + captures int + }{ + "second delta as large as the first": {sizes: []int64{1000, 500, 500}, captures: 3}, + "second delta larger than the first": {sizes: []int64{1000, 500, 600}, captures: 3}, + "shrinking but never fitting": {sizes: []int64{1000, 900, 800, 700, 600}, captures: 5}, + } { + t.Run(name, func(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + eng := fakeOf(mc.s) + eng.migrateSizes = tt.sizes + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Millisecond); !errors.Is(err, ErrMoveDowntime) { + t.Fatalf("move: %v, want ErrMoveDowntime", err) + } + eng.mu.Lock() + captures := len(eng.captures) + eng.mu.Unlock() + if captures != tt.captures { + t.Errorf("%d captures, want %d", captures, tt.captures) + } + assertStillOnSource(t, mc, sb) + }) + } +} + func TestMoveLeavesAnExitedVMMToLivenessUntilItEnds(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) From 1ffbadfb996c1a156f7b450025b2fa539b31d298 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:21:06 +0800 Subject: [PATCH 36/53] pool: user calls answer 503 at once during a move's final round Hibernate, fork, promote and checkpoint took Transition and so waited out the final round before answering. lockTransition refuses a claim mid-move, locks, and checks again. --- docs/sandboxd-api.md | 7 +++---- sandboxd/pool/capture.go | 7 +++---- sandboxd/pool/claim.go | 3 --- sandboxd/pool/failover.go | 7 +++---- sandboxd/pool/fork.go | 4 +++- sandboxd/pool/hibernate.go | 12 +++++++----- sandboxd/pool/move_out.go | 19 +++++++++++++++---- sandboxd/pool/move_test.go | 37 ++++++++++++++++++++++++++++++++++--- 8 files changed, 68 insertions(+), 28 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 2270b3f7..df00f263 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -856,10 +856,9 @@ and never this node (else 400), `max_downtime_ms` optional with a default of 2000 and a floor: 400 or less answers 400 at once, because the target's receive and start alone are allowed that long. Answers `{"owner_addr": "", "rounds": n, "downtime_ms": d}` once the peer runs the guest and this node dropped the claim; `downtime_ms` is the pause -from the final capture to the peer's commit. While the move runs, release of -the claim answers 503 `sandbox moving to another node; retry`, and fork, -checkpoint, promote and hibernate answer the same during the rounds and wait -out the final round. Env writes, renew and data-plane calls keep working until +from the final capture to the peer's commit. While the move runs, release, fork, +checkpoint, promote, hibernate and volume changes of the claim answer 503 +`sandbox moving to another node; retry`. Env writes, renew and data-plane calls keep working until the final round and answer the same 503 during it, because the peer would miss them. Every error leaves the claim running here, except a 503 with the registry down after the final round paused the guest: the guest stays paused until the diff --git a/sandboxd/pool/capture.go b/sandboxd/pool/capture.go index bfee3b1c..8d26c15b 100644 --- a/sandboxd/pool/capture.go +++ b/sandboxd/pool/capture.go @@ -34,9 +34,6 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { if m.claimed[sb.ID] != sb { return ErrUnknownSandbox } - if m.midMoveLocked(sb) { - return ErrEvacuating - } if _, err := m.fenceWatch(sb); err != nil { return err } @@ -48,7 +45,9 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { // exportSource captures a claimed sandbox into exportDir and returns its snapshot, VM name and guest-env presence. func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir string) (snap, vmName string, guestEnv bool, err error) { - sb.Transition.Lock() + if lockErr := m.lockTransition(sb); lockErr != nil { + return "", "", false, lockErr + } if captureErr := m.captureAllowed(sb); captureErr != nil { sb.Transition.Unlock() return "", "", false, captureErr diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 6c331f0d..1abe2925 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -118,9 +118,6 @@ func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { if !ok { return ErrUnknownSandbox } - if m.movingNow(sb) { - return ErrEvacuating - } if m.archives != nil { return m.releaseRowed(ctx, id, sb) } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 8d8ffee5..068b4910 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -323,11 +323,10 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox) { } func (m *Manager) releaseRowed(ctx context.Context, id string, sb *types.Sandbox) error { - sb.Transition.Lock() - defer sb.Transition.Unlock() - if m.movingNow(sb) { - return ErrEvacuating + if err := m.lockTransition(sb); err != nil { + return err } + defer sb.Transition.Unlock() if err := m.resolveIntent(context.WithoutCancel(ctx), sb); err != nil { return err } diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index e8803ee2..0536669f 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -75,7 +75,9 @@ func (m *Manager) forkClones(ctx context.Context, sb *types.Sandbox, count int) // forkSource captures the fork source under the transition lock, against a racing hibernate. func (m *Manager) forkSource(ctx context.Context, sb *types.Sandbox) (create vmProvisioner, cleanup func(), inherited bool, err error) { - sb.Transition.Lock() + if err = m.lockTransition(sb); err != nil { + return nil, nil, false, err + } defer sb.Transition.Unlock() if err = m.captureAllowed(sb); err != nil { return nil, nil, false, err diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 4dd63c0c..9033ee49 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -19,7 +19,9 @@ func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { if !ok { return ErrUnknownSandbox } - sb.Transition.Lock() + if err := m.lockTransition(sb); err != nil { + return err + } defer sb.Transition.Unlock() return m.hibernateLocked(ctx, sb) } @@ -62,9 +64,6 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error if sb.ArchiveCk != "" { return nil // already off the node } - if m.movingNow(sb) { - return ErrEvacuating - } if _, err := m.fenceWatch(sb); err != nil { return err } @@ -72,8 +71,11 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return ErrFailed } m.mu.Lock() - volumeBlocked := m.hasVolumeHolds(sb) + moving, volumeBlocked := m.midMoveLocked(sb), m.hasVolumeHolds(sb) m.mu.Unlock() + if moving { + return ErrEvacuating + } if volumeBlocked { return ErrVolumeCapture } diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 96c69ba5..a8a479e6 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -387,10 +387,21 @@ func (m *Manager) movingFinalLocked(id string) bool { return mv != nil && mv.final } -func (m *Manager) movingNow(sb *types.Sandbox) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.midMoveLocked(sb) +func (m *Manager) lockTransition(sb *types.Sandbox) error { + midMove := func() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.midMoveLocked(sb) + } + if midMove() { + return ErrEvacuating + } + sb.Transition.Lock() + if midMove() { + sb.Transition.Unlock() + return ErrEvacuating + } + return nil } func (m *Manager) midMoveLocked(sb *types.Sandbox) bool { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index e4dcd25c..450ca859 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -210,7 +210,7 @@ func TestMoveKeepsThePausedGuestUntilItsAbortLands(t *testing.T) { t.Fatal("the source resumed its guest while the registry was down") } mc.moveReg.SetDown(false) - waitFor(t, func() bool { return !pausedOn(mc.s, sb.VMName) && !mc.s.movingNow(sb) }) + waitFor(t, func() bool { return !pausedOn(mc.s, sb.VMName) && !movingOn(mc.s, sb) }) assertStillOnSource(t, mc, sb) if rec, ok, _ := mc.moveReg.Get(t.Context(), sb.ID); ok { t.Errorf("record %+v after the abort, want it deleted", rec) @@ -497,7 +497,7 @@ func TestMoveEndsWhenAFailoverTookTheClaimFromTheSource(t *testing.T) { if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveTarget) { t.Fatalf("move of a claim another node took: %v, want ErrMoveTarget", err) } - if mc.s.movingNow(sb) { + if movingOn(mc.s, sb) { t.Fatal("the move still holds its slot") } if !sb.Transition.TryLock() { @@ -581,6 +581,31 @@ func TestMoveHoldsOffVolumeChanges(t *testing.T) { }) } +func TestMoveFinalRoundAnswersUserCallsAtOnce(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + cred := Cred{Token: sb.Token} + ctx := t.Context() + mc.p.commitHook = func(int) error { + _, forkErr := mc.s.Fork(ctx, sb.ID, cred, 1, time.Minute, "", "") + _, ckErr := mc.s.Checkpoint(ctx, sb.ID, cred, "ck", "") + _, _, promoteErr := mc.s.Promote(ctx, sb.ID, cred, "tpl", "") + for op, err := range map[string]error{ + "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, + "hibernate": mc.s.Hibernate(ctx, sb.ID, cred), + "release": mc.s.Release(ctx, sb.ID, cred), + } { + if !errors.Is(err, ErrEvacuating) { + t.Errorf("%s during the final round: %v, want ErrEvacuating", op, err) + } + } + return nil + } + if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move: %v", err) + } +} + func TestMoveGivesUpOnDeltasThatStopShrinking(t *testing.T) { for name, tt := range map[string]struct { sizes []int64 @@ -1093,7 +1118,7 @@ func TestMoveSourceBackFromAnOutageNeverResumesAClaimThatMovedOn(t *testing.T) { down.setDown(false) waitFor(t, func() bool { _, held := mc.s.byID(sb.ID) - return !held && !mc.s.movingNow(sb) + return !held && !movingOn(mc.s, sb) }) eng := fakeOf(mc.s) eng.mu.Lock() @@ -1396,6 +1421,12 @@ func dialEgress(t *testing.T, sock string) { _ = conn.Close() } +func movingOn(m *Manager, sb *types.Sandbox) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.midMoveLocked(sb) +} + func pausedOn(m *Manager, vm string) bool { eng := fakeOf(m) eng.mu.Lock() From c0256ee5003e2d06fc661a8632eb3315f298fc7b Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:22:06 +0800 Subject: [PATCH 37/53] review: one predicate per move question, one retry loop, and callers guard markMoveStarted moveBlocker reads midMoveLocked, env writes read pausedForMoveLocked so a writer never reaches a paused guest, dialPassive and SetInstanceMetadata read runningShaped, resumeMoves calls retryMoveOut, and the reap tests the move gate after the deadline. --- sandboxd/pool/claim.go | 5 ++--- sandboxd/pool/env.go | 2 +- sandboxd/pool/move.go | 6 +++++- sandboxd/pool/move_in.go | 15 ++++++++------- sandboxd/pool/move_out.go | 17 ++++++----------- sandboxd/pool/operator.go | 2 +- sandboxd/pool/reap.go | 2 +- sandboxd/pool/reconcile.go | 7 +------ 8 files changed, 25 insertions(+), 31 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 1abe2925..6314f327 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -257,9 +257,8 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. if !sb.Transition.TryLock() { return nil, ErrPaused } - failed := sb.Failed != "" - paused, sock := sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || sb.MoveIn != nil, sb.VsockSocket - if !paused && !failed { + failed, paused, sock := sb.Failed != "", !runningShaped(sb), sb.VsockSocket + if !paused { sb.Hold() } sb.Transition.Unlock() diff --git a/sandboxd/pool/env.go b/sandboxd/pool/env.go index f8dc5168..7e266d6c 100644 --- a/sandboxd/pool/env.go +++ b/sandboxd/pool/env.go @@ -50,7 +50,7 @@ func (m *Manager) writeEnv(ctx context.Context, id, tenant, verb string, next fu m.mu.Unlock() return ErrUnknownSandbox } - if sb.MoveIn != nil || !locked && m.movingFinalLocked(id) { + if m.pausedForMoveLocked(sb) { m.mu.Unlock() return ErrEvacuating } diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 0d09ddcb..36abf114 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -64,6 +64,10 @@ type outMove struct { pausedAt time.Time } +func (mv *outMove) owns(rec moves.Record, node string) bool { + return rec.From == node && rec.Epoch >= mv.epoch +} + func (mv *outMove) result() MoveResult { res := MoveResult{Rounds: mv.rounds} if !mv.pausedAt.IsZero() { @@ -106,7 +110,7 @@ func (m *Manager) sweepStaleMoves(ctx context.Context, now time.Time) { switch rec.State { case moves.Committed: if rec.To == m.nodeID { - m.markMoveStarted(ctx, rec.ID, rec.Epoch, nil) + m.markMoveStarted(ctx, rec.ID, rec.Epoch) } continue case moves.Pending, moves.Final: diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index 830e95eb..fd703a29 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -154,7 +154,9 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, committed m.recordUsage(ctx, usage.Event{Event: "vmm_exit", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) m.failLocked(ctx, sb, reasonMoveStartFailed) _ = os.RemoveAll(m.moveDir(sb.ID)) - m.markMoveStarted(ctx, sb.ID, committed, persistErr) + if committed != 0 && persistErr == nil { + m.markMoveStarted(ctx, sb.ID, committed) + } return } if sock, probeErr := m.probeReady(ctx, sb.VMName, sb.VsockSocket, claimProbeTimeout); probeErr != nil { @@ -178,15 +180,14 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, committed sb.Touch() m.recordUsage(ctx, usage.Event{Event: "move_in", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant, Deadline: m.deadlineOf(sb)}) _ = os.RemoveAll(m.moveDir(sb.ID)) - m.markMoveStarted(ctx, sb.ID, committed, err) + if committed != 0 && err == nil { + m.markMoveStarted(ctx, sb.ID, committed) + } logger.Infof(ctx, "sandbox %s moved here from %s", sb.ID, from) } -func (m *Manager) markMoveStarted(ctx context.Context, id string, committed int64, persistErr error) { - if committed == 0 || persistErr != nil { - return - } - if _, err := m.moves.Advance(ctx, id, committed, moves.Committed, moves.Started); err != nil { +func (m *Manager) markMoveStarted(ctx context.Context, id string, epoch int64) { + if _, err := m.moves.Advance(ctx, id, epoch, moves.Committed, moves.Started); err != nil { log.WithFunc("pool.markMoveStarted").Warnf(ctx, "mark the move of %s started, leaving it to the sweep: %v", id, err) } } diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index a8a479e6..e2696d35 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -189,7 +189,6 @@ func (m *Manager) markMoving(sb *types.Sandbox, mv *outMove) error { // moveBlocker names why sb cannot move, "" when it can; callers hold m.mu. func (m *Manager) moveBlocker(sb *types.Sandbox) string { _, archiving := m.archiving[sb.ID] - _, moving := m.moving[sb.ID] switch { case sb.ArchiveCk != "" || archiving: return "it is archived or archiving" @@ -199,7 +198,7 @@ func (m *Manager) moveBlocker(sb *types.Sandbox) string { return "it failed" case sb.Waking: return "it is waking" - case sb.MoveIn != nil || moving: + case m.midMoveLocked(sb): return "a move of it is already running" case hasVolumes(sb) || m.hasVolumeHolds(sb): return "it has volumes" @@ -266,7 +265,7 @@ func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM logger.Warnf(ctx, "read the move of %s, leaving it to the sweep: %v", sb.ID, err) return } - if ok && rec.From == m.nodeID && rec.Epoch >= mv.epoch && rec.State == moves.Started { + if ok && mv.owns(rec, m.nodeID) && rec.State == moves.Started { if err = m.moves.Delete(ctx, sb.ID, rec.Epoch); err != nil { logger.Warnf(ctx, "delete the move of %s, leaving it to the sweep: %v", sb.ID, err) } @@ -294,12 +293,12 @@ func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMov func (m *Manager) retryMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) { for { - time.Sleep(moveRetryInterval) _, err := m.settleMoveOut(ctx, sb, mv) if err == nil { return } log.WithFunc("pool.retryMoveOut").Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) + time.Sleep(moveRetryInterval) } } @@ -310,7 +309,7 @@ func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM if err != nil { return false, err } - ours := ok && rec.From == m.nodeID && rec.Epoch >= mv.epoch + ours := ok && mv.owns(rec, m.nodeID) switch { case !ours || rec.State == moves.Committed || rec.State == moves.Started: m.finishMoveOut(ctx, sb, mv) @@ -379,12 +378,8 @@ func (m *Manager) pausedForMove(sb *types.Sandbox) bool { } func (m *Manager) pausedForMoveLocked(sb *types.Sandbox) bool { - return sb.MoveIn != nil || m.movingFinalLocked(sb.ID) -} - -func (m *Manager) movingFinalLocked(id string) bool { - mv := m.moving[id] - return mv != nil && mv.final + mv := m.moving[sb.ID] + return sb.MoveIn != nil || mv != nil && mv.final } func (m *Manager) lockTransition(sb *types.Sandbox) error { diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index d9179506..231a6091 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -57,7 +57,7 @@ func (m *Manager) SetInstanceMetadata(ctx context.Context, id string, doc []byte return ErrArchived case sb.Failed != "": return ErrFailed - case sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.MoveIn != nil: + case !runningShaped(sb): return ErrPaused } return m.eng.WriteInstanceMetadata(ctx, sb.VsockSocket, doc) diff --git a/sandboxd/pool/reap.go b/sandboxd/pool/reap.go index 756c3418..4922ce46 100644 --- a/sandboxd/pool/reap.go +++ b/sandboxd/pool/reap.go @@ -36,7 +36,7 @@ func (m *Manager) reapOnce(ctx context.Context) { var unrows []*types.Sandbox for id, sb := range m.claimed { // A zero deadline means no expiry (an archived claim kept forever). - if m.midMoveLocked(sb) || sb.Deadline.IsZero() || !now.After(sb.Deadline) { + if sb.Deadline.IsZero() || !now.After(sb.Deadline) || m.midMoveLocked(sb) { continue } switch { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index b7a49b60..227ef047 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -166,12 +166,7 @@ func (m *Manager) resumeMoves(ctx context.Context, outs map[*types.Sandbox]*outM go func() { sb.Transition.Lock() mv.locked = true - for { - if _, err := m.settleMoveOut(ctx, sb, mv); err == nil { - return - } - time.Sleep(moveRetryInterval) - } + m.retryMoveOut(ctx, sb, mv) }() } for _, sb := range ins { From 8fe82e407f9efc4693f3ef2e68d6202c35656ebf Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:22:40 +0800 Subject: [PATCH 38/53] review: the move journal holds only what recovery reads MoveIn becomes a marker, the source node is logged from the record or the reservation. MoveOut shrinks to its epoch and series and embeds in outMove. MigrateRound drops the round and paused fields nothing reads, the source stops scrubbing a field the target scrubs, and engine/migrate.go loses an err shadow. --- sandboxd/engine/migrate.go | 2 +- sandboxd/engine/migrate_test.go | 2 +- sandboxd/pool/claims.go | 2 +- sandboxd/pool/liveness.go | 2 +- sandboxd/pool/move.go | 7 +++---- sandboxd/pool/move_in.go | 17 ++++++++--------- sandboxd/pool/move_out.go | 29 ++++++++++++++--------------- sandboxd/pool/move_test.go | 22 +++++++++++----------- sandboxd/pool/pool_test.go | 2 +- sandboxd/pool/reconcile.go | 6 +++--- sandboxd/types/types.go | 15 +++------------ 11 files changed, 47 insertions(+), 59 deletions(-) diff --git a/sandboxd/engine/migrate.go b/sandboxd/engine/migrate.go index d169c189..c07d453d 100644 --- a/sandboxd/engine/migrate.go +++ b/sandboxd/engine/migrate.go @@ -42,7 +42,7 @@ func (e *Engine) MigrateCapture(ctx context.Context, vmName, series string, fina return types.MigrateRound{}, err } var round types.MigrateRound - if err := json.Unmarshal(out, &round); err != nil { + if err = json.Unmarshal(out, &round); err != nil { return types.MigrateRound{}, fmt.Errorf("parse migrate capture: %w", err) } return round, nil diff --git a/sandboxd/engine/migrate_test.go b/sandboxd/engine/migrate_test.go index a7740a65..e94f035f 100644 --- a/sandboxd/engine/migrate_test.go +++ b/sandboxd/engine/migrate_test.go @@ -34,7 +34,7 @@ func TestMigrateCallsBuildCocoonsArgsAndParseItsJSON(t *testing.T) { if err != nil { t.Fatalf("final capture: %v", err) } - want := types.MigrateRound{Series: "s1", Dir: "/run/xfer/s1", Round: 2, Paused: true, Files: []types.MigrateFile{{Name: "state.json", Size: 3}, {Name: "cow.raw", Size: 4096}}} + want := types.MigrateRound{Series: "s1", Dir: "/run/xfer/s1", Files: []types.MigrateFile{{Name: "state.json", Size: 3}, {Name: "cow.raw", Size: 4096}}} if !reflect.DeepEqual(round, want) { t.Errorf("round = %+v, want %+v", round, want) } diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index e01f4cbd..195a04d4 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -65,7 +65,7 @@ type claimDTO struct { RestartedAt time.Time `json:"restarted_at,omitzero"` Failed string `json:"failed,omitempty"` MoveOut *types.MoveOut `json:"move_out,omitempty"` - MoveIn *types.MoveIn `json:"move_in,omitempty"` + MoveIn bool `json:"move_in,omitzero"` } // claimRow keeps its encoded member until set replaces the row; set never edits one. diff --git a/sandboxd/pool/liveness.go b/sandboxd/pool/liveness.go index 1a65e691..0e3084e2 100644 --- a/sandboxd/pool/liveness.go +++ b/sandboxd/pool/liveness.go @@ -316,7 +316,7 @@ func (m *Manager) vmSettledLocked(sb *types.Sandbox) bool { // runningShaped reports whether sb's VM should be up; callers hold m.mu. func runningShaped(sb *types.Sandbox) bool { - return sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.Failed == "" && sb.MoveIn == nil + return sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.Failed == "" && !sb.MoveIn } // coldBootBlocker names why sb's VM cannot be cold-booted, "" when it can. diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 36abf114..868dca1c 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -54,10 +54,9 @@ type MovePeer interface { // outMove is a source's move in flight; final is guarded by m.mu, the rest belongs to the goroutine driving the move. type outMove struct { + types.MoveOut addr string to string - epoch int64 - series string rounds int final bool locked bool @@ -65,7 +64,7 @@ type outMove struct { } func (mv *outMove) owns(rec moves.Record, node string) bool { - return rec.From == node && rec.Epoch >= mv.epoch + return rec.From == node && rec.Epoch >= mv.Epoch } func (mv *outMove) result() MoveResult { @@ -134,5 +133,5 @@ func (m *Manager) moveHeld(id string) bool { _, out := m.moving[id] _, in := m.incoming[id] sb := m.claimed[id] - return out || in || sb != nil && sb.MoveIn != nil + return out || in || sb != nil && sb.MoveIn } diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index fd703a29..7eca04b6 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -104,7 +104,7 @@ func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch m.destroy(ctx, name) return fmt.Errorf("%w: the reservation for %s ended", ErrNotMovable, id) } - sb.MoveIn = &types.MoveIn{From: admitted.from, Epoch: epoch} + sb.MoveIn = true delete(m.incoming, id) m.addClaimLocked(sb) js := m.store.set(sb) @@ -122,7 +122,7 @@ func (m *Manager) CommitMove(ctx context.Context, id string, claim []byte, epoch go m.settleMoveIn(ctx, sb) return fmt.Errorf("move %s: %w", id, err) } - m.startMoveIn(ctx, sb, committed) + m.startMoveIn(ctx, sb, admitted.from, committed) return nil } @@ -138,9 +138,8 @@ func (m *Manager) DropMove(id string) { } // startMoveIn starts a committed claim's guest and serves it, then marks the record started at committed when that is its epoch; a guest that cannot start fails the claim here, never back to the source. The caller holds sb.Transition. -func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, committed int64) { +func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, from string, committed int64) { logger := log.WithFunc("pool.startMoveIn") - from := sb.MoveIn.From var err error for range 2 { if err = m.eng.Start(ctx, sb.VMName); err == nil { @@ -149,7 +148,7 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, committed logger.Warnf(ctx, "start moved claim %s: %v", sb.ID, err) } if err != nil { - _, persistErr := m.commitIfLive(ctx, sb, func() { sb.MoveIn = nil }) + _, persistErr := m.commitIfLive(ctx, sb, func() { sb.MoveIn = false }) sb.ExitRecorded = true m.recordUsage(ctx, usage.Event{Event: "vmm_exit", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) m.failLocked(ctx, sb, reasonMoveStartFailed) @@ -169,7 +168,7 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, committed if proxyErr := m.out.ArmProxy(ctx, sb); proxyErr != nil { logger.Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) } - live, err := m.commitIfLive(ctx, sb, func() { sb.MoveIn = nil }) + live, err := m.commitIfLive(ctx, sb, func() { sb.MoveIn = false }) if !live { m.out.Disarm(sb.ID, true) return @@ -213,7 +212,7 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - pending := m.claimed[sb.ID] == sb && sb.MoveIn != nil + pending := m.claimed[sb.ID] == sb && sb.MoveIn m.mu.Unlock() if !pending { return true @@ -226,10 +225,10 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { toHere := ok && rec.To == m.nodeID switch { case toHere && rec.State == moves.Committed: - m.startMoveIn(ctx, sb, rec.Epoch) + m.startMoveIn(ctx, sb, rec.From, rec.Epoch) return true case toHere && rec.State == moves.Started: - m.startMoveIn(ctx, sb, 0) + m.startMoveIn(ctx, sb, rec.From, 0) return true case toHere && (rec.State == moves.Pending || rec.State == moves.Final): if _, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted); err != nil { diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index e2696d35..a3c2dbce 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -62,7 +62,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b if err != nil { return err } - mv.epoch = epoch + mv.Epoch = epoch if err = m.journalMoveOut(ctx, sb, mv); err != nil { return err } @@ -70,7 +70,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b if err != nil { return fmt.Errorf("%w: baseline capture: %w", ErrNotMovable, err) } - mv.series = round.Series + mv.Series = round.Series if err = m.journalMoveOut(ctx, sb, mv); err != nil { return err } @@ -93,7 +93,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b } } previous = size - if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, false); err != nil { + if round, err = m.eng.MigrateCapture(ctx, vm, mv.Series, false); err != nil { return fmt.Errorf("delta capture: %w", err) } } @@ -108,10 +108,10 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b return err } mv.pausedAt = time.Now() - if round, err = m.eng.MigrateCapture(ctx, vm, mv.series, true); err != nil { + if round, err = m.eng.MigrateCapture(ctx, vm, mv.Series, true); err != nil { return fmt.Errorf("final capture: %w", err) } - final, err := m.moves.Advance(ctx, sb.ID, mv.epoch, moves.Pending, moves.Final) + final, err := m.moves.Advance(ctx, sb.ID, mv.Epoch, moves.Pending, moves.Final) if err != nil { return err } @@ -121,7 +121,6 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b m.mu.Lock() dto := dtoOf(sb) m.mu.Unlock() - dto.MoveOut = nil claim, err := json.Marshal(&dto) if err != nil { return fmt.Errorf("encode claim: %w", err) @@ -139,7 +138,7 @@ func (m *Manager) beginMove(ctx context.Context, sb *types.Sandbox, to string) ( return 0, err } m.mu.Lock() - settled := m.claimed[sb.ID] == sb && sb.MoveIn == nil && sb.MoveOut == nil + settled := m.claimed[sb.ID] == sb && !sb.MoveIn && sb.MoveOut == nil m.mu.Unlock() if !ok || rec.State != moves.Committed || !settled { return 0, moves.ErrLost @@ -209,8 +208,8 @@ func (m *Manager) moveBlocker(sb *types.Sandbox) string { } func (m *Manager) journalMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) error { - journal := &types.MoveOut{To: mv.to, Epoch: mv.epoch, Series: mv.series} - live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = journal }) + journal := mv.MoveOut + live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = &journal }) if !live { return ErrUnknownSandbox } @@ -274,7 +273,7 @@ func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM // failMoveOut resolves a failed move through its record: only proof that the move stayed here resumes the guest, anything else drops the claim. func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, cause error) (MoveResult, error) { - if mv.epoch == 0 { + if mv.Epoch == 0 { _ = m.abortMoveOut(ctx, sb, mv, 0) return MoveResult{}, cause } @@ -334,9 +333,9 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo m.mu.Lock() vm, live := sb.VMName, m.claimed[sb.ID] == sb m.mu.Unlock() - if live && mv.series != "" { - if err := m.eng.MigrateAbort(ctx, vm, mv.series); err != nil { - return fmt.Errorf("abort series %s: %w", mv.series, err) + if live && mv.Series != "" { + if err := m.eng.MigrateAbort(ctx, vm, mv.Series); err != nil { + return fmt.Errorf("abort series %s: %w", mv.Series, err) } } if mv.addr != "" && mv.to != "" { @@ -379,7 +378,7 @@ func (m *Manager) pausedForMove(sb *types.Sandbox) bool { func (m *Manager) pausedForMoveLocked(sb *types.Sandbox) bool { mv := m.moving[sb.ID] - return sb.MoveIn != nil || mv != nil && mv.final + return sb.MoveIn || mv != nil && mv.final } func (m *Manager) lockTransition(sb *types.Sandbox) error { @@ -401,7 +400,7 @@ func (m *Manager) lockTransition(sb *types.Sandbox) error { func (m *Manager) midMoveLocked(sb *types.Sandbox) bool { _, moving := m.moving[sb.ID] - return moving || sb.MoveIn != nil + return moving || sb.MoveIn } func roundBytes(round types.MigrateRound) int64 { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 450ca859..599a89e9 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -55,7 +55,7 @@ func TestMoveHandsARunningClaimToThePeerWithBothNodesCountingIt(t *testing.T) { t.Errorf("counts after the move %v, want 0/0/0 on the source and 1/1/1 on the target", got) } moved, ok := mc.t.byID(sb.ID) - if !ok || moved.Token != sb.Token || moved.Tenant != "acme" || moved.MoveIn != nil || !moved.Deadline.Equal(sb.Deadline) { + if !ok || moved.Token != sb.Token || moved.Tenant != "acme" || moved.MoveIn || !moved.Deadline.Equal(sb.Deadline) { t.Fatalf("target claim %+v %v, want the source's record started", moved, ok) } if _, held := mc.s.byID(sb.ID); held || !fakeOf(mc.s).removed(vm) { @@ -181,7 +181,7 @@ func TestMoveWhoseCommitAnswerIsLostResolvesThroughTheRecord(t *testing.T) { if _, held := mc.s.byID(sb.ID); held { t.Error("the source kept a claim the target committed") } - if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn != nil { + if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn { t.Errorf("target claim %+v %v, want it started", moved, ok) } if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { @@ -742,9 +742,9 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { } sb := &types.Sandbox{ID: "sb_moved", VMName: vm, Key: testKey, Token: "tok", Deadline: time.Now().Add(time.Hour)} if tt.node == "a" { - sb.MoveOut = &types.MoveOut{To: "b", Epoch: journalEpoch, Series: series} + sb.MoveOut = &types.MoveOut{Epoch: journalEpoch, Series: series} } else { - sb.MoveIn = &types.MoveIn{From: "a", Epoch: journalEpoch} + sb.MoveIn = true } dataDir := t.TempDir() seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) @@ -752,7 +752,7 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { t.Fatal(err) } eng := newFakeEngine() - eng.vms[vm], eng.stopped[vm] = "/vsock/"+vm, sb.MoveIn != nil + eng.vms[vm], eng.stopped[vm] = "/vsock/"+vm, sb.MoveIn eng.migrateSeries[vm] = series m := c.nodeOn(t, eng, tt.node, dataDir, keepAnHour) m.liftFence() @@ -768,7 +768,7 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { if ok != tt.wantClaim { t.Fatalf("claim held %v, want %v", ok, tt.wantClaim) } - if ok && (held.MoveOut != nil || held.MoveIn != nil) { + if ok && (held.MoveOut != nil || held.MoveIn) { t.Errorf("claim journal out %+v in %+v, want both cleared", held.MoveOut, held.MoveIn) } eng.mu.Lock() @@ -817,8 +817,8 @@ func TestStaleMoveRecordsAreDeletedOnlyByTheirSource(t *testing.T) { func TestMoveStaleSweepLeavesACommittedRecordItsTargetStillHoldsMoveInFor(t *testing.T) { mc := newMoveCluster(t) ctx, reg := t.Context(), mc.moveReg - epoch := seedMoveRecord(t, reg, "sb_held", "a", "b", moves.Committed) - sb := &types.Sandbox{ID: "sb_held", VMName: "sbx-held", Key: testKey, Deadline: time.Now().Add(time.Hour), MoveIn: &types.MoveIn{From: "a", Epoch: epoch}} + seedMoveRecord(t, reg, "sb_held", "a", "b", moves.Committed) + sb := &types.Sandbox{ID: "sb_held", VMName: "sbx-held", Key: testKey, Deadline: time.Now().Add(time.Hour), MoveIn: true} mc.t.mu.Lock() mc.t.addClaimLocked(sb) mc.t.store.set(sb) @@ -950,8 +950,8 @@ func TestMoveBeginLeavesARecordOnlyItsHolderMaySettle(t *testing.T) { }{ "a move in flight": {reach: moves.Pending}, "final": {reach: moves.Final}, - "held under a journal": {reach: moves.Committed, edit: func(sb *types.Sandbox) { sb.MoveOut = &types.MoveOut{To: "b", Epoch: 1} }}, - "received and not started": {reach: moves.Committed, edit: func(sb *types.Sandbox) { sb.MoveIn = &types.MoveIn{From: "a", Epoch: 1} }}, + "held under a journal": {reach: moves.Committed, edit: func(sb *types.Sandbox) { sb.MoveOut = &types.MoveOut{Epoch: 1} }}, + "received and not started": {reach: moves.Committed, edit: func(sb *types.Sandbox) { sb.MoveIn = true }}, } { t.Run(name, func(t *testing.T) { mc := newMoveCluster(t) @@ -969,7 +969,7 @@ func TestMoveBeginLeavesARecordOnlyItsHolderMaySettle(t *testing.T) { t.Errorf("record %+v, want %s untouched", rec, tt.reach) } mc.s.mu.Lock() - sb.MoveOut, sb.MoveIn = nil, nil + sb.MoveOut, sb.MoveIn = nil, false mc.s.mu.Unlock() }) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 2d239618..b4538843 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1886,7 +1886,7 @@ func (f *fakeEngine) MigrateCapture(_ context.Context, vmName, series string, fi size, f.migrateSizes = f.migrateSizes[0], f.migrateSizes[1:] } f.paused[vmName] = final - return types.MigrateRound{Series: series, Dir: dir, Round: round, Paused: final, Files: []types.MigrateFile{{Name: "state.json", Size: size}}}, nil + return types.MigrateRound{Series: series, Dir: dir, Files: []types.MigrateFile{{Name: "state.json", Size: size}}}, nil } func (f *fakeEngine) MigrateReceive(_ context.Context, fromDir, name string, _ types.PoolKey) (types.VMRecord, error) { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 227ef047..2d78af08 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -111,7 +111,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { // A crash mid-export leaves a fork-* staging dir nothing in this life reuses _ = utils.RemoveDirEntries(m.dataDir, func(name string) bool { return strings.HasPrefix(name, "fork-") }) _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, readerDir), nil) - _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, movesDir), func(id string) bool { return claims[id] == nil || claims[id].MoveIn == nil }) + _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, movesDir), func(id string) bool { return claims[id] == nil || !claims[id].MoveIn }) if fenced { m.reclaimOrphanArchiveCks(ctx, claims) @@ -151,9 +151,9 @@ func (m *Manager) pendingMovesLocked() (map[*types.Sandbox]*outMove, []*types.Sa for id, sb := range m.claimed { switch { case sb.MoveOut != nil: - mv := &outMove{to: sb.MoveOut.To, epoch: sb.MoveOut.Epoch, series: sb.MoveOut.Series, final: true} + mv := &outMove{MoveOut: *sb.MoveOut, final: true} m.moving[id], outs[sb] = mv, mv - case sb.MoveIn != nil: + case sb.MoveIn: ins = append(ins, sb) } } diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index ec2a83fb..69372004 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -307,7 +307,7 @@ type Sandbox struct { // MoveOut journals a live move this node runs as the source; nil when none. MoveOut *MoveOut `json:"move_out,omitempty"` // MoveIn marks a claim a live move delivered here and this node has not started yet. - MoveIn *MoveIn `json:"move_in,omitempty"` + MoveIn bool `json:"move_in,omitzero"` // RestartLog holds the recent restart attempts that the loop budget counts; guarded by Transition. RestartLog []time.Time `json:"-"` // ExitRecorded marks a VMM exit already journaled as vmm_exit; guarded by Transition. @@ -458,31 +458,22 @@ func (r VMRecord) TapDevice() string { return r.NetworkConfigs[0].TAP } -// MoveOut is the source's journal of a live move: the target node, the record's first epoch and cocoon's series. +// MoveOut is the source's journal of a live move: the record's first epoch and cocoon's series. type MoveOut struct { - To string `json:"to"` Epoch int64 `json:"epoch"` Series string `json:"series,omitempty"` } -// MoveIn is the target's journal of a received claim: the source node and the epoch its commit wrote. -type MoveIn struct { - From string `json:"from"` - Epoch int64 `json:"epoch"` -} - // MigrateFile is one file a migrate round wrote, named relative to the round's Dir. type MigrateFile struct { Name string `json:"name"` Size int64 `json:"size"` } -// MigrateRound is what `cocoon vm migrate capture` prints for one round; round 0 is the baseline. +// MigrateRound is what `cocoon vm migrate capture` prints for one round. type MigrateRound struct { Series string `json:"series"` Dir string `json:"dir"` - Round int `json:"round"` - Paused bool `json:"paused"` Files []MigrateFile `json:"files"` } From 5e7c56e4bae03300cfe73d1b1a6b039dac761a39 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:23:06 +0800 Subject: [PATCH 39/53] review: a round's PUT is one peerCall peerCall takes a body and a timeout, zero leaving the deadline to the caller. The round route answers 204, so SendRound now accepts 200 and 204 rather than any 2xx. --- sandboxd/store/peer/adopt.go | 2 +- sandboxd/store/peer/broadcast.go | 16 +++++++++------- sandboxd/store/peer/move.go | 24 +----------------------- 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/sandboxd/store/peer/adopt.go b/sandboxd/store/peer/adopt.go index e99aae0d..70f776ba 100644 --- a/sandboxd/store/peer/adopt.go +++ b/sandboxd/store/peer/adopt.go @@ -17,7 +17,7 @@ type Adopter struct { func (a *Adopter) Adopt(ctx context.Context, id, token string) bool { logger := log.WithFunc("peer.Adopter.Adopt") for _, addr := range a.Peers() { - err := peerCall(ctx, http.MethodGet, URL(addr, "/v1/sandboxes/"+url.PathEscape(id)+"/owner"), token, http.StatusOK) + err := peerCall(ctx, http.MethodGet, URL(addr, "/v1/sandboxes/"+url.PathEscape(id)+"/owner"), token, nil, peerCallTimeout, http.StatusOK) if err == nil { return true } diff --git a/sandboxd/store/peer/broadcast.go b/sandboxd/store/peer/broadcast.go index 142283a1..8ca2915f 100644 --- a/sandboxd/store/peer/broadcast.go +++ b/sandboxd/store/peer/broadcast.go @@ -28,7 +28,7 @@ func (b *Broadcaster) Delete(ctx context.Context, id string) { for _, addr := range addrs { wg.Go(func() { // 404 means the peer never healed a copy — not a failure worth a retry. - if err := peerCall(ctx, http.MethodDelete, checkpointURL(addr, id, "?no_forward=1"), b.Token, http.StatusNoContent, http.StatusNotFound); err != nil { + if err := peerCall(ctx, http.MethodDelete, checkpointURL(addr, id, "?no_forward=1"), b.Token, nil, peerCallTimeout, http.StatusNoContent, http.StatusNotFound); err != nil { logger.Warnf(ctx, "broadcast delete %s to %s: %v", id, addr, err) } }) @@ -36,12 +36,14 @@ func (b *Broadcaster) Delete(ctx context.Context, id string) { wg.Wait() } -// peerCall bounds one bodyless request to a peer; any status outside ok is an error. -func peerCall(ctx context.Context, method, target, token string, ok ...int) error { - ctx, cancel := context.WithTimeout(ctx, peerCallTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, method, target, nil) +// peerCall bounds one request to a peer unless timeout is zero; any status outside ok is an error. +func peerCall(ctx context.Context, method, target, token string, body io.Reader, timeout time.Duration, ok ...int) error { + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + req, err := http.NewRequestWithContext(ctx, method, target, body) if err != nil { return err } diff --git a/sandboxd/store/peer/move.go b/sandboxd/store/peer/move.go index 2385ba9f..1c8d5096 100644 --- a/sandboxd/store/peer/move.go +++ b/sandboxd/store/peer/move.go @@ -25,7 +25,7 @@ func SendRound(ctx context.Context, addr, token, path, dir string, names []strin _ = pw.CloseWithError(err) written <- err }() - err := putRound(ctx, URL(addr, path), token, pr) + err := peerCall(ctx, http.MethodPut, URL(addr, path), token, pr, 0, http.StatusOK, http.StatusNoContent) _ = pr.Close() if writeErr := <-written; writeErr != nil && !errors.Is(writeErr, io.ErrClosedPipe) { return fmt.Errorf("send round to %s: %w", addr, writeErr) @@ -45,28 +45,6 @@ func ReceiveRound(r io.Reader, dst string) error { return err } -func putRound(ctx context.Context, target, token string, body io.Reader) error { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body) - if err != nil { - return err - } - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - if resp.StatusCode/100 != 2 { - return fmt.Errorf("unexpected status %d", resp.StatusCode) - } - return nil -} - func writeRound(dir string, names []string, w io.Writer) error { tw := tar.NewWriter(w) scratch := sync.OnceValue(func() []byte { return make([]byte, store.DigestChunkSize) }) From 47bd8cdc86f94071f7ef5aa4d388c080549fb636 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:23:57 +0800 Subject: [PATCH 40/53] review: the old replica is deleted after the move ends, and tests share Reach and dialEgress The blob delete no longer runs under Transition while the guest is paused. On a crash in between, Reconcile's orphan reclaim finds it through the loaded claim. --- sandboxd/moves/movestest/movestest.go | 31 +++++++++++--------- sandboxd/pool/liveness_test.go | 6 +--- sandboxd/pool/move.go | 1 + sandboxd/pool/move_out.go | 17 ++++++----- sandboxd/pool/move_test.go | 42 +++++++++++++++++---------- 5 files changed, 56 insertions(+), 41 deletions(-) diff --git a/sandboxd/moves/movestest/movestest.go b/sandboxd/moves/movestest/movestest.go index 03c1311c..46cb3c3f 100644 --- a/sandboxd/moves/movestest/movestest.go +++ b/sandboxd/moves/movestest/movestest.go @@ -129,6 +129,23 @@ func Contract(t *testing.T, open Opener) { t.Run("stale lists records written before a time", func(t *testing.T) { testStale(t, open(t)) }) } +// Reach advances a record begun at epoch to state and returns its last epoch. +func Reach(t *testing.T, r moves.Registry, id string, epoch int64, state moves.State) int64 { + t.Helper() + path := map[moves.State][]moves.State{ + moves.Final: {moves.Final}, + moves.Committed: {moves.Final, moves.Committed}, + moves.Started: {moves.Final, moves.Committed, moves.Started}, + moves.Aborted: {moves.Aborted}, + }[state] + from := moves.Pending + for _, to := range path { + epoch = mustAdvance(t, r, id, epoch, from, to) + from = to + } + return epoch +} + func testBeginGet(t *testing.T, r moves.Registry) { ctx := t.Context() if _, ok, err := r.Get(ctx, "sb_1"); ok || err != nil { @@ -351,17 +368,5 @@ func mustAdvance(t *testing.T, r moves.Registry, id string, expect int64, from, func mustReach(t *testing.T, r moves.Registry, id string, state moves.State) int64 { t.Helper() - epoch := mustBegin(t, r, id) - path := map[moves.State][]moves.State{ - moves.Final: {moves.Final}, - moves.Committed: {moves.Final, moves.Committed}, - moves.Started: {moves.Final, moves.Committed, moves.Started}, - moves.Aborted: {moves.Aborted}, - }[state] - from := moves.Pending - for _, to := range path { - epoch = mustAdvance(t, r, id, epoch, from, to) - from = to - } - return epoch + return Reach(t, r, id, mustBegin(t, r, id), state) } diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 4f197b2f..01c40439 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -247,11 +247,7 @@ func TestRestartKeepsAnEgressDoorThatSurvivedTheVMM(t *testing.T) { m.recoverVMM(t.Context(), sb) - conn, err := net.Dial("unix", engine.EgressSocketPath(sb.VsockSocket)) - if err != nil { - t.Fatalf("egress door after the restart: %v", err) - } - _ = conn.Close() + dialEgress(t, sb.VsockSocket) } func TestRestartRedeliversTheGuestEnv(t *testing.T) { diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 868dca1c..750d4509 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -57,6 +57,7 @@ type outMove struct { types.MoveOut addr string to string + replica string rounds int final bool locked bool diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index a3c2dbce..9e3107db 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -104,7 +104,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b mv.locked = true ctx, cancel := context.WithTimeout(ctx, moveFinalTimeout) defer cancel() - if err = m.unrowForMove(ctx, sb); err != nil { + if err = m.unrowForMove(ctx, sb, mv); err != nil { return err } mv.pausedAt = time.Now() @@ -220,7 +220,7 @@ func (m *Manager) journalMoveOut(ctx context.Context, sb *types.Sandbox, mv *out } // unrowForMove deletes a replicated claim's row and replica, so no third node adopts a stale copy while the target runs it; the caller holds sb.Transition. -func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox) error { +func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox, mv *outMove) error { if m.archives == nil { return nil } @@ -238,13 +238,11 @@ func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox) error { } else if err != nil { return err } + mv.replica = replica live, err := m.commitIfLive(ctx, sb, func() { sb.ReplicaCk, sb.ArchiveEpoch = "", 0 }) if !live { return ErrUnknownSandbox } - if replica != "" { - m.deleteOrphanArchiveCk(ctx, replica) - } return err } @@ -257,7 +255,7 @@ func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM teardown() m.recordUsage(ctx, usage.Event{Event: "move_out", ID: sb.ID, VMName: vm, Tenant: sb.Tenant}) } - m.endMoveOut(sb, mv) + m.endMoveOut(ctx, sb, mv) logger := log.WithFunc("pool.finishMoveOut") rec, ok, err := m.moves.Get(ctx, sb.ID) if err != nil { @@ -344,7 +342,7 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo } } _, journalErr := m.commitIfLive(ctx, sb, func() { sb.MoveOut = nil }) - m.endMoveOut(sb, mv) + m.endMoveOut(ctx, sb, mv) if journalErr != nil { logger.Warnf(ctx, "clear the move journal of %s, leaving its record to the sweep: %v", sb.ID, journalErr) return nil @@ -357,7 +355,7 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo return nil } -func (m *Manager) endMoveOut(sb *types.Sandbox, mv *outMove) { +func (m *Manager) endMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) { m.mu.Lock() if m.moving[sb.ID] == mv { delete(m.moving, sb.ID) @@ -367,6 +365,9 @@ func (m *Manager) endMoveOut(sb *types.Sandbox, mv *outMove) { mv.locked = false sb.Transition.Unlock() } + if mv.replica != "" { + m.deleteOrphanArchiveCk(ctx, mv.replica) + } } // pausedForMove reports whether sb's guest is paused by a move, in a source's final round or received and not started, when dials and wakes answer ErrEvacuating. diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 599a89e9..874d3a84 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -420,6 +420,31 @@ func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { } } +func TestMoveAbortedAfterDroppingTheRowPutsItBackOnTheNextRound(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) + sb := mustClaim(t, mc.s, testKey) + replicateNow(t, mc.s, sb) + mc.s.mu.Lock() + replica := sb.ReplicaCk + mc.s.mu.Unlock() + mc.p.commitHook = func(int) error { return ErrMoveTarget } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrMoveTarget) { + t.Fatalf("move: %v, want ErrMoveTarget", err) + } + assertStillOnSource(t, mc, sb) + if ckExists(t, mc.s, replica) { + t.Error("the aborted move left the replica it dropped") + } + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); ok { + t.Fatal("the row survived the final round") + } + mc.p.commitHook = nil + replicateNow(t, mc.s, sb) + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok { + t.Error("the next round did not put the row back") + } +} + func TestMoveHoldsOffCapturesReleasesAndEvacuationWhileItRuns(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) @@ -1317,24 +1342,11 @@ func (r *lossyMoves) setDown(down bool) { r.down.Store(down) } func seedMoveRecord(t *testing.T, reg moves.Registry, id, from, to string, state moves.State) int64 { t.Helper() - ctx := t.Context() - epoch, err := reg.Begin(ctx, id, from, to) + epoch, err := reg.Begin(t.Context(), id, from, to) if err != nil { t.Fatalf("begin %s: %v", id, err) } - path := map[moves.State][]moves.State{ - moves.Final: {moves.Final}, - moves.Committed: {moves.Final, moves.Committed}, - moves.Started: {moves.Final, moves.Committed, moves.Started}, - moves.Aborted: {moves.Aborted}, - }[state] - at, reached := epoch, moves.Pending - for _, next := range path { - if at, err = reg.Advance(ctx, id, at, reached, next); err != nil { - t.Fatalf("advance %s to %s: %v", id, next, err) - } - reached = next - } + movestest.Reach(t, reg, id, epoch, state) return epoch } From 7743fee9bfbec56bd8807a9452e1ec0d2966b91d Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:23:59 +0800 Subject: [PATCH 41/53] review: move comments to the budget --- sandboxd/pool/move.go | 5 ++--- sandboxd/pool/move_in.go | 12 ++++-------- sandboxd/pool/move_out.go | 17 ++++++++--------- sandboxd/server/move.go | 2 +- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 750d4509..f9729b5c 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -19,7 +19,6 @@ const ( MoveStartAllowance = 400 * time.Millisecond movesDir = "moves" - // A reservation with no round or commit for this long belongs to a source that gave up. moveReservationIdle = 2 * time.Minute moveRetryInterval = 2 * time.Second moveDeltaLimit = 4 @@ -81,7 +80,7 @@ func (m *Manager) WithMovePeer(p MovePeer) { m.movePeer = p } -// moveBudgetErr refuses a move past move_concurrency, counting both directions; callers hold m.mu. +// Callers hold m.mu. func (m *Manager) moveBudgetErr() error { limit := m.view.Load().moveConcurrency if n := len(m.moving) + len(m.incoming); n >= limit { @@ -94,7 +93,7 @@ func (m *Manager) moveDir(id string) string { return filepath.Join(m.dataDir, movesDir, id) } -// sweepStaleMoves ends this node's stale moves no move in flight holds: a target marks a committed one started, either side aborts an open one, and only the source deletes an ended one. +// A target only marks a committed move started; only the source deletes an ended one. func (m *Manager) sweepStaleMoves(ctx context.Context, now time.Time) { logger := log.WithFunc("pool.sweepStaleMoves") recs, err := m.moves.Stale(ctx, now.Add(-moveStaleAge)) diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index 7eca04b6..7edf5e53 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -19,7 +19,7 @@ import ( const reasonMoveStartFailed = "vmm did not start after a live move" -// inMove is a target's reservation for a move in flight; guarded by m.mu. +// Guarded by m.mu. type inMove struct { tenant string from string @@ -137,7 +137,7 @@ func (m *Manager) DropMove(id string) { } } -// startMoveIn starts a committed claim's guest and serves it, then marks the record started at committed when that is its epoch; a guest that cannot start fails the claim here, never back to the source. The caller holds sb.Transition. +// A guest that cannot start fails the claim here, never back to the source; the caller holds sb.Transition. func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, from string, committed int64) { logger := log.WithFunc("pool.startMoveIn") var err error @@ -191,7 +191,6 @@ func (m *Manager) markMoveStarted(ctx context.Context, id string, epoch int64) { } } -// dropMoveIn forgets a received claim whose move did not commit and removes its VM. func (m *Manager) dropMoveIn(ctx context.Context, sb *types.Sandbox) { if holds, ok := m.forgetClaim(ctx, sb); ok { m.out.Disarm(sb.ID, true) @@ -200,14 +199,13 @@ func (m *Manager) dropMoveIn(ctx context.Context, sb *types.Sandbox) { _ = os.RemoveAll(m.moveDir(sb.ID)) } -// settleMoveIn resolves a received claim through its record until the registry answers. func (m *Manager) settleMoveIn(ctx context.Context, sb *types.Sandbox) { for !m.resolveMoveIn(ctx, sb) { time.Sleep(moveRetryInterval) } } -// resolveMoveIn starts a received claim whose record committed to this node, ends a move its source left open and drops any other; false while the registry is down. +// False while the registry is down. func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -259,7 +257,6 @@ func (m *Manager) endReceive(id string, in *inMove) (current, reserved bool) { return held == in, held != nil } -// dropIdleReservations ends the reservations of sources that went quiet. func (m *Manager) dropIdleReservations() { now := time.Now() var idle []string @@ -275,7 +272,7 @@ func (m *Manager) dropIdleReservations() { } } -// incomingOf counts tenant's reservations; callers hold m.mu. +// Callers hold m.mu. func (m *Manager) incomingOf(tenant string) int { n := 0 for _, in := range m.incoming { @@ -286,7 +283,6 @@ func (m *Manager) incomingOf(tenant string) int { return n } -// admitMoveLocked counts a move like a new claim and reserves its slot; callers hold m.mu. func (m *Manager) admitMoveLocked(id string, a MoveAdmission) error { if _, held := m.claimed[id]; held { return fmt.Errorf("%w: this node already holds %s", ErrNotMovable, id) diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 9e3107db..a009e0ea 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -45,7 +45,7 @@ func (m *Manager) Move(ctx context.Context, id, addr string, maxDowntime time.Du return result, nil } -// moveOut runs the rounds up to the target's commit; on any error the caller resolves the move through its record. +// On any error the caller settles the move through its record. func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, budget time.Duration) error { m.mu.Lock() admission, vm := MoveAdmission{Tenant: sb.Tenant, Key: sb.Key, From: m.nodeID}, sb.VMName @@ -160,7 +160,6 @@ func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round t return m.movePeer.SendRound(ctx, mv.addr, id, mv.rounds, round.Dir, names) } -// markMoving admits a move of sb and gates its captures, hibernates, releases and reaps until the move ends. func (m *Manager) markMoving(sb *types.Sandbox, mv *outMove) error { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -185,7 +184,7 @@ func (m *Manager) markMoving(sb *types.Sandbox, mv *outMove) error { return nil } -// moveBlocker names why sb cannot move, "" when it can; callers hold m.mu. +// Callers hold m.mu. func (m *Manager) moveBlocker(sb *types.Sandbox) string { _, archiving := m.archiving[sb.ID] switch { @@ -219,7 +218,7 @@ func (m *Manager) journalMoveOut(ctx context.Context, sb *types.Sandbox, mv *out return nil } -// unrowForMove deletes a replicated claim's row and replica, so no third node adopts a stale copy while the target runs it; the caller holds sb.Transition. +// Without the row no third node adopts a stale copy; the caller holds sb.Transition. func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox, mv *outMove) error { if m.archives == nil { return nil @@ -246,7 +245,7 @@ func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox, mv *outMo return err } -// finishMoveOut drops a claim that moved on, like a claim another node adopted, and deletes the record once its target marked it started. +// Deletes the record only once the target marked it started. func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) { m.mu.Lock() vm := sb.VMName @@ -269,7 +268,7 @@ func (m *Manager) finishMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM } } -// failMoveOut resolves a failed move through its record: only proof that the move stayed here resumes the guest, anything else drops the claim. +// Only proof the move stayed here resumes the guest; anything else drops the claim. func (m *Manager) failMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, cause error) (MoveResult, error) { if mv.Epoch == 0 { _ = m.abortMoveOut(ctx, sb, mv, 0) @@ -299,7 +298,7 @@ func (m *Manager) retryMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo } } -// settleMoveOut resumes the guest only on a record of this move still open or aborted; a missing record, another node's, or a committed or started one means the claim moved on. An error leaves the move as it is. +// An error leaves the move as it is. func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) (bool, error) { for range 2 { rec, ok, err := m.moves.Get(ctx, sb.ID) @@ -325,7 +324,7 @@ func (m *Manager) settleMoveOut(ctx context.Context, sb *types.Sandbox, mv *outM return false, fmt.Errorf("the move record of %s kept changing", sb.ID) } -// abortMoveOut ends the series, which resumes a paused guest, releases the target and the budget slot, and deletes the aborted record only once the journal no longer needs it. +// Deletes the aborted record only once the journal no longer needs it. func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, aborted int64) error { logger := log.WithFunc("pool.abortMoveOut") m.mu.Lock() @@ -370,13 +369,13 @@ func (m *Manager) endMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove } } -// pausedForMove reports whether sb's guest is paused by a move, in a source's final round or received and not started, when dials and wakes answer ErrEvacuating. func (m *Manager) pausedForMove(sb *types.Sandbox) bool { m.mu.Lock() defer m.mu.Unlock() return m.pausedForMoveLocked(sb) } +// Paused: a source's final round, or a received claim not yet started. func (m *Manager) pausedForMoveLocked(sb *types.Sandbox) bool { mv := m.moving[sb.ID] return sb.MoveIn || mv != nil && mv.final diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go index 02a7de1b..6a529b70 100644 --- a/sandboxd/server/move.go +++ b/sandboxd/server/move.go @@ -48,7 +48,7 @@ func (c *MoveClient) Drop(ctx context.Context, addr, id string) error { return c.call(ctx, http.MethodDelete, addr, movePath(id, ""), nil, nil) } -// call maps the target's refusals onto the pool errors the source answers with, passing the target's reason through. +// Maps the target's status codes onto pool errors. func (c *MoveClient) call(ctx context.Context, method, addr, path string, in, out any) error { ctx, cancel := context.WithTimeout(ctx, moveCallTimeout) defer cancel() From cd583efe0cb1ec8b37dc5d276b8973d50ba526bd Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:28:36 +0800 Subject: [PATCH 42/53] review: pin that the delta fast fail starts at the second delta --- sandboxd/pool/move_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 874d3a84..36bde7e4 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -636,9 +636,10 @@ func TestMoveGivesUpOnDeltasThatStopShrinking(t *testing.T) { sizes []int64 captures int }{ - "second delta as large as the first": {sizes: []int64{1000, 500, 500}, captures: 3}, - "second delta larger than the first": {sizes: []int64{1000, 500, 600}, captures: 3}, - "shrinking but never fitting": {sizes: []int64{1000, 900, 800, 700, 600}, captures: 5}, + "second delta as large as the first": {sizes: []int64{1000, 500, 500}, captures: 3}, + "second delta larger than the first": {sizes: []int64{1000, 500, 600}, captures: 3}, + "shrinking but never fitting": {sizes: []int64{1000, 900, 800, 700, 600}, captures: 5}, + "first delta larger than the baseline": {sizes: []int64{100, 200, 150, 120, 110}, captures: 5}, } { t.Run(name, func(t *testing.T) { mc := newMoveCluster(t) From dd6519ad3bcff49f585d4bfe40a0a5bd098e4d63 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:34:44 +0800 Subject: [PATCH 43/53] review: exported URL above the helpers, and gofumpt on the move constants --- sandboxd/pool/move.go | 2 +- sandboxd/store/peer/transport.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index f9729b5c..2adeb5ec 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -18,7 +18,7 @@ const ( // MoveStartAllowance is what the target's receive and start add to the final round's transfer, so a smaller downtime budget never fits. MoveStartAllowance = 400 * time.Millisecond - movesDir = "moves" + movesDir = "moves" moveReservationIdle = 2 * time.Minute moveRetryInterval = 2 * time.Second moveDeltaLimit = 4 diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index ff8322a3..b208cdb7 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -142,6 +142,13 @@ func Untar(r io.Reader, dst string) error { } } +func URL(addr, path string) string { + if !strings.Contains(addr, "://") { + addr = "http://" + addr + } + return addr + path +} + func untarRegular(r io.Reader, hdr *tar.Header, target string, sizes map[string]int64) error { mode := os.FileMode(hdr.Mode).Perm() //nolint:gosec // Perm masks to 0777 if raw, ok := hdr.PAXRecords[paxFileSize]; ok { @@ -205,13 +212,6 @@ func checkpointURL(addr, id, suffix string) string { return URL(addr, "/v1/checkpoints/"+url.PathEscape(id)+suffix) } -func URL(addr, path string) string { - if !strings.Contains(addr, "://") { - addr = "http://" + addr - } - return addr + path -} - // safeJoin resolves name under root, rejecting absolute paths and names that escape root. func safeJoin(root, name string) (string, error) { if !filepath.IsLocal(name) || strings.Contains(name, `\`) { From 7001946553e1715f48da03840c91addf7691df54 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:58:30 +0800 Subject: [PATCH 44/53] pool: a move's downtime estimate reads the bytes on the wire A round lists a sparse disk at its apparent size, so the baseline counted about 11.8 GB where 1.9 GB crossed the wire. On hardware the 2000 ms budget predicted 1.84 s and paused for 10.8 s, 3 of 3. SendRound now returns the bytes it wrote, and the rate and the last round's size come from them. The round's deadline stays on the listed size, an upper bound known before the send. --- docs/cluster.md | 2 +- sandboxd/pool/move.go | 2 +- sandboxd/pool/move_out.go | 10 +++---- sandboxd/pool/move_test.go | 45 +++++++++++++++++++++++++++----- sandboxd/server/move.go | 9 ++++--- sandboxd/store/peer/move.go | 24 ++++++++++++----- sandboxd/store/peer/move_test.go | 12 ++++++--- 7 files changed, 76 insertions(+), 28 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 592e162b..a1f5d758 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -368,7 +368,7 @@ writes `aborted` over a `pending` or `final` one, and the source deletes the ended ones. **Rounds.** The source sends a baseline round, then up to 4 deltas, each over -the control port with the root token. Once the last delta's size at the rate +the control port with the root token. Once the last delta's bytes on the wire at the rate the rounds reached, plus 400 ms for the receive and the start, fits `max_downtime_ms` (default 2000, and above 400 since those 400 ms alone would never fit), the source pauses the guest for the final diff --git a/sandboxd/pool/move.go b/sandboxd/pool/move.go index 2adeb5ec..70f4ea99 100644 --- a/sandboxd/pool/move.go +++ b/sandboxd/pool/move.go @@ -45,7 +45,7 @@ type MoveResult struct { type MovePeer interface { // Admit reserves a claim slot on the target and returns its node id. Admit(ctx context.Context, addr, id string, a MoveAdmission) (string, error) - SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error + SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) (int64, error) // Commit hands the target the claim record and the final epoch. Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) error Drop(ctx context.Context, addr, id string) error diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index a009e0ea..2810074c 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -78,11 +78,11 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b var sending time.Duration for delta := 0; ; delta++ { start := time.Now() - if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { - return err + size, sendErr := m.sendRound(ctx, sb.ID, mv, round) + if sendErr != nil { + return sendErr } sending += time.Since(start) - size := roundBytes(round) sent += size if delta > 0 { if finalPause(size, sent, sending) <= budget { @@ -115,7 +115,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b if err != nil { return err } - if err = m.sendRound(ctx, sb.ID, mv, round); err != nil { + if _, err = m.sendRound(ctx, sb.ID, mv, round); err != nil { return err } m.mu.Lock() @@ -149,7 +149,7 @@ func (m *Manager) beginMove(ctx context.Context, sb *types.Sandbox, to string) ( return m.moves.Begin(ctx, sb.ID, m.nodeID, to) } -func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round types.MigrateRound) error { +func (m *Manager) sendRound(ctx context.Context, id string, mv *outMove, round types.MigrateRound) (int64, error) { mv.rounds++ names := make([]string, len(round.Files)) for i, f := range round.Files { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 36bde7e4..7913d423 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -280,7 +280,7 @@ func TestMoveTargetRefusesARoundThatOutlivedItsReservation(t *testing.T) { w.WriteHeader(http.StatusNoContent) })) defer capture.Close() - if err := peer.SendRound(t.Context(), capture.URL, "", "/", src, []string{"state.json"}); err != nil { + if _, err := peer.SendRound(t.Context(), capture.URL, "", "/", src, []string{"state.json"}); err != nil { t.Fatalf("capture a round: %v", err) } body, stream := io.Pipe() @@ -631,6 +631,32 @@ func TestMoveFinalRoundAnswersUserCallsAtOnce(t *testing.T) { } } +func TestMoveEstimatesThePauseFromTheBytesOnTheWire(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + eng := fakeOf(mc.s) + eng.migrateSizes = []int64{10 << 30, 520 << 20, 500 << 20, 480 << 20, 460 << 20} + mc.p.wireSizes = []int64{1400 << 20, 520 << 20, 500 << 20, 480 << 20, 460 << 20} + mc.p.roundHook = func(round int) error { + if round == 1 { + time.Sleep(250 * time.Millisecond) + } else { + time.Sleep(50 * time.Millisecond) + } + return nil + } + if _, err := mc.s.Move(t.Context(), sb.ID, "b", MoveStartAllowance+50*time.Millisecond); !errors.Is(err, ErrMoveDowntime) { + t.Fatalf("move of a hot guest: %v, want ErrMoveDowntime", err) + } + eng.mu.Lock() + wentFinal := slices.ContainsFunc(eng.captures, func(c string) bool { return strings.HasSuffix(c, ":true") }) + eng.mu.Unlock() + if wentFinal { + t.Error("the move paused the guest for a final round that cannot fit") + } + assertStillOnSource(t, mc, sb) +} + func TestMoveGivesUpOnDeltasThatStopShrinking(t *testing.T) { for name, tt := range map[string]struct { sizes []int64 @@ -645,7 +671,7 @@ func TestMoveGivesUpOnDeltasThatStopShrinking(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) eng := fakeOf(mc.s) - eng.migrateSizes = tt.sizes + mc.p.wireSizes = tt.sizes if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Millisecond); !errors.Is(err, ErrMoveDowntime) { t.Fatalf("move: %v, want ErrMoveDowntime", err) } @@ -1217,6 +1243,7 @@ type fakeMovePeer struct { loseCommitReply bool dropHook func() roundBudgets []time.Duration + wireSizes []int64 commitKilled <-chan struct{} } @@ -1242,19 +1269,23 @@ func (p *fakeMovePeer) Admit(_ context.Context, addr, id string, a MoveAdmission return p.target(addr).AdmitMove(id, a) } -func (p *fakeMovePeer) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error { +func (p *fakeMovePeer) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) (int64, error) { if deadline, ok := ctx.Deadline(); ok { p.roundBudgets = append(p.roundBudgets, time.Until(deadline)) } if hook := p.roundHook; hook != nil { if err := hook(round); err != nil { - return err + return 0, err } } - if err := peer.SendRound(ctx, p.srv.URL, "", "/"+addr+"/"+id, dir, files); err != nil { - return errors.Join(ErrMoveTarget, err) + wire, err := peer.SendRound(ctx, p.srv.URL, "", "/"+addr+"/"+id, dir, files) + if err != nil { + return wire, errors.Join(ErrMoveTarget, err) + } + if len(p.wireSizes) > 0 { + wire, p.wireSizes = p.wireSizes[0], p.wireSizes[1:] } - return nil + return wire, nil } func (p *fakeMovePeer) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) error { diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go index 6a529b70..bccede22 100644 --- a/sandboxd/server/move.go +++ b/sandboxd/server/move.go @@ -33,11 +33,12 @@ func (c *MoveClient) Admit(ctx context.Context, addr, id string, a pool.MoveAdmi return out.Node, err } -func (c *MoveClient) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) error { - if err := peer.SendRound(ctx, addr, c.Token, movePath(id, "/rounds/"+strconv.Itoa(round)), dir, files); err != nil { - return fmt.Errorf("%w: %w", pool.ErrMoveTarget, err) +func (c *MoveClient) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) (int64, error) { + wire, err := peer.SendRound(ctx, addr, c.Token, movePath(id, "/rounds/"+strconv.Itoa(round)), dir, files) + if err != nil { + return wire, fmt.Errorf("%w: %w", pool.ErrMoveTarget, err) } - return nil + return wire, nil } func (c *MoveClient) Commit(ctx context.Context, addr, id string, claim []byte, epoch int64) error { diff --git a/sandboxd/store/peer/move.go b/sandboxd/store/peer/move.go index 1c8d5096..d151f3cd 100644 --- a/sandboxd/store/peer/move.go +++ b/sandboxd/store/peer/move.go @@ -16,24 +16,36 @@ import ( const partialSuffix = ".partial" -// SendRound PUTs the named files of dir to a peer as one tar stream, in the given order, sparse files kept sparse. -func SendRound(ctx context.Context, addr, token, path, dir string, names []string) error { +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} + +// SendRound PUTs the named files of dir to a peer as one tar stream, in the given order, sparse files kept sparse, and returns the bytes it wrote. +func SendRound(ctx context.Context, addr, token, path, dir string, names []string) (int64, error) { pr, pw := io.Pipe() + counted := &countingWriter{w: pw} written := make(chan error, 1) go func() { - err := writeRound(dir, names, pw) + err := writeRound(dir, names, counted) _ = pw.CloseWithError(err) written <- err }() err := peerCall(ctx, http.MethodPut, URL(addr, path), token, pr, 0, http.StatusOK, http.StatusNoContent) _ = pr.Close() if writeErr := <-written; writeErr != nil && !errors.Is(writeErr, io.ErrClosedPipe) { - return fmt.Errorf("send round to %s: %w", addr, writeErr) + return counted.n, fmt.Errorf("send round to %s: %w", addr, writeErr) } if err != nil { - return fmt.Errorf("send round to %s: %w", addr, err) + return counted.n, fmt.Errorf("send round to %s: %w", addr, err) } - return nil + return counted.n, nil } // ReceiveRound writes a SendRound stream into dst, replacing same-named files; a file takes its name only once all of it arrived. diff --git a/sandboxd/store/peer/move_test.go b/sandboxd/store/peer/move_test.go index 4452507d..4abedac8 100644 --- a/sandboxd/store/peer/move_test.go +++ b/sandboxd/store/peer/move_test.go @@ -43,9 +43,13 @@ func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing })) defer srv.Close() - if err := SendRound(t.Context(), srv.URL, "fleet-token", "/v1/moves/m1/rounds/1", src, names); err != nil { + wire, err := SendRound(t.Context(), srv.URL, "fleet-token", "/v1/moves/m1/rounds/1", src, names) + if err != nil { t.Fatalf("SendRound: %v", err) } + if wire != int64(body.Len()) || wire >= size { + t.Errorf("SendRound counted %d bytes, want the %d on the wire, far below the sparse file's %d", wire, body.Len(), size) + } if method != http.MethodPut || path != "/v1/moves/m1/rounds/1" || auth != "Bearer fleet-token" { t.Errorf("request = %s %s auth %q, want PUT /v1/moves/m1/rounds/1 with the fleet token", method, path, auth) } @@ -210,10 +214,10 @@ func TestSendRoundFailsOnARefusedPutOrAMissingFile(t *testing.T) { })) defer srv.Close() - if err := SendRound(t.Context(), srv.URL, "wrong", "/r", src, []string{"state.json"}); err == nil || !strings.Contains(err.Error(), "401") { + if _, err := SendRound(t.Context(), srv.URL, "wrong", "/r", src, []string{"state.json"}); err == nil || !strings.Contains(err.Error(), "401") { t.Errorf("refused PUT = %v, want the 401", err) } - if err := SendRound(t.Context(), srv.URL, "fleet-token", "/r", src, []string{"state.json", "missing"}); err == nil || !errors.Is(err, os.ErrNotExist) { + if _, err := SendRound(t.Context(), srv.URL, "fleet-token", "/r", src, []string{"state.json", "missing"}); err == nil || !errors.Is(err, os.ErrNotExist) { t.Errorf("round with a missing file = %v, want the stat error", err) } } @@ -230,7 +234,7 @@ func TestSendRoundFailsWhenATargetStopsReading(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - if err := SendRound(ctx, srv.URL, "", "/r", src, []string{"memory-ranges"}); !errors.Is(err, context.DeadlineExceeded) { + if _, err := SendRound(ctx, srv.URL, "", "/r", src, []string{"memory-ranges"}); !errors.Is(err, context.DeadlineExceeded) { t.Errorf("round to a target that stopped reading: %v, want the deadline", err) } } From 33646c4b474edc40caf381d09158731afb0de7e0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:58:36 +0800 Subject: [PATCH 45/53] pool: a delta that shrinks by under 10% ends the move at once Hardware's hot guest shrank each delta a little, 519 MB then 513 MB, so the not-smaller test never fired and a budget of 1000 answered 503 only at 61.6 s. Wire bytes, from the second delta. --- docs/cluster.md | 2 +- sandboxd/pool/move_out.go | 2 +- sandboxd/pool/move_test.go | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index a1f5d758..c110fd5f 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -374,7 +374,7 @@ the rounds reached, plus 400 ms for the receive and the start, fits never fit), the source pauses the guest for the final round and the target receives the series and starts the guest. A move whose deltas never fit answers 503 and the guest runs on; from the second delta on, one -not smaller than the delta before it and still too large answers at once. `downtime_ms` in the answer +that shrinks by under 10% and is still too large answers at once. `downtime_ms` in the answer is the pause the source measured, from the final capture to the target's commit. Each round has one minute plus its size at 10 MB/s to arrive, so a target that stops reading fails the move. The final capture, its transfer and the target's commit have 2 minutes diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 2810074c..b1c791d4 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -88,7 +88,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b if finalPause(size, sent, sending) <= budget { break } - if delta == moveDeltaLimit || delta > 1 && size >= previous { + if delta == moveDeltaLimit || delta > 1 && size*10 >= previous*9 { return fmt.Errorf("%w: delta %d of %d bytes does not fit %s", ErrMoveDowntime, delta, size, budget) } } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 7913d423..6ba81d66 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -664,6 +664,10 @@ func TestMoveGivesUpOnDeltasThatStopShrinking(t *testing.T) { }{ "second delta as large as the first": {sizes: []int64{1000, 500, 500}, captures: 3}, "second delta larger than the first": {sizes: []int64{1000, 500, 600}, captures: 3}, + "second delta shrinks by 1%": {sizes: []int64{1000, 519, 513}, captures: 3}, + "second delta shrinks by exactly 10%": {sizes: []int64{1000, 500, 450}, captures: 3}, + "third delta shrinks by under 10%": {sizes: []int64{1000, 800, 600, 550}, captures: 4}, + "each delta shrinks by over 10%": {sizes: []int64{1000, 500, 449, 400, 350}, captures: 5}, "shrinking but never fitting": {sizes: []int64{1000, 900, 800, 700, 600}, captures: 5}, "first delta larger than the baseline": {sizes: []int64{100, 200, 150, 120, 110}, captures: 5}, } { From dddccc4bb63646aa969e31e8897e8620fa5d86da Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 21:58:51 +0800 Subject: [PATCH 46/53] pool: a delta or final capture that fails answers 409 like the baseline A VMM that died mid-move answered 500, which the documented list lacks. --- docs/sandboxd-api.md | 2 +- sandboxd/pool/move_out.go | 4 ++-- sandboxd/pool/move_test.go | 20 ++++++++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index df00f263..7bc6fca9 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -868,7 +868,7 @@ node drops the claim (see [moving a running claim](cluster.md#moving-a-running-c - 404 unknown sandbox - 409 the claim cannot move (hibernated, archived, failed, waking, with volumes, egress lane, a move already running), cocoon or its run dir cannot - migrate, the target refused (`target : `), or the node lacks + migrate, a capture failed (the VMM died mid-move), the target refused (`target : `), or the node lacks `archive_failover` or `mesh.cluster_key` - 429 `move_concurrency` reached here or on the target, or the target's admission refused (`target : `) diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index b1c791d4..d0ae647d 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -94,7 +94,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b } previous = size if round, err = m.eng.MigrateCapture(ctx, vm, mv.Series, false); err != nil { - return fmt.Errorf("delta capture: %w", err) + return fmt.Errorf("%w: delta capture: %w", ErrNotMovable, err) } } m.mu.Lock() @@ -109,7 +109,7 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b } mv.pausedAt = time.Now() if round, err = m.eng.MigrateCapture(ctx, vm, mv.Series, true); err != nil { - return fmt.Errorf("final capture: %w", err) + return fmt.Errorf("%w: final capture: %w", ErrNotMovable, err) } final, err := m.moves.Advance(ctx, sb.ID, mv.Epoch, moves.Pending, moves.Final) if err != nil { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 6ba81d66..80ef51d6 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -136,10 +136,18 @@ func TestMoveFailingAtAnyStepKeepsOneOwner(t *testing.T) { "baseline": {fail: func(mc *moveCluster) { fakeOf(mc.s).captureErr = errors.New("no reflink") }, want: ErrNotMovable}, "round transfer": {fail: func(mc *moveCluster) { mc.p.roundHook = failRound(2) }, want: ErrMoveTarget}, "downtime budget": {budget: time.Millisecond, want: ErrMoveDowntime}, - "final capture": {fail: func(mc *moveCluster) { fakeOf(mc.s).finalCaptureErr = errors.New("pause failed") }}, - "final transfer": {fail: func(mc *moveCluster) { mc.p.roundHook = failRound(3) }, want: ErrMoveTarget}, - "receive": {fail: func(mc *moveCluster) { fakeOf(mc.t).receiveErr = errors.New("restore failed") }}, - "target memory": {fail: func(mc *moveCluster) { mc.p.commitHook = func(int) error { mc.t.memoryLow.Store(true); return nil } }, want: ErrQuota}, + "delta capture": {fail: func(mc *moveCluster) { + mc.p.roundHook = func(round int) error { + if round == 1 { + fakeOf(mc.s).captureErr = errors.New("vmm gone") + } + return nil + } + }, want: ErrNotMovable}, + "final capture": {fail: func(mc *moveCluster) { fakeOf(mc.s).finalCaptureErr = errors.New("pause failed") }, want: ErrNotMovable}, + "final transfer": {fail: func(mc *moveCluster) { mc.p.roundHook = failRound(3) }, want: ErrMoveTarget}, + "receive": {fail: func(mc *moveCluster) { fakeOf(mc.t).receiveErr = errors.New("restore failed") }}, + "target memory": {fail: func(mc *moveCluster) { mc.p.commitHook = func(int) error { mc.t.memoryLow.Store(true); return nil } }, want: ErrQuota}, "source aborted first": {fail: func(mc *moveCluster) { mc.p.commitHook = func(int) error { abortRecord(mc); return nil } }, want: ErrNotMovable}, @@ -708,8 +716,8 @@ func TestMoveLeavesAnExitedVMMToLivenessUntilItEnds(t *testing.T) { } return nil } - if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err == nil { - t.Fatal("a move whose VMM exited succeeded") + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, ErrNotMovable) { + t.Fatalf("a move whose VMM exited: %v, want ErrNotMovable", err) } if _, held := mc.t.byID(sb.ID); held { t.Error("the target holds a claim whose VMM exited mid-move") From a575a826d60c6538dd287667309a874fe21e162c Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 22:03:17 +0800 Subject: [PATCH 47/53] review: an err shadow in the round test --- sandboxd/store/peer/move_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/store/peer/move_test.go b/sandboxd/store/peer/move_test.go index 4abedac8..aa069829 100644 --- a/sandboxd/store/peer/move_test.go +++ b/sandboxd/store/peer/move_test.go @@ -58,7 +58,7 @@ func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing } sent := int64(body.Len()) dst := t.TempDir() - if err := ReceiveRound(&body, dst); err != nil { + if err = ReceiveRound(&body, dst); err != nil { t.Fatalf("ReceiveRound: %v", err) } for _, name := range names { From e3a6164ab00b16cf56218fc99b138d159e3ba49c Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 22:06:54 +0800 Subject: [PATCH 48/53] docs: a move can give up on the downtime before its fourth delta --- docs/sandboxd-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 7bc6fca9..0a2a37d4 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -874,7 +874,7 @@ node drops the claim (see [moving a running claim](cluster.md#moving-a-running-c admission refused (`target : `) - 502 the target is unreachable or failed - 503 the move or archive registry is down, the node lease is overdue, or the - final round would not fit `max_downtime_ms` after 4 deltas + final round would not fit `max_downtime_ms`, after at most 4 deltas The target serves root-only peer routes for a move, `POST /v1/moves/{id}`, `PUT /v1/moves/{id}/rounds/{n}`, `POST /v1/moves/{id}/commit` and From 5ab54e7f5d87fb967911921da287ef71526670f2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 22:15:22 +0800 Subject: [PATCH 49/53] pool: a target that begins to stop keeps a received guest for its restart Since #444 a cocoon command asked for after Close answers an error. A target stopping between its commit and its vm start read that as a guest that cannot start, and failed a claim whose guest was intact, so its next start cold-booted it without its memory. The journal now keeps the claim and the restart starts the guest. --- sandboxd/engine/engine.go | 4 ++-- sandboxd/engine/engine_test.go | 4 ++-- sandboxd/pool/move_in.go | 4 ++++ sandboxd/pool/move_test.go | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 53f7c422..bdfece4f 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -70,7 +70,7 @@ const ( ) var ( - errStopping = errors.New("sandboxd is stopping") + ErrStopping = errors.New("sandboxd is stopping") // capacitySignatures mean the node cannot attach another VM, not that this VM failed. capacitySignatures = []string{ @@ -445,7 +445,7 @@ func (e *Engine) run(ctx context.Context, args ...string) ([]byte, error) { func (e *Engine) runFor(ctx context.Context, timeout, termGrace time.Duration, args ...string) ([]byte, error) { if !e.begin() { - return nil, errStopping + return nil, ErrStopping } defer e.running.Done() ctx, cancel := context.WithTimeout(ctx, timeout) diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index 64528423..8f65e83f 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -323,8 +323,8 @@ func TestCloseWaitsForARunningCommandAndRefusesNewOnes(t *testing.T) { t.Fatalf("the running hibernate: %v", err) } <-closed - if _, err := e.List(t.Context()); !errors.Is(err, errStopping) { - t.Errorf("List after Close = %v, want %v", err, errStopping) + if _, err := e.List(t.Context()); !errors.Is(err, ErrStopping) { + t.Errorf("List after Close = %v, want %v", err, ErrStopping) } } diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index 7edf5e53..b9ac7d59 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -11,6 +11,7 @@ import ( "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/moves" "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -145,6 +146,9 @@ func (m *Manager) startMoveIn(ctx context.Context, sb *types.Sandbox, from strin if err = m.eng.Start(ctx, sb.VMName); err == nil { break } + if errors.Is(err, engine.ErrStopping) { + return + } logger.Warnf(ctx, "start moved claim %s: %v", sb.ID, err) } if err != nil { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 80ef51d6..0de147c4 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -1111,6 +1111,41 @@ func TestMoveTargetKilledBetweenCommitAndStartStartsTheGuestAfterItsRestart(t *t } } +func TestMoveTargetStoppingBeforeItsStartKeepsTheGuestForItsRestart(t *testing.T) { + mc := newMoveCluster(t) + sb := mustClaim(t, mc.s, testKey) + ctx := t.Context() + eng := fakeOf(mc.t) + eng.mu.Lock() + eng.startErr = engine.ErrStopping + eng.mu.Unlock() + + if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move whose target began to stop: %v, want the committed move", err) + } + held, ok := mc.t.byID(sb.ID) + if !ok || !held.MoveIn || held.Failed != "" { + t.Fatalf("target claim %+v %v, want it journaled and not failed", held, ok) + } + + eng.mu.Lock() + eng.startErr = nil + eng.mu.Unlock() + restarted := mc.restart(t, mc.t) + restarted.liftFence() + if err := restarted.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + waitFor(t, func() bool { + rec, found, _ := mc.moveReg.Get(ctx, sb.ID) + return !restarted.moveHeld(sb.ID) && found && rec.State == moves.Started + }) + moved, ok := restarted.byID(sb.ID) + if !ok || moved.Failed != "" || moved.MoveIn { + t.Fatalf("restarted target claim %+v %v, want the guest started", moved, ok) + } +} + func TestMoveTargetMarksTheRecordStartedOnlyAfterItsJournalPersists(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) From 6ea596986217411884f7491b814dc2c5d8a3885a Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 22:49:56 +0800 Subject: [PATCH 50/53] pool: a move clears its claim's replica fields before it deletes the row A delete whose reply was lost, or a crash right after one, left the claim naming a row that was gone. The next replica round or the restart read that as another node holding the claim and removed the only guest while the target had not committed. With the fields cleared first, the claim names no row whatever the delete did: the restart deletes a row it still finds, and the next round puts a new one. --- sandboxd/pool/failover_test.go | 8 ++++- sandboxd/pool/move_out.go | 15 +++++---- sandboxd/pool/move_test.go | 56 ++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 62af2f95..1c837dca 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -1974,6 +1974,7 @@ type hookRegistry struct { loseConsumeReply bool loseTakeReply bool loseReleaseReply bool + loseDeleteReply bool losePutReply bool failPut bool replaces int @@ -2043,7 +2044,12 @@ func (r *hookRegistry) Delete(ctx context.Context, id string, epoch int64) error r.beforeDelete = nil hook() } - return r.Registry.Delete(ctx, id, epoch) + err := r.Registry.Delete(ctx, id, epoch) + if err == nil && r.loseDeleteReply { + r.loseDeleteReply = false + return archives.ErrUnavailable + } + return err } func (r *hookRegistry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index d0ae647d..14a9005d 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -232,17 +232,20 @@ func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox, mv *outMo if epoch == 0 { return nil } - if err := m.archives.Delete(ctx, sb.ID, epoch); errors.Is(err, archives.ErrLost) { + live, err := m.commitIfLive(ctx, sb, func() { sb.ReplicaCk, sb.ArchiveEpoch = "", 0 }) + if !live { + return ErrUnknownSandbox + } + if err != nil { + return err + } + if err = m.archives.Delete(ctx, sb.ID, epoch); errors.Is(err, archives.ErrLost) { return fmt.Errorf("%w: its replica row moved under it", ErrNotMovable) } else if err != nil { return err } mv.replica = replica - live, err := m.commitIfLive(ctx, sb, func() { sb.ReplicaCk, sb.ArchiveEpoch = "", 0 }) - if !live { - return ErrUnknownSandbox - } - return err + return nil } // Deletes the record only once the target marked it started. diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 0de147c4..7a6af880 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -19,12 +19,14 @@ import ( "testing" "time" + "github.com/cocoonstack/sandbox/sandboxd/archives" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/moves" "github.com/cocoonstack/sandbox/sandboxd/moves/movestest" "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" "github.com/cocoonstack/sandbox/sandboxd/store/peer" + "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -453,6 +455,60 @@ func TestMoveAbortedAfterDroppingTheRowPutsItBackOnTheNextRound(t *testing.T) { } } +func TestMoveWhoseRowDeleteReplyIsLostKeepsTheOnlyGuest(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) + sb := mustClaim(t, mc.s, testKey) + replicateNow(t, mc.s, sb) + mc.s.archives = &hookRegistry{Registry: mc.reg, loseDeleteReply: true} + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, archives.ErrUnavailable) { + t.Fatalf("move: %v, want the lost reply's error", err) + } + assertStillOnSource(t, mc, sb) + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); ok { + t.Fatal("the delete did not land under its lost reply") + } + replicateNow(t, mc.s, sb) + if _, held := mc.s.byID(sb.ID); !held { + t.Fatal("the next replica round dropped the claim as adopted") + } + if rec, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || rec.Owner != "a" { + t.Errorf("row %+v %v after the next round, want this node's", rec, ok) + } +} + +func TestMoveRestartBeforeTheRowDeleteKeepsTheClaim(t *testing.T) { + c := newFailoverCluster(t) + reg := movestest.NewMem() + c.moves = reg + ctx := t.Context() + epoch, _ := reg.Begin(ctx, "sb_moved", "a", "b") + if _, err := c.reg.Put(ctx, archives.Record{ID: "sb_moved", TokenSHA256: tenants.TokenSHA256("tok"), Owner: "a", Checkpoint: "ck_00000000000000aa", Claim: []byte("{}")}); err != nil { + t.Fatalf("seed row: %v", err) + } + sb := &types.Sandbox{ID: "sb_moved", VMName: "sbx-moved", Key: testKey, Token: "tok", Deadline: time.Now().Add(time.Hour), MoveOut: &types.MoveOut{Epoch: epoch, Series: "s1"}} + dataDir := t.TempDir() + seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) + eng := newFakeEngine() + eng.vms["sbx-moved"] = "/vsock/sbx-moved" + m := c.nodeOn(t, eng, "a", dataDir, keepAnHour) + m.liftFence() + if err := m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + waitFor(t, func() bool { return !m.moveHeld(sb.ID) }) + if _, held := m.byID(sb.ID); !held { + t.Fatal("the restart dropped a claim whose move had not committed") + } + if _, ok, _ := c.reg.Get(ctx, sb.ID); ok { + t.Error("the row the journal no longer names survived the restart") + } + eng.mu.Lock() + defer eng.mu.Unlock() + if !slices.Contains(eng.aborts, "sbx-moved:s1") || slices.Contains(eng.removes, "sbx-moved") { + t.Errorf("aborts %v removes %v, want the series aborted and the VM kept", eng.aborts, eng.removes) + } +} + func TestMoveHoldsOffCapturesReleasesAndEvacuationWhileItRuns(t *testing.T) { mc := newMoveCluster(t) sb := mustClaim(t, mc.s, testKey) From c1919c9714a72de2013c134ffdb6ee847dd2697d Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 22:50:17 +0800 Subject: [PATCH 51/53] pool: a target resolves the tenant before it admits a move The admission reaches the target on the root route, so no token resolve fills the tenant cache. A tenant the cache had not seen read as uncapped, so a move could pass the tenant's max_claims, and a removed tenant was admitted too. --- sandboxd/pool/move_in.go | 5 +++- sandboxd/pool/move_test.go | 43 ++++++++++++++++++++++++++++++---- sandboxd/server/move.go | 2 +- sandboxd/server/server.go | 2 +- sandboxd/server/server_test.go | 4 +++- 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index b9ac7d59..80810810 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -29,13 +29,16 @@ type inMove struct { } // AdmitMove reserves a claim slot for a live move of id to this node and returns this node's id. -func (m *Manager) AdmitMove(id string, a MoveAdmission) (string, error) { +func (m *Manager) AdmitMove(ctx context.Context, id string, a MoveAdmission) (string, error) { if m.moves == nil { return "", errMovesOff } if err := m.validate(a.Key); err != nil { return "", fmt.Errorf("%w: %w", ErrNotMovable, err) } + if err := m.tenantRemoved(ctx, a.Tenant); err != nil { + return "", err + } m.mu.Lock() err := m.admitMoveLocked(id, a) m.mu.Unlock() diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 7a6af880..41a317ad 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -241,6 +241,24 @@ func TestMoveTargetCountsAReservationLikeAClaim(t *testing.T) { mustClaim(t, mc.t, testKey) } +func TestMoveTargetResolvesATenantItsCacheHasNotSeen(t *testing.T) { + mc := newMoveCluster(t) + claimTenant(t, mc.t, "acme") + setTenantCaps(t, mc.t, map[string]int{"acme": 1}) + mc.t.tenants = &coldTenants{Source: mc.t.tenants} + if _, err := mc.t.AdmitMove(t.Context(), "sb_cold", MoveAdmission{Tenant: "acme", Key: testKey, From: "a"}); !errors.Is(err, ErrQuota) { + t.Errorf("admit past a capped tenant the cache has not seen: %v, want ErrQuota", err) + } + if _, err := mc.t.AdmitMove(t.Context(), "sb_gone", MoveAdmission{Tenant: "gone", Key: testKey, From: "a"}); !errors.Is(err, ErrTenantRemoved) { + t.Errorf("admit for a removed tenant: %v, want ErrTenantRemoved", err) + } + mc.t.mu.Lock() + defer mc.t.mu.Unlock() + if len(mc.t.incoming) != 0 { + t.Errorf("reservations %v, want none", mc.t.incoming) + } +} + func TestMoveTargetDropsAReservationItsSourceAbandoned(t *testing.T) { mc := newMoveCluster(t) reserveMove(t, mc.t, "sb_r", "acme") @@ -321,7 +339,7 @@ func TestMoveTargetLetsTheSameSourceRetryAndRefusesAnother(t *testing.T) { if err := os.WriteFile(stale, []byte("a crashed source's round"), 0o600); err != nil { t.Fatal(err) } - if _, err := mc.t.AdmitMove("sb_r", MoveAdmission{Tenant: "acme", Key: testKey, From: "other"}); !errors.Is(err, ErrNotMovable) { + if _, err := mc.t.AdmitMove(t.Context(), "sb_r", MoveAdmission{Tenant: "acme", Key: testKey, From: "other"}); !errors.Is(err, ErrNotMovable) { t.Errorf("admission from another source: %v, want ErrNotMovable", err) } if _, err := os.Stat(stale); err != nil { @@ -1365,11 +1383,11 @@ func newFakeMovePeer(t *testing.T, c *failoverCluster) *fakeMovePeer { return p } -func (p *fakeMovePeer) Admit(_ context.Context, addr, id string, a MoveAdmission) (string, error) { +func (p *fakeMovePeer) Admit(ctx context.Context, addr, id string, a MoveAdmission) (string, error) { if p.admitErr != nil { return "", p.admitErr } - return p.target(addr).AdmitMove(id, a) + return p.target(addr).AdmitMove(ctx, id, a) } func (p *fakeMovePeer) SendRound(ctx context.Context, addr, id string, round int, dir string, files []string) (int64, error) { @@ -1424,6 +1442,23 @@ func (p *fakeMovePeer) target(addr string) *Manager { return p.c.nodes[i] } +type coldTenants struct { + tenants.Source + warm atomic.Bool +} + +func (c *coldTenants) Peek(name string) (tenants.TenantRecord, tenants.Presence) { + if !c.warm.Load() { + return tenants.TenantRecord{}, tenants.Unknown + } + return c.Source.Peek(name) +} + +func (c *coldTenants) Lookup(ctx context.Context, name string) (tenants.TenantRecord, bool, error) { + c.warm.Store(true) + return c.Source.Lookup(ctx, name) +} + type lossyMoves struct { moves.Registry downAfterCommit bool @@ -1513,7 +1548,7 @@ func moveCounts(m *Manager, tenant string) [3]int { func reserveMove(t *testing.T, m *Manager, id, tenant string) { t.Helper() - if _, err := m.AdmitMove(id, MoveAdmission{Tenant: tenant, Key: testKey, From: "zz"}); err != nil { + if _, err := m.AdmitMove(t.Context(), id, MoveAdmission{Tenant: tenant, Key: testKey, From: "zz"}); err != nil { t.Fatalf("reserve %s: %v", id, err) } } diff --git a/sandboxd/server/move.go b/sandboxd/server/move.go index bccede22..371350ea 100644 --- a/sandboxd/server/move.go +++ b/sandboxd/server/move.go @@ -142,7 +142,7 @@ func (s *Server) handleAdmitMove(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - node, err := s.mgr.AdmitMove(id, pool.MoveAdmission{Tenant: req.Tenant, Key: req.Key.Defaulted(), From: req.From}) + node, err := s.mgr.AdmitMove(r.Context(), id, pool.MoveAdmission{Tenant: req.Tenant, Key: req.Key.Defaulted(), From: req.From}) writeResult(w, r, "admit move", id, "admit move failed", err, func() { writeJSON(w, http.StatusOK, moveAdmitResponse{Node: node}) }) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 80560e82..3e7fbac3 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -147,7 +147,7 @@ type Manager interface { Info() ([]pool.PoolInfo, pool.Gauges) Templates() []pool.TemplateInfo Move(ctx context.Context, id, addr string, maxDowntime time.Duration) (pool.MoveResult, error) - AdmitMove(id string, a pool.MoveAdmission) (string, error) + AdmitMove(ctx context.Context, id string, a pool.MoveAdmission) (string, error) ReceiveMoveRound(id string, body io.Reader) error CommitMove(ctx context.Context, id string, claim []byte, epoch int64) error DropMove(id string) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index d7edd338..e690a9a6 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2603,7 +2603,9 @@ func (f *fakeManager) Move(_ context.Context, id, addr string, maxDowntime time. return f.move(id, addr, maxDowntime) } -func (f *fakeManager) AdmitMove(string, pool.MoveAdmission) (string, error) { return "", nil } +func (f *fakeManager) AdmitMove(context.Context, string, pool.MoveAdmission) (string, error) { + return "", nil +} func (f *fakeManager) ReceiveMoveRound(string, io.Reader) error { return nil } From 759047d1e2e178c83462765af6e9098dfb6c0fdc Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 22:52:06 +0800 Subject: [PATCH 52/53] config: a live move needs a meta store cell and an encrypted mesh, not archive_failover archive_failover carries an S3 checkpoint store and a node lease for restoring archived and replicated claims. A move sends its rounds straight to the peer and uses neither; it only needs PostgreSQL for its record. Tying the two kept nodes without S3 from moving at all. --- docs/cluster.md | 4 ++++ docs/deploy.md | 2 +- docs/sandboxd-api.md | 2 +- sandboxd/config/config.go | 6 +++--- sandboxd/config/config_test.go | 20 ++++++++++++-------- sandboxd/pool/failover_test.go | 20 ++++++++++++-------- sandboxd/pool/move_test.go | 19 ++++++++++++++++++- sandboxd/pool/pool.go | 6 ++++-- 8 files changed, 55 insertions(+), 24 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index c110fd5f..f6b42320 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -322,6 +322,10 @@ memory. Evacuation stays the cold path: | Target | any peer that answers | the one the caller names | | When it cannot move | the claim is skipped and keeps serving | the call fails and the claim keeps running here | +A move needs `meta_store` with a `cell`, which holds its record, and +`mesh.cluster_key`. It does not need `archive_failover` or an S3 +`checkpoint_store`, since its rounds go straight to the peer. + Only a none-lane claim without volumes moves; a hibernated, archived, failed or waking claim, an egress-lane claim and a claim with volumes answer 409. Open exec and pty streams, port forwards and egress connections end; the client diff --git a/docs/deploy.md b/docs/deploy.md index f6f9e1c0..9f4c3251 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -160,7 +160,7 @@ the intersection of the two. | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim first tries a redirect to a warm peer, else to a peer that serves the pool; a volume claim answers the 429 with no redirect) | | `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was; a claim takes the same redirect as `max_claims`. Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A wake cannot move to another node, so its 429 stays until a running claim sleeps or goes. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | -| `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `archive_failover` and `mesh.cluster_key`: a node that sets this without them refuses to load, and one that leaves it unset cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or a master build), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM keeps the VMM it started with), a run dir that supports reflink such as XFS, nodes of the same CPU model, and the same path strings in cocoon's `root_dir` and `run_dir` on both nodes ([same paths](cluster.md#moving-a-running-claim)). Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | +| `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `meta_store` with a `cell`, for the move record, and `mesh.cluster_key`, but not `archive_failover` or an S3 `checkpoint_store`: a node that sets this without them refuses to load, and one that leaves it unset without them cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or a master build), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM keeps the VMM it started with), a run dir that supports reflink such as XFS, nodes of the same CPU model, and the same path strings in cocoon's `root_dir` and `run_dir` on both nodes ([same paths](cluster.md#moving-a-running-claim)). Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token; a config reload records as op `config_reload` with `changed`, and a runtime volume change as op `volumes` with `added` or `removed`. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no warm VMs on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | | `memory_limit` | unset | `{"anon": "300G", "psi_some_avg10": 40}`, either key alone: while `AnonPages` in `/proc/meminfo` is above `anon`, or the `some avg10` of `/proc/pressure/memory` is above `psi_some_avg10`, checked every 2 s, the node refuses new claims, forks and checkpoint claims with 429 and parks refill; `/v1/info` reports it as at capacity and the mesh sees no pools on it, so a warm or pool redirect skips it; a template or volume held only there still redirects to it and answers 429. Wakes of its own claims, hibernates, checkpoints and archives go on, and running sandboxes keep running. Guest memory is a private file mapping, so page cache and clean guest pages are reclaimable and only anon is what a hibernate has to free. Both signals are node-wide; `AnonPages` leaves out swapped pages and explicit hugepages. A signal the node cannot read, such as PSI on a kernel without it, logs an error and is ignored. PSI also counts a task stalled by its own cgroup limit and weighs by busy CPUs, so on a mostly idle node a few throttled tasks in one small cgroup can cross it: 16 tasks under a 64M `MemoryHigh` reached 75 on an idle 384-CPU host with memory to spare. Treat `anon` as the main limit, and set `psi_some_avg10` only where no workload runs under a tight memory cgroup. Size `anon` so the running guests' further growth still fits under physical memory | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 0a2a37d4..497c8664 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -869,7 +869,7 @@ node drops the claim (see [moving a running claim](cluster.md#moving-a-running-c - 409 the claim cannot move (hibernated, archived, failed, waking, with volumes, egress lane, a move already running), cocoon or its run dir cannot migrate, a capture failed (the VMM died mid-move), the target refused (`target : `), or the node lacks - `archive_failover` or `mesh.cluster_key` + a `meta_store` cell or `mesh.cluster_key` - 429 `move_concurrency` reached here or on the target, or the target's admission refused (`target : `) - 502 the target is unreachable or failed diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index d9ab5103..9a5e0c5d 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -299,9 +299,9 @@ func (c *Config) HasEgress() bool { return len(c.Bridges) > 0 || len(c.Networks) > 0 } -// Movable reports whether the node can run live moves: they need the move registry and an encrypted mesh. +// Movable reports whether the node can run live moves: they need a meta store cell for the move record and an encrypted mesh. func (c *Config) Movable() bool { - return c.ArchiveFailover && c.Mesh != nil && c.Mesh.ClusterKey != "" + return c.MetaStore != nil && c.MetaStore.Cell != "" && c.Mesh != nil && c.Mesh.ClusterKey != "" } func (c *Config) S3Checkpoints() bool { @@ -398,7 +398,7 @@ func (c *Config) validate() error { return fmt.Errorf("checkpoint_peer_heal requires api_token: without it resolveScope leaves the raw checkpoint blob GET reachable with no credential") } if c.MoveConcurrency > 0 && !c.Movable() { - return fmt.Errorf("move_concurrency requires archive_failover and an encrypted mesh (set mesh.cluster_key)") + return fmt.Errorf("move_concurrency requires meta_store with a cell and an encrypted mesh (set mesh.cluster_key)") } for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMemoryLimit, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { if err := check(); err != nil { diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 75c259d5..c779b717 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -263,23 +263,27 @@ func TestLoadReplicateSeconds(t *testing.T) { func TestLoadMoveConcurrency(t *testing.T) { t.Setenv("SANDBOX_TEST_DSN", "postgres://sandboxd@db/sandboxd") - failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},` + meta := `"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},` key := `"cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="` body := func(on, mesh, moves string) string { return `{"api_token":"r","advertise_addr":"node1:7777",` + on + `"mesh":{"bind":"node1:7946"` + mesh + `},` + moves + `"pools":[]}` } - cfg := mustLoad(t, writeConfig(t, body(failover, ","+key, `"move_concurrency":3,`))) + cfg := mustLoad(t, writeConfig(t, body(meta, ","+key, `"move_concurrency":3,`))) if cfg.MoveConcurrency != 3 || !cfg.Movable() { t.Fatalf("Load: move_concurrency %d movable %v, want 3 and movable", cfg.MoveConcurrency, cfg.Movable()) } - for name, tt := range map[string]struct{ on, mesh, moves, want string }{ - "no failover": {"", "," + key, `"move_concurrency":2,`, "move_concurrency requires"}, - "plain mesh": {failover, "", `"move_concurrency":2,`, "move_concurrency requires"}, - "negative": {failover, "," + key, `"move_concurrency":-1,`, "must not be negative"}, - "unset on a plain mesh": {failover, "", "", ""}, + for name, tt := range map[string]struct { + on, mesh, moves, want string + movable bool + }{ + "no meta store": {on: "", mesh: "," + key, moves: `"move_concurrency":2,`, want: "move_concurrency requires"}, + "plain mesh": {on: meta, mesh: "", moves: `"move_concurrency":2,`, want: "move_concurrency requires"}, + "negative": {on: meta, mesh: "," + key, moves: `"move_concurrency":-1,`, want: "must not be negative"}, + "unset on a plain mesh": {on: meta}, + "unset with a key": {on: meta, mesh: "," + key, movable: true}, } { cfg, err := Load(writeConfig(t, body(tt.on, tt.mesh, tt.moves))) - if tt.want == "" && (err != nil || cfg.Movable()) || tt.want != "" && (err == nil || !strings.Contains(err.Error(), tt.want)) { + if tt.want == "" && (err != nil || cfg.Movable() != tt.movable) || tt.want != "" && (err == nil || !strings.Contains(err.Error(), tt.want)) { t.Errorf("%s: %v, want %q", name, err, tt.want) } } diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 1c837dca..e18e4fba 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -1869,13 +1869,14 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { } type failoverCluster struct { - reg *archivestest.Mem - moves moves.Registry - ckDir string - replicate int - volumes []config.VolumeSpec - nodes []*Manager - egressPol *egress.Policy + reg *archivestest.Mem + moves moves.Registry + ckDir string + replicate int + volumes []config.VolumeSpec + nodes []*Manager + egressPol *egress.Policy + noArchives bool } func newFailoverCluster(t *testing.T) *failoverCluster { @@ -1942,7 +1943,10 @@ func (c *failoverCluster) nodeOn(t *testing.T, eng *fakeEngine, name, dataDir st cfg.EgressCA = writeTestEgressCA(t) } b := testBackends(t, cfg) - b.Archives, b.Moves = c.reg, c.moves + b.Moves = c.moves + if !c.noArchives { + b.Archives = c.reg + } m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), b) if err != nil { t.Fatalf("manager %s: %v", name, err) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 41a317ad..7cb8b9c8 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -107,7 +107,7 @@ func TestMoveRefusedByTheTargetLeavesTheClaimRunningOnTheSource(t *testing.T) { claimTenant(t, mc.t, "acme") }, "memory": func(_ *testing.T, mc *moveCluster) { mc.t.memoryLow.Store(true) }, - "move budget": func(t *testing.T, mc *moveCluster) { reserveMove(t, mc.t, "sb_other", "zed") }, + "move budget": func(t *testing.T, mc *moveCluster) { reserveMove(t, mc.t, "sb_other", "acme") }, } { t.Run(name, func(t *testing.T) { mc := newMoveCluster(t) @@ -473,6 +473,23 @@ func TestMoveAbortedAfterDroppingTheRowPutsItBackOnTheNextRound(t *testing.T) { } } +func TestMoveNeedsNoArchiveRegistry(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.noArchives = true }) + sb := claimTenant(t, mc.s, "acme") + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); err != nil { + t.Fatalf("move without archive_failover: %v", err) + } + if _, held := mc.s.byID(sb.ID); held { + t.Error("the source kept the claim") + } + if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn { + t.Errorf("target claim %+v %v, want it started", moved, ok) + } + if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + t.Errorf("counts %v, want the claim on the target only", got) + } +} + func TestMoveWhoseRowDeleteReplyIsLostKeepsTheOnlyGuest(t *testing.T) { mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) sb := mustClaim(t, mc.s, testKey) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 2448434f..e724433e 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -132,7 +132,7 @@ var ( errNoArchiveRow = errors.New("archived before archive_failover was on, so no registry row names it") errArchiveBusy = errors.New("an archive of the claim is in flight") errNoAdopter = errors.New("no peer adopted the claim") - errMovesOff = fmt.Errorf("%w: live moves need archive_failover and an encrypted mesh", ErrNotMovable) + errMovesOff = fmt.Errorf("%w: live moves need a meta store cell and an encrypted mesh", ErrNotMovable) ) // Engine is the slice of the cocoon driver the manager consumes. @@ -444,8 +444,10 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.out = outbound.New(m.outboundOptions(ca)) m.configSeedHash = poolSeedHash(cfg.Pools) start := &fenceState{ch: make(chan struct{}), fenced: m.archives != nil} - if m.archives != nil { + if m.archives != nil || m.moves != nil { m.nodeID = cfg.Mesh.ID() + } + if m.archives != nil { close(start.ch) } m.fence.Store(start) From 4a17e6a6b29a8c788fdb7f6e2b72df7757f03df3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 23:46:18 +0800 Subject: [PATCH 53/53] pool: a move consumes its claim's row before it clears the claim's fields Clearing the fields first made a claim forget its row when the delete never ran. A peer that adopted the replica during the outage then went unnoticed: unfence only checks claims with a replica, and the next round's Put succeeded once the adopter deleted its tombstone, so both nodes served the claim. A tombstone is the positive proof: no node adopts one, a refused consume names an adoption, and until it lands the claim keeps its fields and the fence stays in charge. --- docs/cluster.md | 15 +++-- sandboxd/pool/move_out.go | 8 ++- sandboxd/pool/move_test.go | 121 +++++++++++++++++++++++++++++-------- 3 files changed, 110 insertions(+), 34 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index f6b42320..e3dba7f4 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -417,11 +417,16 @@ metadata`, the call answers 502 and the guest resumes on the source. - A move call on a claim that moved further than the peer it named answers 200 with that peer as `owner_addr`; `Lookup` finds the claim. -**Replicated claims.** Before the final round the source deletes the claim's -registry row and its replica, so no third node can adopt a stale replica while -the target runs the claim. The target's first replica round puts a new row one -interval after the start, and an aborted move lets the source's next round put -it again. +**Replicated claims.** Before the final round the source consumes the claim's +registry row, since no node adopts a tombstone, then clears the claim's replica +fields and deletes the row, so no third node can adopt a stale replica while +the target runs the claim. It keeps the replica until the move ends. The target's +first replica round puts a new row one interval after the start, and an aborted +move lets the source's next round put it again. A source cut off from the +registry changes nothing and its move fails: it keeps the row's epoch, so the +fence still finds a peer that adopted the claim during the outage, and the source +drops it. A consume whose reply was lost leaves the source's own tombstone, which +the hourly sweep puts back on the replica. **Failures.** Any failure before the commit aborts: the source resumes the guest, the target drops its reservation, files and any received VM, and the diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 14a9005d..3a3d4ccf 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -232,6 +232,10 @@ func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox, mv *outMo if epoch == 0 { return nil } + consumed, err := m.consumeRow(ctx, sb, epoch, "move") + if err != nil { + return err + } live, err := m.commitIfLive(ctx, sb, func() { sb.ReplicaCk, sb.ArchiveEpoch = "", 0 }) if !live { return ErrUnknownSandbox @@ -239,9 +243,7 @@ func (m *Manager) unrowForMove(ctx context.Context, sb *types.Sandbox, mv *outMo if err != nil { return err } - if err = m.archives.Delete(ctx, sb.ID, epoch); errors.Is(err, archives.ErrLost) { - return fmt.Errorf("%w: its replica row moved under it", ErrNotMovable) - } else if err != nil { + if err = m.archives.Delete(ctx, sb.ID, consumed); err != nil && !errors.Is(err, archives.ErrLost) { return err } mv.replica = replica diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 7cb8b9c8..00711f0b 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -511,36 +511,93 @@ func TestMoveWhoseRowDeleteReplyIsLostKeepsTheOnlyGuest(t *testing.T) { } } -func TestMoveRestartBeforeTheRowDeleteKeepsTheClaim(t *testing.T) { - c := newFailoverCluster(t) - reg := movestest.NewMem() - c.moves = reg - ctx := t.Context() - epoch, _ := reg.Begin(ctx, "sb_moved", "a", "b") - if _, err := c.reg.Put(ctx, archives.Record{ID: "sb_moved", TokenSHA256: tenants.TokenSHA256("tok"), Owner: "a", Checkpoint: "ck_00000000000000aa", Claim: []byte("{}")}); err != nil { - t.Fatalf("seed row: %v", err) +func TestMoveWhoseRowConsumeReplyIsLostKeepsTheGuestAndItsReplica(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) + sb := mustClaim(t, mc.s, testKey) + replicateNow(t, mc.s, sb) + mc.s.mu.Lock() + replica := sb.ReplicaCk + mc.s.mu.Unlock() + mc.s.archives = &hookRegistry{Registry: mc.reg, loseConsumeReply: true} + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, archives.ErrUnavailable) { + t.Fatalf("move: %v, want the lost reply's error", err) } - sb := &types.Sandbox{ID: "sb_moved", VMName: "sbx-moved", Key: testKey, Token: "tok", Deadline: time.Now().Add(time.Hour), MoveOut: &types.MoveOut{Epoch: epoch, Series: "s1"}} - dataDir := t.TempDir() - seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) - eng := newFakeEngine() - eng.vms["sbx-moved"] = "/vsock/sbx-moved" - m := c.nodeOn(t, eng, "a", dataDir, keepAnHour) - m.liftFence() - if err := m.Reconcile(ctx); err != nil { - t.Fatalf("reconcile: %v", err) + assertStillOnSource(t, mc, sb) + if rec, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || !rec.Consumed || rec.Owner != "a" { + t.Fatalf("row %+v %v, want this node's tombstone", rec, ok) } - waitFor(t, func() bool { return !m.moveHeld(sb.ID) }) - if _, held := m.byID(sb.ID); !held { - t.Fatal("the restart dropped a claim whose move had not committed") + if !ckExists(t, mc.s, replica) { + t.Fatal("the failed move deleted the replica the claim still names") } - if _, ok, _ := c.reg.Get(ctx, sb.ID); ok { - t.Error("the row the journal no longer names survived the restart") + mc.s.sweepOrphanArchives(t.Context()) + if rec, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || rec.Consumed || rec.Checkpoint != replica { + t.Errorf("row %+v %v after the sweep, want the replica back on it", rec, ok) } - eng.mu.Lock() - defer eng.mu.Unlock() - if !slices.Contains(eng.aborts, "sbx-moved:s1") || slices.Contains(eng.removes, "sbx-moved") { - t.Errorf("aborts %v removes %v, want the series aborted and the VM kept", eng.aborts, eng.removes) + replicateNow(t, mc.s, sb) + if _, held := mc.s.byID(sb.ID); !held { + t.Error("the next replica round dropped the claim") + } +} + +func TestMoveCutOffFromTheRegistryLeavesAnAdoptionToTheFence(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) + u := mc.moveNode(t, "c") + sb := mustClaim(t, mc.s, testKey) + replicateNow(t, mc.s, sb) + mc.s.archives = &unreachableRowWrites{Registry: mc.reg} + if _, err := mc.s.Move(t.Context(), sb.ID, "b", time.Minute); !errors.Is(err, archives.ErrUnavailable) { + t.Fatalf("move: %v, want the registry's error", err) + } + mc.s.archives = mc.reg + mc.expire("a") + mustAdopt(t, u, sb, "adoption while the source was cut off") + mc.renew(t, "a") + mc.s.trip(t.Context()) + mc.s.unfence(t.Context(), time.Now()) + waitFor(t, func() bool { return fakeOf(mc.s).removed(sb.VMName) }) + if _, held := mc.s.byID(sb.ID); held { + t.Error("the source kept serving a claim a peer adopted") + } +} + +func TestMoveRestartAfterTheConsumeKeepsTheClaim(t *testing.T) { + for name, fieldsCleared := range map[string]bool{"before its fields are cleared": false, "after its fields are cleared": true} { + t.Run(name, func(t *testing.T) { + c := newFailoverCluster(t) + reg := movestest.NewMem() + c.moves = reg + ctx := t.Context() + epoch, _ := reg.Begin(ctx, "sb_moved", "a", "b") + rowEpoch, err := c.reg.Put(ctx, archives.Record{ID: "sb_moved", TokenSHA256: tenants.TokenSHA256("tok"), Owner: "a", Checkpoint: "ck_00000000000000aa", Claim: []byte("{}")}) + if err != nil { + t.Fatalf("seed row: %v", err) + } + if _, err = c.reg.Consume(ctx, "sb_moved", rowEpoch); err != nil { + t.Fatalf("seed tombstone: %v", err) + } + sb := &types.Sandbox{ID: "sb_moved", VMName: "sbx-moved", Key: testKey, Token: "tok", Deadline: time.Now().Add(time.Hour), MoveOut: &types.MoveOut{Epoch: epoch, Series: "s1"}} + if !fieldsCleared { + sb.ReplicaCk, sb.ArchiveEpoch = "ck_00000000000000aa", rowEpoch + } + dataDir := t.TempDir() + seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) + eng := newFakeEngine() + eng.vms["sbx-moved"] = "/vsock/sbx-moved" + m := c.nodeOn(t, eng, "a", dataDir, keepAnHour) + m.liftFence() + if err = m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + waitFor(t, func() bool { return !m.moveHeld(sb.ID) }) + if _, held := m.byID(sb.ID); !held { + t.Fatal("the restart dropped a claim whose move had not committed") + } + eng.mu.Lock() + defer eng.mu.Unlock() + if !slices.Contains(eng.aborts, "sbx-moved:s1") || slices.Contains(eng.removes, "sbx-moved") { + t.Errorf("aborts %v removes %v, want the series aborted and the VM kept", eng.aborts, eng.removes) + } + }) } } @@ -1476,6 +1533,18 @@ func (c *coldTenants) Lookup(ctx context.Context, name string) (tenants.TenantRe return c.Source.Lookup(ctx, name) } +type unreachableRowWrites struct { + archives.Registry +} + +func (r *unreachableRowWrites) Consume(context.Context, string, int64) (int64, error) { + return 0, archives.ErrUnavailable +} + +func (r *unreachableRowWrites) Delete(context.Context, string, int64) error { + return archives.ErrUnavailable +} + type lossyMoves struct { moves.Registry downAfterCommit bool