From a2434c726b33948e23c977d128cbfac0fba1aeca Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 13:57:48 +0800 Subject: [PATCH 1/8] feat(vm): capture a running VM in rounds and restore it on another node vm migrate capture/receive/abort build on the CH diff snapshot series. Disk deltas come from comparing reflinked extents, so a round costs what the guest changed. Transfer, ownership and round count stay with the caller. A generated VM ID could never collide, so prereserveVM rolled back on a failed reservation. With --id that rollback would delete the live record of the VM holding the ID, so it goes. --- cmd/vm/commands.go | 47 +++- cmd/vm/migrate.go | 71 +++++ cmd/vm/run.go | 8 +- hypervisor/clone.go | 6 +- hypervisor/cloudhypervisor/clone.go | 16 +- hypervisor/cloudhypervisor/direct.go | 10 +- hypervisor/cloudhypervisor/migrate.go | 97 +++++++ hypervisor/cloudhypervisor/migrate_test.go | 128 +++++++++ hypervisor/cloudhypervisor/snapshot.go | 2 +- hypervisor/cloudhypervisor/utils.go | 15 +- hypervisor/diskdelta.go | 228 ++++++++++++++++ hypervisor/diskdelta_test.go | 199 ++++++++++++++ hypervisor/disks.go | 23 +- hypervisor/firecracker/clone.go | 3 +- hypervisor/hypervisor.go | 2 + hypervisor/migrate.go | 294 +++++++++++++++++++++ hypervisor/migrate_test.go | 244 +++++++++++++++++ hypervisor/snapshot.go | 61 +++-- hypervisor/state_test.go | 2 +- hypervisor/utils.go | 2 + metering/metering.go | 1 + utils/extents.go | 9 + utils/extents_linux.go | 78 ++++++ utils/extents_linux_test.go | 57 ++++ utils/extents_other.go | 18 ++ utils/id.go | 8 + utils/id_test.go | 11 + utils/reflink_linux.go | 20 ++ utils/reflink_other.go | 10 +- 29 files changed, 1620 insertions(+), 50 deletions(-) create mode 100644 cmd/vm/migrate.go create mode 100644 hypervisor/cloudhypervisor/migrate.go create mode 100644 hypervisor/cloudhypervisor/migrate_test.go create mode 100644 hypervisor/diskdelta.go create mode 100644 hypervisor/diskdelta_test.go create mode 100644 hypervisor/migrate.go create mode 100644 hypervisor/migrate_test.go create mode 100644 utils/extents.go create mode 100644 utils/extents_linux.go create mode 100644 utils/extents_linux_test.go create mode 100644 utils/extents_other.go diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index 1c61523b..2019c3cb 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -198,6 +198,7 @@ func Command(h Handler) *cobra.Command { buildDeviceCommand(h), buildDiskCommand(h), buildNetCommand(h), + buildMigrateCommand(h), ) return vmCmd } @@ -215,6 +216,43 @@ func buildNetCommand(h Handler) *cobra.Command { return cmd } +func buildMigrateCommand(h Handler) *cobra.Command { + parent := cliutil.GroupCommand("migrate", "Capture a running VM in rounds and restore it on another node (CH only)") + + capture := &cobra.Command{ + Use: "capture VM", + Short: "Start a series with a full baseline, or add a delta to one with --series", + Args: cobra.ExactArgs(1), + RunE: h.MigrateCapture, + } + capture.Flags().String("series", "", "series to add a delta to (empty = start a new series)") + capture.Flags().Bool("final", false, "leave the guest paused after this delta; needs --series") + cliutil.AddOutputFlag(capture) + + receive := &cobra.Command{ + Use: "receive --from-dir DIR", + Short: "Restore a transferred series into a new VM with its guest paused; vm start resumes it", + Args: cobra.NoArgs, + RunE: h.MigrateReceive, + } + addReceiveFlags(receive) + receive.Flags().String("id", "", "VM ID (default: generated)") + _ = receive.MarkFlagRequired("from-dir") + cliutil.AddOutputFlag(receive) + + abort := &cobra.Command{ + Use: "abort VM", + Short: "End a series, resume the guest and delete the series directory", + Args: cobra.ExactArgs(1), + RunE: h.MigrateAbort, + } + abort.Flags().String("series", "", "series to abort (required)") + _ = abort.MarkFlagRequired("series") + + parent.AddCommand(capture, receive, abort) + return parent +} + func buildDiskCommand(h Handler) *cobra.Command { parent := cliutil.GroupCommand("disk", "Attach/detach an extra raw data disk to a running VM (CH, or FC created with --pci)") @@ -339,8 +377,14 @@ func addVMFlags(cmd *cobra.Command) { } func addCloneFlags(cmd *cobra.Command) { - cmd.Flags().String("name", "", "VM name (default: cocoon-clone-)") + addReceiveFlags(cmd) cmd.Flags().Int("nics", 0, "override NIC count (omit to inherit from snapshot; on Firecracker only for --pci snapshots)") + cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)") +} + +// addReceiveFlags holds the clone flags that need no device change after restore: one would resume a received guest. +func addReceiveFlags(cmd *cobra.Command) { + cmd.Flags().String("name", "", "VM name (default: cocoon-clone-)") cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)") cmd.Flags().Int("disk-queue-size", 0, "virtio-blk ring depth per device (0 = inherit from snapshot)") cmd.Flags().Int("cpu-weight", 0, "cgroup cpu.weight, 1..10000 (0 = vCPU count; snapshot knobs are never inherited)") @@ -352,7 +396,6 @@ func addCloneFlags(cmd *cobra.Command) { cmd.Flags().Bool("no-direct-io", false, "disable O_DIRECT on writable disks (inherit from snapshot if not set)") addRestoreModeFlag(cmd) cmd.Flags().Bool("pull", false, "auto-pull base image if not found locally (for cross-node clone)") - cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)") cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT") } diff --git a/cmd/vm/migrate.go b/cmd/vm/migrate.go new file mode 100644 index 00000000..d680d586 --- /dev/null +++ b/cmd/vm/migrate.go @@ -0,0 +1,71 @@ +package vm + +import ( + "fmt" + + "github.com/projecteru2/core/log" + "github.com/spf13/cobra" + + "github.com/cocoonstack/cocoon/cmd/cliutil" + cmdcore "github.com/cocoonstack/cocoon/cmd/core" + "github.com/cocoonstack/cocoon/hypervisor" +) + +func (h Handler) MigrateCapture(cmd *cobra.Command, args []string) error { + ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate capture", hypervisor.ErrMigrateUnsupported) + if err != nil { + return err + } + req := hypervisor.MigrateRequest{Series: cliutil.FlagStr(cmd, "series"), Final: cliutil.FlagBool(cmd, "final")} + if req.Final && req.Series == "" { + return fmt.Errorf("--final needs --series: the baseline is never the last round") + } + round, err := m.MigrateCapture(ctx, args[0], req) + if err != nil { + return err + } + return outputOrLog(cmd, round, func() { + log.WithFunc("cmd.vm.migrate.capture").Infof(ctx, "series %s round %d in %s (paused: %t, %d files)", round.Series, round.Round, round.Dir, round.Paused, len(round.Files)) + }) +} + +func (h Handler) MigrateReceive(cmd *cobra.Command, _ []string) error { + ctx, conf := h.Init(cmd) + logger := log.WithFunc("cmd.vm.migrate.receive") + dir := cliutil.FlagStr(cmd, "from-dir") + cfg, err := readVerifiedEnvelope(dir) + if err != nil { + return err + } + conf.UseFirecracker = isFirecracker(cfg.Hypervisor) + hyper, err := cmdcore.InitHypervisor(ctx, conf) + if err != nil { + return err + } + m, ok := hyper.(hypervisor.Migrator) + if !ok { + return fmt.Errorf("migrate receive: backend %s: %w", hyper.Type(), hypervisor.ErrMigrateUnsupported) + } + cs, err := h.prepareClone(ctx, cmd, conf, hyper, cfg) + if err != nil { + return err + } + defer cs.unlock() + vm, err := m.MigrateReceive(ctx, cs.vmID, cs.vmCfg, cs.netSetup, &cfg, dir) + if err != nil { + rollbackNetwork(ctx, cs.netProvider, cs.vmID) + cs.rollback() + return fmt.Errorf("migrate receive: %w", err) + } + return outputOrLog(cmd, vm, func() { + logger.Infof(ctx, "VM received: %s (name: %s), guest paused; resume with: cocoon vm start %s", vm.ID, vm.Config.Name, vm.ID) + }) +} + +func (h Handler) MigrateAbort(cmd *cobra.Command, args []string) error { + ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate abort", hypervisor.ErrMigrateUnsupported) + if err != nil { + return err + } + return m.MigrateAbort(ctx, args[0], cliutil.FlagStr(cmd, "series")) +} diff --git a/cmd/vm/run.go b/cmd/vm/run.go index 1f2ca5ad..142a82af 100644 --- a/cmd/vm/run.go +++ b/cmd/vm/run.go @@ -286,7 +286,12 @@ func (h Handler) prepareClone(ctx context.Context, cmd *cobra.Command, conf *con if err != nil { return cloneSetup{}, err } - vmID := utils.GenerateID() + vmID := cliutil.FlagStr(cmd, "id") + if vmID == "" { + vmID = utils.GenerateID() + } else if !utils.ValidID(vmID) { + return cloneSetup{}, fmt.Errorf("--id must have the shape of a generated VM ID, got %q", vmID) + } vmCfg.Name = cmp.Or(vmCfg.Name, "cocoon-clone-"+network.VMIDPrefix(vmID)) if err = vmCfg.Validate(); err != nil { return cloneSetup{}, err @@ -500,7 +505,6 @@ func prereserveVM(ctx context.Context, hyper hypervisor.Hypervisor, vmID string, return nil, nil, err } if err := hyper.PrereserveVM(ctx, vmID, vmCfg, blobIDs); err != nil { - hyper.RollbackCreate(ctx, vmID, vmCfg.Name) unlock() return nil, nil, fmt.Errorf("reserve VM record: %w", err) } diff --git a/hypervisor/clone.go b/hypervisor/clone.go index 75a75ab2..a16614fd 100644 --- a/hypervisor/clone.go +++ b/hypervisor/clone.go @@ -57,8 +57,8 @@ func (b *Backend) RunningCloneRecord(rec *VMRecord, vmCfg *types.VMConfig, stora return info } -// FinalizeClone persists the record and emits the clone open-interval pair. -func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string) error { +// FinalizeClone persists the record and emits the open-interval pair. +func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string, reason metering.Reason) error { if err := b.UpdateRecord(ctx, vmID, func(r *VMRecord) error { r.VM = *info // The subpackage building info cannot reach markTransition; without this the clone commits at generation zero. @@ -72,7 +72,7 @@ func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM }); err != nil { return err } - b.emitOpenInterval(ctx, info, metering.ReasonClone, sourceSnapshotID, timeNow()) + b.emitOpenInterval(ctx, info, reason, sourceSnapshotID, timeNow()) return nil } diff --git a/hypervisor/cloudhypervisor/clone.go b/hypervisor/cloudhypervisor/clone.go index 608e0ba6..81992a61 100644 --- a/hypervisor/cloudhypervisor/clone.go +++ b/hypervisor/cloudhypervisor/clone.go @@ -13,6 +13,7 @@ import ( "github.com/cocoonstack/cocoon/config" "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/metering" "github.com/cocoonstack/cocoon/network" "github.com/cocoonstack/cocoon/types" "github.com/cocoonstack/cocoon/utils" @@ -28,6 +29,7 @@ type cloneResumeOpts struct { networkConfigs []*types.NetworkConfig snapshotCfg *chVMConfig queueCPUs []int + keepPaused bool } func (ch *CloudHypervisor) Clone(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, snapshot io.Reader) (*types.VM, error) { @@ -39,10 +41,10 @@ func (ch *CloudHypervisor) cloneAfterExtract(ctx context.Context, rec *hyperviso if err != nil { return nil, fmt.Errorf("parse CH config: %w", err) } - return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, chCfg) + return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, chCfg, false) } -func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string, chCfg *chVMConfig) (*types.VM, error) { +func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string, chCfg *chVMConfig, migrate bool) (*types.VM, error) { vmID, runDir := rec.ID, rec.RunDir networkConfigs := net.NetworkConfigs logger := log.WithFunc("cloudhypervisor.cloneAfterExtractParsed") @@ -136,13 +138,18 @@ func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, rec *hyp networkConfigs: networkConfigs, snapshotCfg: chCfg, queueCPUs: queueCPUs, + keepPaused: migrate, }); err != nil { return nil, err } saveConsolePTY(ctx, vmID, runDir, sockPath, directBoot) info := ch.RunningCloneRecord(rec, vmCfg, storageConfigs, net) - if err := ch.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID); err != nil { + reason := metering.ReasonClone + if migrate { + reason = metering.ReasonMigrate + } + if err := ch.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID, reason); err != nil { ch.AbortLaunch(ctx, pid, sockPath, runDir, runtimeFiles) return nil, fmt.Errorf("finalize VM record: %w", err) } @@ -186,6 +193,9 @@ func (ch *CloudHypervisor) restoreAndResumeClone(ctx context.Context, pid int, s if err = ch.ArmCPUQuota(opts.vmID, &opts.vmCfg.Config); err != nil { return err } + if opts.keepPaused { + return nil + } if err = resumeVM(ctx, hc); err != nil { return fmt.Errorf("vm.resume: %w", err) } diff --git a/hypervisor/cloudhypervisor/direct.go b/hypervisor/cloudhypervisor/direct.go index 8849df2c..3e024e41 100644 --- a/hypervisor/cloudhypervisor/direct.go +++ b/hypervisor/cloudhypervisor/direct.go @@ -17,7 +17,7 @@ func (ch *CloudHypervisor) DirectClone(ctx context.Context, vmID string, vmCfg * spec := hypervisor.CloneSpec{ VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { - return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg) + return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg, false) }, } return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { @@ -56,8 +56,12 @@ func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig if err != nil { return nil, fmt.Errorf("parse source config: %w", err) } + return chCfg, hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, classifySnapshotFile(chCfg)) +} + +func classifySnapshotFile(chCfg *chVMConfig) func(name string) hypervisor.SnapshotFileKind { cowFiles := identifyCOWFiles(chCfg) - return chCfg, hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, func(name string) hypervisor.SnapshotFileKind { + return func(name string) hypervisor.SnapshotFileKind { switch { case strings.HasPrefix(name, memoryRangeFile): return hypervisor.SnapshotFileMemory @@ -66,7 +70,7 @@ func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig default: return hypervisor.SnapshotFileMeta } - }) + } } // cleanSnapshotFiles enumerates by name so stale data-*.raw and cocoon.json from a previous incarnation don't linger; COW files are overwritten anyway. diff --git a/hypervisor/cloudhypervisor/migrate.go b/hypervisor/cloudhypervisor/migrate.go new file mode 100644 index 00000000..644c60f1 --- /dev/null +++ b/hypervisor/cloudhypervisor/migrate.go @@ -0,0 +1,97 @@ +package cloudhypervisor + +import ( + "context" + "fmt" + "maps" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/types" +) + +const ( + snapshotTypeDiff = "diff" + memoryDiffPrefix = "memory-ranges.diff." +) + +var _ hypervisor.Migrator = (*CloudHypervisor)(nil) + +func (ch *CloudHypervisor) MigrateCapture(ctx context.Context, ref string, req hypervisor.MigrateRequest) (*hypervisor.MigrateRound, error) { + return ch.MigrateCaptureSequence(ctx, ref, req, ch.migrateSpec(ctx)) +} + +func (ch *CloudHypervisor) MigrateAbort(ctx context.Context, ref, series string) error { + return ch.MigrateAbortSequence(ctx, ref, series, ch.migrateSpec(ctx)) +} + +func (ch *CloudHypervisor) MigrateReceive(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) { + var srcCfg *chVMConfig + spec := hypervisor.CloneSpec{ + VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, + AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { + return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg, true) + }, + } + return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { + var err error + srcCfg, err = cloneSeriesFiles(ctx, dstDir, srcDir) + return err + }) +} + +func (ch *CloudHypervisor) migrateSpec(ctx context.Context) hypervisor.MigrateSpec { + spec := ch.snapshotSpec(ctx) + spec.Capture = func(_ *hypervisor.VMRecord, hc *http.Client, dir string) error { + if err := snapshotVM(ctx, hc, dir, snapshotTypeDiff); err != nil { + return fmt.Errorf("snapshot: %w", err) + } + return nil + } + return hypervisor.MigrateSpec{ + SnapshotSpec: spec, + EndSeries: func(_ *hypervisor.VMRecord, hc *http.Client) error { + return endSnapshotSeries(context.WithoutCancel(ctx), hc) + }, + } +} + +func cloneSeriesFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig, error) { + chCfg, err := parseCHConfig(filepath.Join(srcDir, configJSONName)) + if err != nil { + return nil, fmt.Errorf("parse source config: %w", err) + } + classify := classifySnapshotFile(chCfg) + if err = hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, func(name string) hypervisor.SnapshotFileKind { + if hypervisor.IsMigrateDeltaFile(name) { + return hypervisor.SnapshotFileSkip + } + return classify(name) + }); err != nil { + return nil, err + } + rounds, err := countMemoryDiffs(srcDir) + if err != nil { + return nil, err + } + disks := slices.Collect(maps.Keys(identifyCOWFiles(chCfg))) + return chCfg, hypervisor.ApplyDiskDeltas(ctx, dstDir, srcDir, disks, rounds) +} + +func countMemoryDiffs(dir string) (int, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return 0, fmt.Errorf("read series dir: %w", err) + } + rounds := 0 + for _, e := range entries { + if strings.HasPrefix(e.Name(), memoryDiffPrefix) { + rounds++ + } + } + return rounds, nil +} diff --git a/hypervisor/cloudhypervisor/migrate_test.go b/hypervisor/cloudhypervisor/migrate_test.go new file mode 100644 index 00000000..c6c64278 --- /dev/null +++ b/hypervisor/cloudhypervisor/migrate_test.go @@ -0,0 +1,128 @@ +package cloudhypervisor + +import ( + "encoding/json/v2" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + + "github.com/cocoonstack/cocoon/config" + "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +func TestSnapshotVMSendsTheType(t *testing.T) { + tests := []struct { + name string + snapshotType string + want string + }{ + {name: "plain snapshot omits it", want: ""}, + {name: "diff", snapshotType: snapshotTypeDiff, want: "diff"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got map[string]string + sock := serveCHHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.UnmarshalRead(r.Body, &got) + w.WriteHeader(http.StatusNoContent) + })) + if err := snapshotVM(t.Context(), utils.NewSocketHTTPClient(sock), "/d", tt.snapshotType); err != nil { + t.Fatalf("snapshotVM: %v", err) + } + if got["destination_url"] != "file:///d" || got["snapshot_type"] != tt.want { + t.Errorf("payload = %v", got) + } + }) + } +} + +func TestEndSnapshotSeriesCallsTheEndpoint(t *testing.T) { + var path, method string + sock := serveCHHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path, method = r.URL.Path, r.Method + w.WriteHeader(http.StatusNoContent) + })) + if err := endSnapshotSeries(t.Context(), utils.NewSocketHTTPClient(sock)); err != nil { + t.Fatalf("endSnapshotSeries: %v", err) + } + if method != http.MethodPut || !strings.HasSuffix(path, "/vm.end-snapshot-series") { + t.Errorf("%s %s", method, path) + } +} + +func TestRestoreAndResumeCloneKeepPausedSkipsResume(t *testing.T) { + var ( + mu sync.Mutex + paths []string + ) + sock := serveCHHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, filepath.Base(r.URL.Path)) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + scopeParent := t.TempDir() + if err := os.Mkdir(filepath.Join(scopeParent, "vm-T.scope"), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + ch := &CloudHypervisor{ + Backend: &hypervisor.Backend{Conf: tmpScopeConf{Config: NewConfig(&config.Config{}), parent: scopeParent}}, + conf: NewConfig(&config.Config{}), + } + if err := ch.restoreAndResumeClone(t.Context(), 0, sock, t.TempDir(), &cloneResumeOpts{ + vmID: "T", + vmCfg: &types.VMConfig{CPU: 2}, + directBoot: true, + snapshotCfg: &chVMConfig{}, + keepPaused: true, + }); err != nil { + t.Fatalf("restoreAndResumeClone: %v", err) + } + mu.Lock() + defer mu.Unlock() + if slices.Contains(paths, "vm.resume") { + t.Errorf("calls = %v, want no vm.resume", paths) + } +} + +func TestCloneSeriesFilesLeavesDeltasBehind(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + writeMigrateFile(t, src, configJSONName, `{"disks":[{"path":"/run/cow.raw"}]}`) + for _, name := range []string{"cow.raw", memoryRangeFile + "s", "cow.raw.ref", stateJSONName} { + writeMigrateFile(t, src, name, "x") + } + if _, err := cloneSeriesFiles(t.Context(), dst, src); err != nil { + t.Fatalf("cloneSeriesFiles: %v", err) + } + if utils.FileExists(filepath.Join(dst, "cow.raw.ref")) { + t.Error("the reference reached the run dir") + } + for _, name := range []string{"cow.raw", memoryRangeFile + "s", stateJSONName} { + if !utils.FileExists(filepath.Join(dst, name)) { + t.Errorf("%s missing from the run dir", name) + } + } +} + +func TestCloneSeriesFilesFailsWithoutTheDiskDeltaOfARound(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + writeMigrateFile(t, src, configJSONName, `{"disks":[{"path":"/run/cow.raw"}]}`) + writeMigrateFile(t, src, "cow.raw", "x") + writeMigrateFile(t, src, memoryDiffPrefix+"1", "x") + if _, err := cloneSeriesFiles(t.Context(), dst, src); err == nil { + t.Fatal("cloneSeriesFiles accepted a round without its disk delta") + } +} + +func writeMigrateFile(t *testing.T, dir, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } +} diff --git a/hypervisor/cloudhypervisor/snapshot.go b/hypervisor/cloudhypervisor/snapshot.go index eadda364..d90d73eb 100644 --- a/hypervisor/cloudhypervisor/snapshot.go +++ b/hypervisor/cloudhypervisor/snapshot.go @@ -53,7 +53,7 @@ func (ch *CloudHypervisor) snapshotSpec(ctx context.Context) hypervisor.Snapshot }, Resume: func(_ *hypervisor.VMRecord, hc *http.Client) error { return resumeVM(context.WithoutCancel(ctx), hc) }, Capture: func(rec *hypervisor.VMRecord, hc *http.Client, tmpDir string) error { - if err := snapshotVM(ctx, hc, tmpDir); err != nil { + if err := snapshotVM(ctx, hc, tmpDir, ""); err != nil { return fmt.Errorf("snapshot: %w", err) } return hypervisor.CopyWritableDisks(ctx, tmpDir, rec) diff --git a/hypervisor/cloudhypervisor/utils.go b/hypervisor/cloudhypervisor/utils.go index 1c562e89..e7e04f84 100644 --- a/hypervisor/cloudhypervisor/utils.go +++ b/hypervisor/cloudhypervisor/utils.go @@ -48,6 +48,11 @@ var runtimeFiles = []string{hypervisor.APISocketName, pidFileName, hypervisor.Co type chMemoryRestoreMode string +type chSnapshotConfig struct { + DestinationURL string `json:"destination_url"` + SnapshotType string `json:"snapshot_type,omitempty"` +} + type chRestoreConfig struct { SourceURL string `json:"source_url"` MemoryRestoreMode chMemoryRestoreMode `json:"memory_restore_mode,omitempty"` @@ -134,10 +139,16 @@ func isAlreadyInStateError(err error, state string) bool { return strings.Contains(ae.Message, fmt.Sprintf("Invalid transition: InvalidStateTransition(%s, %s)", state, state)) } -func snapshotVM(ctx context.Context, hc *http.Client, destDir string) error { +func snapshotVM(ctx context.Context, hc *http.Client, destDir, snapshotType string) error { hc.Timeout = hypervisor.VMMemTransferTimeout defer func() { hc.Timeout = utils.HTTPTimeout }() - return vmPutJSON(ctx, hc, "vm.snapshot", "snapshot request", map[string]string{"destination_url": "file://" + destDir}, http.StatusNoContent) + cfg := chSnapshotConfig{DestinationURL: "file://" + destDir, SnapshotType: snapshotType} + return vmPutJSON(ctx, hc, "vm.snapshot", "snapshot request", cfg, http.StatusNoContent) +} + +func endSnapshotSeries(ctx context.Context, hc *http.Client) error { + _, err := vmAPIOnce(ctx, hc, "vm.end-snapshot-series") + return err } func restoreVM(ctx context.Context, hc *http.Client, sourceDir, restoreMode string) error { diff --git a/hypervisor/diskdelta.go b/hypervisor/diskdelta.go new file mode 100644 index 00000000..f1551897 --- /dev/null +++ b/hypervisor/diskdelta.go @@ -0,0 +1,228 @@ +package hypervisor + +import ( + "context" + "encoding/binary" + "encoding/json/v2" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strconv" + + "github.com/cocoonstack/cocoon/utils" +) + +const ( + diskDeltaInfix = ".delta." + + // deltaLengthSize is the byte count of the big-endian header length that opens a delta file. + deltaLengthSize = 8 +) + +// diskDelta heads a delta file; the bytes of each non-zero range follow in order. +type diskDelta struct { + Size int64 `json:"size"` + Ranges []deltaRange `json:"ranges"` +} + +type deltaRange struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Zero bool `json:"zero,omitzero"` +} + +// mapping is where one logical position lives; the zero value is a hole. +type mapping struct { + physical int64 + unwritten bool + mapped bool +} + +// ApplyDiskDeltas replays rounds 1..rounds of each named disk from srcDir onto its copy in dstDir. +func ApplyDiskDeltas(ctx context.Context, dstDir, srcDir string, disks []string, rounds int) error { + _, err := utils.Map(ctx, disks, func(_ context.Context, _ int, name string) (struct{}, error) { + for n := 1; n <= rounds; n++ { + if err := applyDiskDelta(filepath.Join(dstDir, name), filepath.Join(srcDir, diskDeltaName(name, n))); err != nil { + return struct{}{}, fmt.Errorf("apply %s round %d: %w", name, n, err) + } + } + return struct{}{}, nil + }) + return err +} + +func diskDeltaName(disk string, round int) string { + return disk + diskDeltaInfix + strconv.Itoa(round) +} + +// writeDiskDelta records how src differs from ref; the two must share extents through reflink, so an extent both map holds the same bytes. +func writeDiskDelta(dst, ref, src string) error { + srcFile, err := os.Open(src) //nolint:gosec + if err != nil { + return fmt.Errorf("open delta source: %w", err) + } + defer srcFile.Close() //nolint:errcheck + fi, err := srcFile.Stat() + if err != nil { + return fmt.Errorf("stat delta source: %w", err) + } + srcExtents, err := utils.FileExtents(srcFile) + if err != nil { + return err + } + refFile, err := os.Open(ref) //nolint:gosec + if err != nil { + return fmt.Errorf("open delta reference: %w", err) + } + defer refFile.Close() //nolint:errcheck + refExtents, err := utils.FileExtents(refFile) + if err != nil { + return err + } + header := diskDelta{Size: fi.Size(), Ranges: diffExtents(refExtents, srcExtents, fi.Size())} + return writeDeltaFile(dst, &header, srcFile) +} + +// diffExtents returns the ranges below size where cur and ref map different storage; a range cur reads as zeros is marked Zero. +func diffExtents(ref, cur []utils.Extent, size int64) []deltaRange { + var out []deltaRange + for pos := int64(0); pos < size; { + refAt, refEnd := extentAt(&ref, pos) + curAt, curEnd := extentAt(&cur, pos) + end := min(refEnd, curEnd, size) + if refAt != curAt { + zero := curAt == (mapping{}) || curAt.unwritten + if n := len(out) - 1; n >= 0 && out[n].Zero == zero && out[n].Offset+out[n].Length == pos { + out[n].Length += end - pos + } else { + out = append(out, deltaRange{Offset: pos, Length: end - pos, Zero: zero}) + } + } + pos = end + } + return out +} + +// extentAt advances *extents past pos and returns the mapping at pos plus the offset where it stops holding. +func extentAt(extents *[]utils.Extent, pos int64) (mapping, int64) { + for len(*extents) > 0 && (*extents)[0].Logical+(*extents)[0].Length <= pos { + *extents = (*extents)[1:] + } + if len(*extents) == 0 { + return mapping{}, math.MaxInt64 + } + e := (*extents)[0] + if pos < e.Logical { + return mapping{}, e.Logical + } + return mapping{physical: e.Physical + pos - e.Logical, unwritten: e.Unwritten, mapped: true}, e.Logical + e.Length +} + +func writeDeltaFile(dst string, header *diskDelta, src *os.File) (err error) { + raw, err := json.Marshal(header) + if err != nil { + return fmt.Errorf("encode delta header: %w", err) + } + tmp := dst + ".tmp" + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) //nolint:gosec + if err != nil { + return fmt.Errorf("create delta: %w", err) + } + defer func() { + err = errors.Join(err, out.Close()) + if err != nil { + _ = os.Remove(tmp) + return + } + err = os.Rename(tmp, dst) + }() + if err = binary.Write(out, binary.BigEndian, uint64(len(raw))); err != nil { + return fmt.Errorf("write delta header: %w", err) + } + if _, err = out.Write(raw); err != nil { + return fmt.Errorf("write delta header: %w", err) + } + for _, r := range header.Ranges { + if r.Zero { + continue + } + if _, err = src.Seek(r.Offset, io.SeekStart); err != nil { + return fmt.Errorf("seek delta source to %d: %w", r.Offset, err) + } + if _, err = io.CopyN(out, src, r.Length); err != nil { + return fmt.Errorf("copy range at %d len %d: %w", r.Offset, r.Length, err) + } + } + return nil +} + +func applyDiskDelta(dst, delta string) error { + in, err := os.Open(delta) //nolint:gosec + if err != nil { + return err + } + defer in.Close() //nolint:errcheck + header, err := readDeltaHeader(in) + if err != nil { + return err + } + out, err := os.OpenFile(dst, os.O_WRONLY, 0) //nolint:gosec + if err != nil { + return err + } + defer out.Close() //nolint:errcheck + if err = out.Truncate(header.Size); err != nil { + return fmt.Errorf("resize %s: %w", dst, err) + } + for _, r := range header.Ranges { + if r.Zero { + if err = utils.PunchHole(out, r.Offset, r.Length); err != nil { + return err + } + continue + } + if _, err = out.Seek(r.Offset, io.SeekStart); err != nil { + return fmt.Errorf("seek %s to %d: %w", dst, r.Offset, err) + } + if _, err = io.CopyN(out, in, r.Length); err != nil { + return fmt.Errorf("write range at %d len %d: %w", r.Offset, r.Length, err) + } + } + return nil +} + +// readDeltaHeader leaves f at the first data byte and rejects a file whose size disagrees with its header. +func readDeltaHeader(f *os.File) (*diskDelta, error) { + var length uint64 + if err := binary.Read(f, binary.BigEndian, &length); err != nil { + return nil, fmt.Errorf("read delta header length: %w", err) + } + fi, err := f.Stat() + if err != nil { + return nil, err + } + if length > uint64(fi.Size()) { //nolint:gosec + return nil, fmt.Errorf("delta header length %d exceeds file size %d", length, fi.Size()) + } + raw := make([]byte, length) + if _, err = io.ReadFull(f, raw); err != nil { + return nil, fmt.Errorf("read delta header: %w", err) + } + var header diskDelta + if err = json.Unmarshal(raw, &header); err != nil { + return nil, fmt.Errorf("decode delta header: %w", err) + } + want := deltaLengthSize + int64(length) + for _, r := range header.Ranges { + if !r.Zero { + want += r.Length + } + } + if fi.Size() != want { + return nil, fmt.Errorf("delta is %d bytes, its header needs %d: the transfer is incomplete", fi.Size(), want) + } + return &header, nil +} diff --git a/hypervisor/diskdelta_test.go b/hypervisor/diskdelta_test.go new file mode 100644 index 00000000..2264360e --- /dev/null +++ b/hypervisor/diskdelta_test.go @@ -0,0 +1,199 @@ +package hypervisor + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/cocoon/utils" +) + +func TestDiffExtents(t *testing.T) { + tests := []struct { + name string + ref []utils.Extent + cur []utils.Extent + size int64 + want []deltaRange + }{ + { + name: "shared extents differ nowhere", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + size: 10, + }, + { + name: "split extent with the same storage differs nowhere", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}, {Logical: 4, Physical: 104, Length: 6}}, + size: 10, + }, + { + name: "rewritten middle", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}, {Logical: 4, Physical: 900, Length: 2}, {Logical: 6, Physical: 106, Length: 4}}, + size: 10, + want: []deltaRange{{Offset: 4, Length: 2}}, + }, + { + name: "new data in a hole", + cur: []utils.Extent{{Logical: 2, Physical: 900, Length: 2}}, + size: 10, + want: []deltaRange{{Offset: 2, Length: 2}}, + }, + { + name: "discarded data", + ref: []utils.Extent{{Logical: 2, Physical: 100, Length: 2}}, + size: 10, + want: []deltaRange{{Offset: 2, Length: 2, Zero: true}}, + }, + { + name: "unwritten extent reads as zeros", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}}, + cur: []utils.Extent{{Logical: 0, Physical: 900, Length: 4, Unwritten: true}}, + size: 4, + want: []deltaRange{{Offset: 0, Length: 4, Zero: true}}, + }, + { + name: "written over a shared unwritten extent", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 4, Unwritten: true}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}}, + size: 4, + want: []deltaRange{{Offset: 0, Length: 4}}, + }, + { + name: "adjacent changes of one kind merge", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}}, + cur: []utils.Extent{{Logical: 0, Physical: 900, Length: 2}, {Logical: 2, Physical: 500, Length: 2}, {Logical: 4, Physical: 700, Length: 2}}, + size: 6, + want: []deltaRange{{Offset: 0, Length: 6}}, + }, + { + name: "extent past the end of the file is cut", + cur: []utils.Extent{{Logical: 0, Physical: 900, Length: 100}}, + size: 10, + want: []deltaRange{{Offset: 0, Length: 10}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := diffExtents(tt.ref, tt.cur, tt.size); !slices.Equal(got, tt.want) { + t.Errorf("got %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestDiskDeltaRoundTrip(t *testing.T) { + dir := t.TempDir() + ref := filepath.Join(dir, "ref") + src := filepath.Join(dir, "src") + writeSparse(t, ref, 1<<20, map[int64]string{0: "base", 1 << 19: "kept"}) + writeSparse(t, src, 2<<20, map[int64]string{0: "next", 1 << 19: "kept", 3 << 19: "grown"}) + delta := filepath.Join(dir, diskDeltaName("disk", 1)) + if err := writeDiskDelta(delta, ref, src); err != nil { + if errors.Is(err, errors.ErrUnsupported) { + t.Skip("needs FIEMAP") + } + t.Fatalf("writeDiskDelta: %v", err) + } + if err := applyDiskDelta(ref, delta); err != nil { + t.Fatalf("applyDiskDelta: %v", err) + } + got, err := os.ReadFile(ref) + if err != nil { + t.Fatalf("read: %v", err) + } + want, err := os.ReadFile(src) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("applied disk differs from the source") + } +} + +func TestApplyDiskDeltaPunchesZeroRanges(t *testing.T) { + dir := t.TempDir() + dst := filepath.Join(dir, "dst") + writeSparse(t, dst, 1<<20, map[int64]string{0: "stale"}) + src, err := os.Create(filepath.Join(dir, "src")) + if err != nil { + t.Fatalf("create: %v", err) + } + defer src.Close() + delta := filepath.Join(dir, "delta") + if err = writeDeltaFile(delta, &diskDelta{Size: 1 << 20, Ranges: []deltaRange{{Offset: 0, Length: 4096, Zero: true}}}, src); err != nil { + t.Fatalf("writeDeltaFile: %v", err) + } + if err = applyDiskDelta(dst, delta); err != nil { + if errors.Is(err, errors.ErrUnsupported) { + t.Skip("needs fallocate") + } + t.Fatalf("applyDiskDelta: %v", err) + } + got, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got[:5], make([]byte, 5)) { + t.Errorf("range reads %q, want zeros", got[:5]) + } +} + +func TestApplyDiskDeltaRejectsATruncatedFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + writeSparse(t, src, 4096, map[int64]string{0: "data"}) + srcFile, err := os.Open(src) + if err != nil { + t.Fatalf("open: %v", err) + } + defer srcFile.Close() + delta := filepath.Join(dir, "delta") + if err = writeDeltaFile(delta, &diskDelta{Size: 4096, Ranges: []deltaRange{{Offset: 0, Length: 4096}}}, srcFile); err != nil { + t.Fatalf("writeDeltaFile: %v", err) + } + fi, err := os.Stat(delta) + if err != nil { + t.Fatalf("stat: %v", err) + } + if err = os.Truncate(delta, fi.Size()-1); err != nil { + t.Fatalf("truncate: %v", err) + } + dst := filepath.Join(dir, "dst") + writeSparse(t, dst, 4096, nil) + if err = applyDiskDelta(dst, delta); err == nil || !strings.Contains(err.Error(), "incomplete") { + t.Errorf("applyDiskDelta: %v, want incomplete transfer", err) + } +} + +func TestApplyDiskDeltasFailsOnAMissingRound(t *testing.T) { + dst, src := t.TempDir(), t.TempDir() + writeSparse(t, filepath.Join(dst, "disk"), 4096, nil) + err := ApplyDiskDeltas(t.Context(), dst, src, []string{"disk"}, 1) + if !errors.Is(err, os.ErrNotExist) { + t.Errorf("ApplyDiskDeltas: %v, want not exist", err) + } +} + +func writeSparse(t *testing.T, path string, size int64, chunks map[int64]string) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer f.Close() + if err = f.Truncate(size); err != nil { + t.Fatalf("truncate: %v", err) + } + for off, data := range chunks { + if _, err = f.WriteAt([]byte(data), off); err != nil { + t.Fatalf("write: %v", err) + } + } +} diff --git a/hypervisor/disks.go b/hypervisor/disks.go index 098d9579..7314c1a2 100644 --- a/hypervisor/disks.go +++ b/hypervisor/disks.go @@ -85,16 +85,10 @@ func RecordedCOWPath(rec *VMRecord) (string, error) { // CopyWritableDisks copies concurrently; durability is paid at persist. func CopyWritableDisks(ctx context.Context, dstDir string, rec *VMRecord) error { - cowPath, err := RecordedCOWPath(rec) + pairs, err := writableDiskPairs(dstDir, rec) if err != nil { return err } - pairs := [][2]string{{filepath.Join(dstDir, filepath.Base(cowPath)), cowPath}} - for _, sc := range rec.StorageConfigs { - if sc.Role == types.StorageRoleData { - pairs = append(pairs, [2]string{filepath.Join(dstDir, DataDiskBaseName(sc.Serial)), sc.Path}) - } - } return copyPairs(ctx, pairs) } @@ -127,6 +121,21 @@ func PrepareDataDisks(ctx context.Context, baseDir string, specs []types.DataDis return out, nil } +// writableDiskPairs returns {copy under dstDir, live disk} for the COW and every data disk. +func writableDiskPairs(dstDir string, rec *VMRecord) ([][2]string, error) { + cowPath, err := RecordedCOWPath(rec) + if err != nil { + return nil, err + } + pairs := [][2]string{{filepath.Join(dstDir, filepath.Base(cowPath)), cowPath}} + for _, sc := range rec.StorageConfigs { + if sc.Role == types.StorageRoleData { + pairs = append(pairs, [2]string{filepath.Join(dstDir, DataDiskBaseName(sc.Serial)), sc.Path}) + } + } + return pairs, nil +} + func copyPairs(ctx context.Context, pairs [][2]string) error { _, err := utils.Map(ctx, pairs, func(ctx context.Context, _ int, p [2]string) (struct{}, error) { if err := utils.ReflinkCopy(ctx, p[0], p[1], utils.NoSync); err != nil { diff --git a/hypervisor/firecracker/clone.go b/hypervisor/firecracker/clone.go index f85dca26..2eb0c9e2 100644 --- a/hypervisor/firecracker/clone.go +++ b/hypervisor/firecracker/clone.go @@ -18,6 +18,7 @@ import ( "github.com/cocoonstack/cocoon/hypervisor" "github.com/cocoonstack/cocoon/lock/flock" "github.com/cocoonstack/cocoon/lock/vmlock" + "github.com/cocoonstack/cocoon/metering" "github.com/cocoonstack/cocoon/snapshot" "github.com/cocoonstack/cocoon/types" "github.com/cocoonstack/cocoon/utils" @@ -109,7 +110,7 @@ func (fc *Firecracker) cloneAfterExtract(ctx context.Context, rec *hypervisor.VM storageConfigs = append(storageConfigs, dataDisks...) info := fc.RunningCloneRecord(rec, vmCfg, storageConfigs, net) - if err := fc.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID); err != nil { + if err := fc.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID, metering.ReasonClone); err != nil { leaseControl.close() fc.AbortLaunch(ctx, pid, sockPath, runDir, runtimeFiles) return nil, fmt.Errorf("finalize VM record: %w", err) diff --git a/hypervisor/hypervisor.go b/hypervisor/hypervisor.go index e3a16d8c..12812ed3 100644 --- a/hypervisor/hypervisor.go +++ b/hypervisor/hypervisor.go @@ -18,6 +18,8 @@ var ( ErrAmbiguous = errors.New("vm ref resolves to multiple backends") ErrHotAttached = errors.New("detach before snapshot or hibernate") + + ErrMigrateUnsupported = errors.New("backend does not support live migration") ) // Hypervisor manages VM lifecycle. diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go new file mode 100644 index 00000000..498a18d8 --- /dev/null +++ b/hypervisor/migrate.go @@ -0,0 +1,294 @@ +package hypervisor + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/cocoonstack/cocoon/snapshot" + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +const diskRefSuffix = ".ref" + +// Migrator captures a running VM in rounds and restores the result on another node. +type Migrator interface { + // MigrateCapture starts a new series when req.Series is empty and adds a round to it otherwise. + MigrateCapture(ctx context.Context, ref string, req MigrateRequest) (*MigrateRound, error) + // MigrateReceive restores a transferred series into a new VM and leaves its guest paused; srcDir stays read-only. + MigrateReceive(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) + // MigrateAbort ends the series, resumes the guest and deletes the series directory. + MigrateAbort(ctx context.Context, ref, series string) error +} + +// MigrateRequest selects the round to capture; Final leaves the guest paused. +type MigrateRequest struct { + Series string + Final bool +} + +// MigrateRound reports one captured round; round 0 is the baseline and Files lists what the round wrote. +type MigrateRound struct { + Series string `json:"series"` + Dir string `json:"dir"` + Round int `json:"round"` + Paused bool `json:"paused"` + Files []MigrateFile `json:"files"` +} + +// MigrateFile is one file a round wrote. +type MigrateFile struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +// MigrateSpec carries backend hooks for the migrate sequences; SnapshotSpec.Capture writes the guest state of one round. +type MigrateSpec struct { + SnapshotSpec + EndSeries TransitionHook +} + +// roundSteps is what a baseline and a delta do differently around the shared pause window; round 0 is the baseline. +type roundSteps struct { + series string + round int + final bool + inPause func() error + afterPause func() error +} + +type fileStamp struct { + size int64 + modTime int64 +} + +// MigrateCaptureSequence runs one round; any failure ends the series, resumes the guest and deletes the directory. +func (b *Backend) MigrateCaptureSequence(ctx context.Context, ref string, req MigrateRequest, spec MigrateSpec) (*MigrateRound, error) { + vmID, rec, unlock, err := b.lockForCapture(ctx, ref) + if err != nil { + return nil, err + } + defer unlock() + var round *MigrateRound + if req.Series == "" { + round, err = b.captureBaseline(ctx, &rec, spec) + } else { + round, err = b.captureDelta(ctx, &rec, req, spec) + } + if err != nil { + return nil, fmt.Errorf("migrate capture VM %s: %w", vmID, err) + } + return round, nil +} + +// MigrateAbortSequence is idempotent; a VM that no longer runs only loses the directory. +func (b *Backend) MigrateAbortSequence(ctx context.Context, ref, series string, spec MigrateSpec) error { + vmID, rec, unlock, err := b.lockForCapture(ctx, ref) + if err != nil { + return err + } + defer unlock() + dir, err := seriesDir(&rec, series) + if err != nil { + return err + } + hc := utils.NewSocketHTTPClient(SocketPath(rec.RunDir)) + runErr := b.WithRunningVM(ctx, &rec, func(_ int) error { + return errors.Join(spec.EndSeries(&rec, hc), resumeAfterCapture(spec.Resume, &rec, hc)) + }) + if errors.Is(runErr, ErrNotRunning) { + runErr = nil + } + if runErr = errors.Join(runErr, os.RemoveAll(dir)); runErr != nil { + return fmt.Errorf("migrate abort VM %s: %w", vmID, runErr) + } + return nil +} + +func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec MigrateSpec) (*MigrateRound, error) { + // A fallback copy of the disks would run inside the pause and share no extents with the live disk. + if err := utils.CheckReflink(rec.RunDir); err != nil { + return nil, fmt.Errorf("live migration needs a reflink filesystem under the run dir: %w", err) + } + dir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) + if err != nil { + return nil, fmt.Errorf("create series dir: %w", err) + } + series := strings.TrimPrefix(filepath.Base(dir), captureDirPrefix) + return b.captureRound(ctx, rec, spec, dir, roundSteps{ + series: series, + inPause: func() error { + return CopyWritableDisks(ctx, dir, rec) + }, + afterPause: func() error { + if err := saveCaptureMeta(rec, spec.SnapshotSpec, dir); err != nil { + return err + } + return snapshot.WriteSnapshotEnvelope(dir, *b.BuildSnapshotConfig(series, rec), nil, nil) + }, + }) +} + +func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRequest, spec MigrateSpec) (*MigrateRound, error) { + dir, err := seriesDir(rec, req.Series) + if err != nil { + return nil, err + } + if _, err = os.Stat(dir); err != nil { + return nil, fmt.Errorf("series %s: %w", req.Series, err) + } + pairs, err := writableDiskPairs(dir, rec) + if err != nil { + return nil, err + } + number := nextRound(pairs[0][0]) + steps := roundSteps{ + series: req.Series, + round: number, + final: req.Final, + inPause: func() error { + if req.Final { + return nil + } + return reflinkNextRefs(ctx, pairs) + }, + afterPause: func() error { + _, mapErr := utils.Map(ctx, pairs, func(_ context.Context, _ int, p [2]string) (struct{}, error) { + return struct{}{}, writeRoundDelta(p, number, req.Final) + }) + return mapErr + }, + } + return b.captureRound(ctx, rec, spec, dir, steps) +} + +func (b *Backend) captureRound(ctx context.Context, rec *VMRecord, spec MigrateSpec, dir string, steps roundSteps) (*MigrateRound, error) { + before, err := stampFiles(dir) + if err != nil { + return nil, errors.Join(err, os.RemoveAll(dir)) + } + hc := utils.NewSocketHTTPClient(SocketPath(rec.RunDir)) + run := func() error { + if steps.round == 0 { + if endErr := spec.EndSeries(rec, hc); endErr != nil { + return fmt.Errorf("end previous series: %w", endErr) + } + } + if pauseErr := spec.Pause(rec, hc); pauseErr != nil { + return fmt.Errorf("pause: %w", pauseErr) + } + if captureErr := spec.Capture(rec, hc, dir); captureErr != nil { + return captureErr + } + if stepErr := steps.inPause(); stepErr != nil { + return stepErr + } + if !steps.final { + if resumeErr := resumeAfterCapture(spec.Resume, rec, hc); resumeErr != nil { + return resumeErr + } + } + return steps.afterPause() + } + if err = b.WithRunningVM(ctx, rec, func(_ int) error { + if runErr := run(); runErr != nil { + return errors.Join(runErr, spec.EndSeries(rec, hc), resumeAfterCapture(spec.Resume, rec, hc)) + } + return nil + }); err != nil { + return nil, errors.Join(err, os.RemoveAll(dir)) + } + files, err := changedFiles(dir, before) + if err != nil { + return nil, err + } + return &MigrateRound{Series: steps.series, Dir: dir, Round: steps.round, Paused: steps.final, Files: files}, nil +} + +// IsMigrateDeltaFile reports a disk delta or reference a clone of a series must leave behind. +func IsMigrateDeltaFile(name string) bool { + return strings.HasSuffix(name, diskRefSuffix) || strings.Contains(name, diskDeltaInfix) +} + +func seriesDir(rec *VMRecord, series string) (string, error) { + if series == "" || !filepath.IsLocal(series) || strings.ContainsRune(series, filepath.Separator) { + return "", fmt.Errorf("invalid series id %q", series) + } + return filepath.Join(rec.RunDir, captureDirPrefix+series), nil +} + +func nextRound(disk string) int { + n := 1 + for utils.FileExists(filepath.Join(filepath.Dir(disk), diskDeltaName(filepath.Base(disk), n))) { + n++ + } + return n +} + +// reflinkNextRefs freezes each disk in the pause window so the delta can be written after the guest resumes. +func reflinkNextRefs(ctx context.Context, pairs [][2]string) error { + refs := make([][2]string, len(pairs)) + for i, p := range pairs { + refs[i] = [2]string{nextRefPath(p[0]), p[1]} + } + return copyPairs(ctx, refs) +} + +func writeRoundDelta(pair [2]string, round int, final bool) error { + copyPath, live := pair[0], pair[1] + ref := copyPath + diskRefSuffix + if !utils.FileExists(ref) { + ref = copyPath + } + dst := filepath.Join(filepath.Dir(copyPath), diskDeltaName(filepath.Base(copyPath), round)) + if final { + return writeDiskDelta(dst, ref, live) + } + next := nextRefPath(copyPath) + if err := writeDiskDelta(dst, ref, next); err != nil { + return err + } + return os.Rename(next, copyPath+diskRefSuffix) +} + +func nextRefPath(copyPath string) string { + return copyPath + ".next" + diskRefSuffix +} + +func stampFiles(dir string) (map[string]fileStamp, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read series dir: %w", err) + } + stamps := make(map[string]fileStamp, len(entries)) + for _, e := range entries { + info, err := e.Info() + if err != nil { + return nil, fmt.Errorf("stat %s: %w", e.Name(), err) + } + if info.Mode().IsRegular() && !strings.HasSuffix(e.Name(), diskRefSuffix) { + stamps[e.Name()] = fileStamp{size: info.Size(), modTime: info.ModTime().UnixNano()} + } + } + return stamps, nil +} + +func changedFiles(dir string, before map[string]fileStamp) ([]MigrateFile, error) { + after, err := stampFiles(dir) + if err != nil { + return nil, err + } + files := []MigrateFile{} + for _, name := range slices.Sorted(maps.Keys(after)) { + if stamp, ok := before[name]; !ok || stamp != after[name] { + files = append(files, MigrateFile{Name: name, Size: after[name].size}) + } + } + return files, nil +} diff --git a/hypervisor/migrate_test.go b/hypervisor/migrate_test.go new file mode 100644 index 00000000..95fa5060 --- /dev/null +++ b/hypervisor/migrate_test.go @@ -0,0 +1,244 @@ +package hypervisor + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/cocoon/snapshot" + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +func TestMigrateBaselineRefusesWithoutReflink(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + if utils.CheckReflink(rec.RunDir) == nil { + t.Skip("the test filesystem supports reflink") + } + calls := &migrateCalls{} + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{}, migrateStubSpec(calls)); err == nil { + t.Fatal("capture succeeded without reflink") + } + if calls.paused != 0 { + t.Error("the guest was paused before the refusal") + } + if dirs, _ := filepath.Glob(filepath.Join(rec.RunDir, captureDirPrefix+"*")); len(dirs) != 0 { + t.Errorf("series dirs left behind: %v", dirs) + } +} + +func TestMigrateMiddleRoundResumesAndHidesTheReference(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + round, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, migrateStubSpec(calls)) + if err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("capture: %v", err) + } + if round.Round != 1 || round.Paused || calls.resumed != 1 { + t.Errorf("round %d paused %t resumed %d, want 1 false 1", round.Round, round.Paused, calls.resumed) + } + if want := []string{"cow.raw.delta.1", "mem.1"}; !slices.Equal(fileNames(round), want) { + t.Errorf("files = %v, want %v", fileNames(round), want) + } + if !utils.FileExists(filepath.Join(dir, "cow.raw.ref")) { + t.Error("the next round has no reference") + } + round, err = b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, migrateStubSpec(calls)) + if err != nil { + t.Fatalf("second capture: %v", err) + } + if round.Round != 2 { + t.Errorf("round = %d, want 2", round.Round) + } +} + +func TestMigrateFinalRoundLeavesTheGuestPaused(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + round, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1", Final: true}, migrateStubSpec(calls)) + if err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("capture: %v", err) + } + if !round.Paused || calls.resumed != 0 || calls.ended != 0 { + t.Errorf("paused %t resumed %d ended %d, want true 0 0", round.Paused, calls.resumed, calls.ended) + } + if utils.FileExists(filepath.Join(dir, "cow.raw.ref")) { + t.Error("the final round reflinked a reference inside the pause") + } +} + +func TestMigrateFailedRoundResumesAndDiscardsTheSeries(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + spec.Capture = func(*VMRecord, *http.Client, string) error { return errors.New("dump failed") } + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1", Final: true}, spec); err == nil || !strings.Contains(err.Error(), "dump failed") { + t.Fatalf("capture: %v, want dump failed", err) + } + if calls.ended != 1 || calls.resumed != 1 { + t.Errorf("ended %d resumed %d, want 1 1", calls.ended, calls.resumed) + } + if utils.FileExists(dir) { + t.Error("the failed series survived") + } +} + +func TestMigrateUnknownSeriesKeepsTheGuestRunning(t *testing.T) { + b, id, _ := newMigrateTestVM(t) + calls := &migrateCalls{} + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "gone"}, migrateStubSpec(calls)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("capture: %v, want not exist", err) + } + if calls.paused != 0 || calls.ended != 0 { + t.Errorf("paused %d ended %d, want 0 0", calls.paused, calls.ended) + } +} + +func TestMigrateAbortIsIdempotent(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + for range 2 { + if err := b.MigrateAbortSequence(t.Context(), id, "s1", migrateStubSpec(calls)); err != nil { + t.Fatalf("abort: %v", err) + } + } + if calls.ended != 2 || calls.resumed != 2 { + t.Errorf("ended %d resumed %d, want 2 2", calls.ended, calls.resumed) + } + if utils.FileExists(dir) { + t.Error("the series survived the abort") + } +} + +func TestMigrateAbortResumesWhenEndingTheSeriesFails(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + spec.EndSeries = func(*VMRecord, *http.Client) error { return errors.New("stop dirty log") } + if err := b.MigrateAbortSequence(t.Context(), id, "s1", spec); err == nil { + t.Fatal("abort hid the failure") + } + if calls.resumed != 1 || utils.FileExists(dir) { + t.Errorf("resumed %d series kept %t, want 1 false", calls.resumed, utils.FileExists(dir)) + } +} + +func TestMigrateAbortOfAStoppedVMDeletesTheSeries(t *testing.T) { + b, _ := newMeteringTestBackend(t) + seedVMRecord(t, b, "stopped", 1, 512, 1024, true) + rec, err := b.LoadRecord(t.Context(), "stopped") + if err != nil { + t.Fatalf("load: %v", err) + } + dir := seedSeries(t, &rec) + calls := &migrateCalls{} + if err = b.MigrateAbortSequence(t.Context(), "stopped", "s1", migrateStubSpec(calls)); err != nil { + t.Fatalf("abort: %v", err) + } + if calls.resumed != 0 || utils.FileExists(dir) { + t.Errorf("resumed %d series kept %t, want 0 false", calls.resumed, utils.FileExists(dir)) + } +} + +func TestSeriesDirRejectsPathsOutOfTheRunDir(t *testing.T) { + rec := &VMRecord{RunDir: "/run/vm"} + for _, series := range []string{"", "..", "../x", "a/b", "/abs"} { + if _, err := seriesDir(rec, series); err == nil { + t.Errorf("series %q accepted", series) + } + } + got, err := seriesDir(rec, "123") + if err != nil || got != "/run/vm/snapshot-123" { + t.Errorf("got %q, %v", got, err) + } +} + +func TestIsMigrateDeltaFile(t *testing.T) { + for name, want := range map[string]bool{ + "cow.raw.delta.1": true, + "data-x.raw.delta.12": true, + "cow.raw.ref": true, + "cow.raw.next.ref": true, + "cow.raw": false, + "memory-ranges.diff.1": false, + "state.json": false, + } { + if got := IsMigrateDeltaFile(name); got != want { + t.Errorf("%s = %t, want %t", name, got, want) + } + } +} + +type migrateCalls struct { + paused int + resumed int + ended int + rounds int +} + +func migrateStubSpec(calls *migrateCalls) MigrateSpec { + return MigrateSpec{ + Pause: func(*VMRecord, *http.Client) error { calls.paused++; return nil }, + Resume: func(*VMRecord, *http.Client) error { calls.resumed++; return nil }, + Capture: func(_ *VMRecord, _ *http.Client, dir string) error { + calls.rounds++ + return os.WriteFile(filepath.Join(dir, "mem."+string(rune('0'+calls.rounds))), []byte("x"), 0o600) + }, + BuildMeta: func(*VMRecord, string) (*snapshot.SnapshotMeta, error) { return &snapshot.SnapshotMeta{}, nil }, + EndSeries: func(*VMRecord, *http.Client) error { calls.ended++; return nil }, + } +} + +func newMigrateTestVM(t *testing.T) (*Backend, string, *VMRecord) { + t.Helper() + b, id := newHibernateTestVM(t) + rec, err := b.LoadRecord(t.Context(), id) + if err != nil { + t.Fatalf("load: %v", err) + } + cow := filepath.Join(rec.RunDir, types.COWRawFileName) + writeSparse(t, cow, 1<<20, map[int64]string{0: "live"}) + if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error { + idx.VMs[id].StorageConfigs = []*types.StorageConfig{{Path: cow, Serial: CowSerial, Role: types.StorageRoleCOW}} + return nil + }); err != nil { + t.Fatalf("seed disks: %v", err) + } + return b, id, &rec +} + +func seedSeries(t *testing.T, rec *VMRecord) string { + t.Helper() + dir := filepath.Join(rec.RunDir, captureDirPrefix+"s1") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + writeSparse(t, filepath.Join(dir, types.COWRawFileName), 1<<20, map[int64]string{0: "base"}) + return dir +} + +func fileNames(round *MigrateRound) []string { + names := make([]string, len(round.Files)) + for i, f := range round.Files { + names[i] = f.Name + } + return names +} + +func skipWithoutFiemap(t *testing.T, err error) { + t.Helper() + if errors.Is(err, errors.ErrUnsupported) { + t.Skip("needs FIEMAP") + } +} diff --git a/hypervisor/snapshot.go b/hypervisor/snapshot.go index 9736f9df..753ea31b 100644 --- a/hypervisor/snapshot.go +++ b/hypervisor/snapshot.go @@ -147,44 +147,41 @@ func (b *Backend) HibernateSequence(ctx context.Context, ref string, spec Hibern } func (b *Backend) prepareSnapshot(ctx context.Context, ref string) (string, VMRecord, string, func(), error) { - vmID, err := b.ResolveRef(ctx, ref) + vmID, rec, unlock, err := b.lockForCapture(ctx, ref) if err != nil { return "", VMRecord{}, "", nil, err } - unlock, err := b.LockVMOps(ctx, vmID) + tmpDir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) if err != nil { - return "", VMRecord{}, "", nil, err - } - fail := func(err error) (string, VMRecord, string, func(), error) { unlock() - return "", VMRecord{}, "", nil, err + return "", VMRecord{}, "", nil, fmt.Errorf("create temp dir: %w", err) } - rec, err := b.EntryGuardLoad(ctx, vmID) + return vmID, rec, tmpDir, unlock, nil +} + +func (b *Backend) lockForCapture(ctx context.Context, ref string) (string, VMRecord, func(), error) { + vmID, err := b.ResolveRef(ctx, ref) if err != nil { - return fail(err) + return "", VMRecord{}, nil, err } - if vErr := validateRecordInvariants(&rec); vErr != nil { - return fail(vErr) + unlock, err := b.LockVMOps(ctx, vmID) + if err != nil { + return "", VMRecord{}, nil, err + } + rec, err := b.EntryGuardLoad(ctx, vmID) + if err == nil { + err = validateRecordInvariants(&rec) } - tmpDir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) if err != nil { - return fail(fmt.Errorf("create temp dir: %w", err)) + unlock() + return "", VMRecord{}, nil, err } - return vmID, rec, tmpDir, unlock, nil + return vmID, rec, unlock, nil } func (b *Backend) finalizeSnapshot(ctx context.Context, vmID string, rec *VMRecord, spec SnapshotSpec, tmpDir string) (*types.SnapshotConfig, error) { - if spec.AfterCapture != nil { - if err := spec.AfterCapture(rec, tmpDir); err != nil { - return nil, err - } - } - meta, err := spec.BuildMeta(rec, tmpDir) - if err != nil { - return nil, fmt.Errorf("build snapshot metadata: %w", err) - } - if err = snapshot.SaveSnapshotMeta(tmpDir, meta); err != nil { - return nil, fmt.Errorf("save snapshot metadata: %w", err) + if err := saveCaptureMeta(rec, spec, tmpDir); err != nil { + return nil, err } snapID, err := b.RecordSnapshot(ctx, vmID) if err != nil { @@ -260,6 +257,22 @@ func ReverseLayers[T any](storageConfigs []*types.StorageConfig, project func(id return out } +func saveCaptureMeta(rec *VMRecord, spec SnapshotSpec, tmpDir string) error { + if spec.AfterCapture != nil { + if err := spec.AfterCapture(rec, tmpDir); err != nil { + return err + } + } + meta, err := spec.BuildMeta(rec, tmpDir) + if err != nil { + return fmt.Errorf("build snapshot metadata: %w", err) + } + if err = snapshot.SaveSnapshotMeta(tmpDir, meta); err != nil { + return fmt.Errorf("save snapshot metadata: %w", err) + } + return nil +} + func resumeAfterCapture(resume TransitionHook, rec *VMRecord, hc *http.Client) error { hc.Timeout = VMMemTransferTimeout if err := resume(rec, hc); err != nil { diff --git a/hypervisor/state_test.go b/hypervisor/state_test.go index 71a8a961..b4597447 100644 --- a/hypervisor/state_test.go +++ b/hypervisor/state_test.go @@ -215,7 +215,7 @@ func TestFinalizeCloneEmitsCloneEntries(t *testing.T) { State: types.VMStateRunning, Config: types.VMConfig{CPU: 2, Memory: 2 << 30, Storage: 20 << 30}, } - if err := b.FinalizeClone(ctx, "vm1", info, nil, nil, "snap-source"); err != nil { + if err := b.FinalizeClone(ctx, "vm1", info, nil, nil, "snap-source", metering.ReasonClone); err != nil { t.Fatalf("FinalizeClone: %v", err) } entries := rec.Entries() diff --git a/hypervisor/utils.go b/hypervisor/utils.go index bc388463..fb49c84b 100644 --- a/hypervisor/utils.go +++ b/hypervisor/utils.go @@ -34,6 +34,8 @@ const ( SnapshotFileCOW // SnapshotFileMeta is small metadata that is plain-copied. SnapshotFileMeta + // SnapshotFileSkip is left out of the clone. + SnapshotFileSkip // CloneLocksDirName holds FC clone locks under the backend run root, outside any VM dir — a clone must be able to lock a source whose dir is already gone. CloneLocksDirName = "clone-locks" diff --git a/metering/metering.go b/metering/metering.go index 09e6aba9..f98a542b 100644 --- a/metering/metering.go +++ b/metering/metering.go @@ -20,6 +20,7 @@ const ( ReasonRestart Reason = "restart" ReasonClone Reason = "clone" ReasonRestore Reason = "restore" + ReasonMigrate Reason = "migrate" ReasonStopUser Reason = "stop-user" ReasonStopCrash Reason = "stop-crash" ReasonVMRemove Reason = "vm-rm" diff --git a/utils/extents.go b/utils/extents.go new file mode 100644 index 00000000..d98d05f4 --- /dev/null +++ b/utils/extents.go @@ -0,0 +1,9 @@ +package utils + +// Extent maps a file range to its on-disk location. +type Extent struct { + Logical int64 + Physical int64 + Length int64 + Unwritten bool +} diff --git a/utils/extents_linux.go b/utils/extents_linux.go new file mode 100644 index 00000000..03ca329f --- /dev/null +++ b/utils/extents_linux.go @@ -0,0 +1,78 @@ +//go:build linux + +package utils + +import ( + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + // fsIocFiemap is FS_IOC_FIEMAP; x/sys/unix does not define it. + fsIocFiemap uintptr = 0xC020660B + + fiemapFlagSync uint32 = 0x1 + fiemapExtentLast uint32 = 0x1 + fiemapExtentUnwritten uint32 = 0x800 + + fiemapBatch = 512 +) + +type fiemapExtent struct { + Logical uint64 + Physical uint64 + Length uint64 + reserved64 [2]uint64 + Flags uint32 + reserved [3]uint32 +} + +type fiemapRequest struct { + Start uint64 + Length uint64 + Flags uint32 + Mapped uint32 + Count uint32 + _ uint32 + Extents [fiemapBatch]fiemapExtent +} + +// FileExtents flushes f and returns its extents in logical order. +func FileExtents(f *os.File) ([]Extent, error) { + var out []Extent + req := &fiemapRequest{Length: ^uint64(0), Flags: fiemapFlagSync, Count: fiemapBatch} + for { + req.Mapped = 0 + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), fsIocFiemap, uintptr(unsafe.Pointer(req))); errno != 0 { //nolint:gosec + return nil, fmt.Errorf("fiemap %s: %w", f.Name(), errno) + } + if req.Mapped == 0 { + return out, nil + } + for _, e := range req.Extents[:req.Mapped] { + out = append(out, Extent{ + Logical: int64(e.Logical), //nolint:gosec + Physical: int64(e.Physical), //nolint:gosec + Length: int64(e.Length), //nolint:gosec + Unwritten: e.Flags&fiemapExtentUnwritten != 0, + }) + } + last := req.Extents[req.Mapped-1] + if last.Flags&fiemapExtentLast != 0 { + return out, nil + } + req.Start = last.Logical + last.Length + req.Length = ^uint64(0) - req.Start + } +} + +// PunchHole deallocates a range of f and keeps its size. +func PunchHole(f *os.File, offset, length int64) error { + if err := unix.Fallocate(int(f.Fd()), unix.FALLOC_FL_PUNCH_HOLE|unix.FALLOC_FL_KEEP_SIZE, offset, length); err != nil { + return fmt.Errorf("punch hole in %s at %d len %d: %w", f.Name(), offset, length, err) + } + return nil +} diff --git a/utils/extents_linux_test.go b/utils/extents_linux_test.go new file mode 100644 index 00000000..7c46c4da --- /dev/null +++ b/utils/extents_linux_test.go @@ -0,0 +1,57 @@ +//go:build linux + +package utils + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFileExtentsReportsDataAndSkipsHoles(t *testing.T) { + path := filepath.Join(t.TempDir(), "f") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer f.Close() + if err = f.Truncate(1 << 22); err != nil { + t.Fatalf("truncate: %v", err) + } + if _, err = f.WriteAt(make([]byte, 4096), 1<<21); err != nil { + t.Fatalf("write: %v", err) + } + extents, err := FileExtents(f) + if err != nil { + t.Skipf("filesystem has no FIEMAP: %v", err) + } + if len(extents) == 0 { + t.Fatal("no extents after a flushed write") + } + first, last := extents[0], extents[len(extents)-1] + if first.Logical > 1<<21 || last.Logical+last.Length < 1<<21+4096 { + t.Errorf("extents %+v do not cover the write", extents) + } +} + +func TestPunchHoleKeepsSize(t *testing.T) { + path := filepath.Join(t.TempDir(), "f") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer f.Close() + if _, err = f.WriteAt([]byte{1, 2, 3, 4}, 0); err != nil { + t.Fatalf("write: %v", err) + } + if err = PunchHole(f, 0, 2); err != nil { + t.Skipf("filesystem has no hole punch: %v", err) + } + fi, err := f.Stat() + if err != nil { + t.Fatalf("stat: %v", err) + } + if fi.Size() != 4 { + t.Errorf("size = %d, want 4", fi.Size()) + } +} diff --git a/utils/extents_other.go b/utils/extents_other.go new file mode 100644 index 00000000..dcb38b22 --- /dev/null +++ b/utils/extents_other.go @@ -0,0 +1,18 @@ +//go:build !linux + +package utils + +import ( + "errors" + "os" +) + +// FileExtents needs FIEMAP, so it fails off Linux. +func FileExtents(*os.File) ([]Extent, error) { + return nil, errors.ErrUnsupported +} + +// PunchHole needs fallocate, so it fails off Linux. +func PunchHole(*os.File, int64, int64) error { + return errors.ErrUnsupported +} diff --git a/utils/id.go b/utils/id.go index 11ed6ad5..a137c553 100644 --- a/utils/id.go +++ b/utils/id.go @@ -2,9 +2,17 @@ package utils import ( "crypto/rand" + "regexp" ) +var validID = regexp.MustCompile(`^[A-Z2-7]{26}$`) + // GenerateID returns a random 26-character crypto-secure base32 ID. func GenerateID() string { return rand.Text() } + +// ValidID reports whether id has the shape GenerateID produces. +func ValidID(id string) bool { + return validID.MatchString(id) +} diff --git a/utils/id_test.go b/utils/id_test.go index 4489827b..36c638c5 100644 --- a/utils/id_test.go +++ b/utils/id_test.go @@ -4,6 +4,17 @@ import ( "testing" ) +func TestValidID(t *testing.T) { + if !ValidID(GenerateID()) { + t.Error("a generated ID is invalid") + } + for _, id := range []string{"", "short", "../../../../../../../etc/pw", "abcdefghijklmnopqrstuvwxyz", "AAAAAAAAAAAAAAAAAAAAAAAAA1"} { + if ValidID(id) { + t.Errorf("%q accepted", id) + } + } +} + func TestGenerateID_Length(t *testing.T) { id := GenerateID() if len(id) != 26 { diff --git a/utils/reflink_linux.go b/utils/reflink_linux.go index d40f17eb..573bba40 100644 --- a/utils/reflink_linux.go +++ b/utils/reflink_linux.go @@ -36,6 +36,26 @@ func ReflinkCopy(ctx context.Context, dst, src string, sync SyncMode) error { return SparseCopy(dst, src, sync) } +// CheckReflink reports whether the filesystem holding dir supports FICLONE; cloning an empty file costs no extent walk. +func CheckReflink(dir string) error { + src, err := os.CreateTemp(dir, ".reflink-probe-*") + if err != nil { + return fmt.Errorf("create reflink probe: %w", err) + } + defer os.Remove(src.Name()) //nolint:errcheck + defer src.Close() //nolint:errcheck + dst, err := os.CreateTemp(dir, ".reflink-probe-*") + if err != nil { + return fmt.Errorf("create reflink probe: %w", err) + } + defer os.Remove(dst.Name()) //nolint:errcheck + defer dst.Close() //nolint:errcheck + if err := unix.IoctlFileClone(int(dst.Fd()), int(src.Fd())); err != nil { + return fmt.Errorf("ficlone in %s: %w", dir, err) + } + return nil +} + func tryFiclone(dst, src string, sync SyncMode) error { return CopyWithCleanup(dst, src, func(srcFile, dstFile *os.File) error { if err := unix.IoctlFileClone(int(dstFile.Fd()), int(srcFile.Fd())); err != nil { diff --git a/utils/reflink_other.go b/utils/reflink_other.go index 70bb2f8d..29dfdbb8 100644 --- a/utils/reflink_other.go +++ b/utils/reflink_other.go @@ -2,9 +2,17 @@ package utils -import "context" +import ( + "context" + "errors" +) // ReflinkCopy copies a single file. On non-Linux, falls back to SparseCopy. func ReflinkCopy(_ context.Context, dst, src string, sync SyncMode) error { return SparseCopy(dst, src, sync) } + +// CheckReflink needs FICLONE, so it fails off Linux. +func CheckReflink(string) error { + return errors.ErrUnsupported +} From eacc9527245ded16f85733e4f15f85c431362950 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 14:35:12 +0800 Subject: [PATCH 2/8] feat(vm): reclaim an interrupted receive and document live migration A receive killed after launch left a VMM on the run dir, and a retry with the same --id started a second one on the same socket. The retry now runs the stale-create reclaim first. --- README.md | 1 + cmd/vm/commands.go | 2 +- cmd/vm/migrate.go | 6 ++++ docs/cli.md | 19 +++++++++++++ docs/daemon.md | 2 +- docs/index.md | 2 +- docs/snapshots.md | 39 ++++++++++++++++++++++++++ hypervisor/firecracker/migrate_test.go | 13 +++++++++ 8 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 hypervisor/firecracker/migrate_test.go diff --git a/README.md b/README.md index a7e1d61b..c041eb21 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Lightweight MicroVM engine with dual hypervisor backends: [Cloud Hypervisor](htt - **OCI VM images** — pull container-registry images with kernel + rootfs layers, content-addressed EROFS blob cache; **cloud images** from HTTP(S) URLs with automatic qcow2 conversion; local file/stdin import - **Docker-like CLI** — `create`, `run`, `start`, `stop`, `list`, `inspect`, `console`, `exec`, `rm`, `clone`, `status` - **Snapshot & clone** — capture a running VM (memory, disks, config) and clone it into new VMs with fresh network identity; clones restore memory copy-on-write by default (page cache shared across sibling clones) and hold shared source leases so parallel clone bursts survive concurrent source removal; restore and atomic hibernate in place; portable export/import between hosts +- **Live migration** — move a running VM to another node (CH): round-based memory and disk deltas, the guest stops only for the last round; the caller owns transfer and round policy - **CNI networking** — multi-queue virtio-net TAPs wired via TC redirect (no bridge in the data path), multi-NIC, bridge mode, NIC hot-resize - **Data disks & runtime attach** — extra virtio-blk disks at create/clone time; hot-plug vhost-user-fs shares and VFIO PCI devices (CH); external raw disks and NIC resize on CH or Firecracker `--pci` VMs; a private copy of a snapshot's COW disk on CH - **Windows guests** — UEFI + Hyper-V enlightenments via the cocoonstack CH/firmware forks diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index 2019c3cb..80ea302a 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -382,7 +382,7 @@ func addCloneFlags(cmd *cobra.Command) { cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)") } -// addReceiveFlags holds the clone flags that need no device change after restore: one would resume a received guest. +// addReceiveFlags holds the clone flags that need no device op after restore: a device op resumes a paused guest. func addReceiveFlags(cmd *cobra.Command) { cmd.Flags().String("name", "", "VM name (default: cocoon-clone-)") cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)") diff --git a/cmd/vm/migrate.go b/cmd/vm/migrate.go index d680d586..2c50aef2 100644 --- a/cmd/vm/migrate.go +++ b/cmd/vm/migrate.go @@ -46,6 +46,12 @@ func (h Handler) MigrateReceive(cmd *cobra.Command, _ []string) error { if !ok { return fmt.Errorf("migrate receive: backend %s: %w", hyper.Type(), hypervisor.ErrMigrateUnsupported) } + // A receive killed mid-flight leaves its placeholder and maybe a VMM under this ID. + if id := cliutil.FlagStr(cmd, "id"); id != "" { + if _, err = hyper.ReconcileStaleCreate(ctx, id); err != nil { + return fmt.Errorf("reclaim interrupted receive: %w", err) + } + } cs, err := h.prepareClone(ctx, cmd, conf, hyper, cfg) if err != nil { return err diff --git a/docs/cli.md b/docs/cli.md index 451ee88b..bd829351 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -39,6 +39,10 @@ cocoon │ │ ├── attach [flags] VM Hot-attach a raw disk file (CH, or FC with --pci) or a snapshot's COW disk copy (CH) │ │ └── detach [flags] VM Detach a hot-attached disk by --name (keeps a --path file, removes a snapshot copy) │ ├── net [flags] VM Resize NIC count on a running VM (CH, or FC with --pci) +│ ├── migrate +│ │ ├── capture [flags] VM Start a migration series, or add a delta with --series (CH only) +│ │ ├── receive [flags] Restore a transferred series into a new VM, guest paused +│ │ └── abort --series ID VM End a series, resume the guest, delete the directory │ └── debug [flags] IMAGE Generate hypervisor launch command (dry run) ├── snapshot │ ├── save [flags] VM Create a snapshot from a running VM @@ -245,6 +249,21 @@ cocoon vm restore my-vm --from-dir /sync/from-host-b --force The dir is read-only across the call, so multiple clones of the same dir (golden image use case) are safe. Pass `--pull` if the base image may not be present locally — `EnsureImage` pulls the envelope's image by digest when it is missing, and `image_blob_ids` pin its blobs against GC for the clone's lifetime. +### Migrate Flags + +See [Live Migration](snapshots.md#live-migration). + +`cocoon vm migrate capture`: + +| Flag | Default | Description | +|------|---------|-------------| +| `--series` | empty | Series to add a delta to; empty starts a new series | +| `--final` | `false` | Leave the guest paused after this delta; needs `--series` | + +`cocoon vm migrate receive` takes `--from-dir` (required), `--id` (default: generated, must have the shape of a generated ID) and the [clone flags](#clone-flags) except `--nics` and `--data-disk`, which would resume the guest. With `--output json`, `capture` prints `{"series", "dir", "round", "paused", "files": [{"name", "size"}]}`; round 0 is the baseline. + +`cocoon vm migrate abort` requires `--series`. + ### Reconcile Stale Create `cocoon vm reconcile-stale-create VM` reclaims a `creating` placeholder whose owning create or clone died mid-flight, freeing the record and its name. A free VM ops lock is the proof of ownerlessness — create and clone hold it from prereserve through the final record commit — so the verb never races an in-flight operation and needs no age heuristics (contrast `vm rm --force`, which queues behind a live clone and then deletes the freshly created VM). With `--output json` it prints `{"id": "...", "outcome": "..."}`, omitting `id` for `not-found`: diff --git a/docs/daemon.md b/docs/daemon.md index cab0112b..6c36aaca 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -63,7 +63,7 @@ The `file` backend writes one JSON object per line to `kind` is one of `vm.compute.start`, `vm.compute.stop`, `vm.storage.start`, `vm.storage.stop`, `snap.storage.start`, `snap.storage.stop` — downstream pairs each `*.start` with its `*.stop` by id. `reason` is one of `boot`, `restart`, -`clone`, `restore`, `stop-user`, `stop-crash`, `vm-rm`, `snap-rm`. +`clone`, `restore`, `migrate`, `stop-user`, `stop-crash`, `vm-rm`, `snap-rm`. One operational note: the daemon is the first long-lived writer of the metering ledger, and the `file` backend holds that file open for its whole lifetime. If diff --git a/docs/index.md b/docs/index.md index b6456ec7..0c93389c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,7 @@ cocoon CLI ──► images: OCI (EROFS layers, direct boot) | cloudimg (qcow2, - [Networking](networking.md) — CNI with TC redirect, bridge mode, multi-NIC, NIC hot-resize - [Snapshots, clone & restore](snapshots.md) — capture, clone, restore, - hibernate, export/import, cross-node moves + hibernate, export/import, cross-node moves, live migration - [Runtime device attach](devices.md) — vhost-user-fs shares, VFIO PCI passthrough, external disk hot-attach - [Windows guests](windows.md) — the `--windows` path and the patched diff --git a/docs/snapshots.md b/docs/snapshots.md index 8863e697..670ce988 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -152,6 +152,45 @@ Pause, capture, persist, and VMM termination share one pause window: the snapsho A canceled or timed-out `vm hibernate` or `snapshot save` waits for the hypervisor to end its memory dump, up to 10 minutes, then resumes the guest, so SIGTERM can take that long to end the command. SIGKILL skips the resume; see [known issues](known-issues.md#a-killed-capture-leaves-the-guest-paused). +### Live Migration + +`vm migrate` captures a running VM in rounds and restores it on another node (Cloud Hypervisor only). The guest stops only for the last round, the copy of its files and the restore, and its processes and memory carry over. Cocoon captures and restores; moving the files, choosing how many rounds to run, and deciding which node owns the VM stay with the caller. + +```bash +# Source: a full baseline, then deltas. The guest runs again after each one. +cocoon vm migrate capture my-vm --output json # prints series, dir, files +cocoon vm migrate capture my-vm --series 123 --output json # optional, repeatable +# Source: the last delta leaves the guest paused. +cocoon vm migrate capture my-vm --series 123 --final --output json + +# Target: after copying every file each round listed into one directory +cocoon vm migrate receive --from-dir /xfer/123 --id --name my-vm --pull +cocoon vm start my-vm # resumes the guest + +# Source: only after the target runs +cocoon vm rm --force my-vm +``` + +To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. It can run again safely. + +Requirements: + +- **Hypervisor build**: the cocoonstack fork at `35b098c65260` or newer. An older VMM answers 404 on the first capture and nothing changes. A VM keeps the VMM it started on, so it needs a restart after an upgrade. +- **Filesystem**: the source run dir must support reflink (XFS, btrfs); capture refuses otherwise before it pauses the guest. On the target, keep the copied directory on the filesystem of the run dir, so memory files hard-link and the directory can go after the receive. +- **Devices**: a hot-attached disk, share or VFIO device refuses the capture, as it does for `snapshot save`. + +Rules for the caller: + +- Copy every file a round lists, in the listed round order. `state.json` and `config.json` are replaced each round. A `.ref` file never leaves the source. A short or missing disk delta fails the receive. +- A VM has one live series. A new baseline ends the previous one, whose directory stays until `abort` or a `gc` 24 hours later. +- A failed round ends the series, resumes the guest and deletes its directory; start again with a new baseline. This includes a call with an old series ID, which also ends the live series. `snapshot save`, a NIC resize, a device detach, a reboot or a stop ends the series too, and the next delta fails. +- A VM cloned or received seconds ago may still eject its old NICs. If that happens during a series, the next delta fails; start again. +- Without `--id` the receive generates an ID. With `--id`, a retry after a killed receive reclaims what it left. A retry after a finished receive fails with `id collision`; `vm inspect` shows the VM. Only `vm migrate receive` reads a series directory: `vm clone --from-dir` would pair the last memory with the baseline disks. + +The received VM gets fresh NICs like a clone, with new MACs, so the guest needs the [post-clone network setup](#post-clone-guest-setup). No reseed runs: the guest state is the same one, not a copy. `--restore-mode ondemand` is rejected by the hypervisor for a series with deltas. Metering opens the new intervals with reason `migrate`. + +A qcow2 overlay is open when captured, so its dirty bit is set and the target VMM rebuilds the refcounts when it opens the file. The overlay hash therefore differs after the receive; compare it before the VMM opens it. + ### Restore Constraints - **VM must be running, stopped, or error.** Restore cold-spawns a fresh hypervisor process from the snapshot; a stopped target (e.g. hibernated) gets the `vm start` network self-heal first. Error-state VMs are admitted because restore rebuilds the run dir — it is the recovery path for a crashed or interrupted restore (which `vm start` refuses). diff --git a/hypervisor/firecracker/migrate_test.go b/hypervisor/firecracker/migrate_test.go new file mode 100644 index 00000000..8b369417 --- /dev/null +++ b/hypervisor/firecracker/migrate_test.go @@ -0,0 +1,13 @@ +package firecracker + +import ( + "testing" + + "github.com/cocoonstack/cocoon/hypervisor" +) + +func TestFirecrackerIsNotAMigrator(t *testing.T) { + if _, ok := any(&Firecracker{}).(hypervisor.Migrator); ok { + t.Error("Firecracker claims live migration; vm migrate would stop returning ErrMigrateUnsupported") + } +} From c3d0a953a3a3d84685734bd031f6d08c679ac951 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 14:51:37 +0800 Subject: [PATCH 3/8] fix(vm): tie abort to the series directory and catch rounds out of step An abort of an old series resumed a guest paused for a newer one, which would split the VM across two nodes. A new baseline now removes older directories and abort acts only while its directory exists, keeping it until the guest resumes so a retry can finish. A capture killed between the memory and disk writes left rounds out of step that only surfaced at the receive. Each round now checks first. The envelope follows the record each round, since a NIC added mid-series does not end the series. --- cmd/vm/commands.go | 11 +- cmd/vm/migrate.go | 26 ++-- cmd/vm/run.go | 31 +++-- docs/cli.md | 2 +- docs/snapshots.md | 6 +- hypervisor/cloudhypervisor/cloudhypervisor.go | 1 + hypervisor/cloudhypervisor/direct.go | 36 +++-- hypervisor/cloudhypervisor/migrate.go | 16 +-- hypervisor/diskdelta.go | 2 +- hypervisor/migrate.go | 69 +++++++--- hypervisor/migrate_test.go | 129 ++++++++++++++++-- hypervisor/utils.go | 5 +- utils/reflink_linux.go | 15 +- 13 files changed, 247 insertions(+), 102 deletions(-) diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index 80ea302a..6a9a20f9 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -235,7 +235,8 @@ func buildMigrateCommand(h Handler) *cobra.Command { Args: cobra.NoArgs, RunE: h.MigrateReceive, } - addReceiveFlags(receive) + addCloneBaseFlags(receive) + receive.Flags().String("from-dir", "", "directory holding the transferred series (required)") receive.Flags().String("id", "", "VM ID (default: generated)") _ = receive.MarkFlagRequired("from-dir") cliutil.AddOutputFlag(receive) @@ -377,13 +378,14 @@ func addVMFlags(cmd *cobra.Command) { } func addCloneFlags(cmd *cobra.Command) { - addReceiveFlags(cmd) + addCloneBaseFlags(cmd) + cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT") cmd.Flags().Int("nics", 0, "override NIC count (omit to inherit from snapshot; on Firecracker only for --pci snapshots)") cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)") } -// addReceiveFlags holds the clone flags that need no device op after restore: a device op resumes a paused guest. -func addReceiveFlags(cmd *cobra.Command) { +// addCloneBaseFlags holds the clone flags that need no device op after restore: a device op resumes a received guest. +func addCloneBaseFlags(cmd *cobra.Command) { cmd.Flags().String("name", "", "VM name (default: cocoon-clone-)") cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)") cmd.Flags().Int("disk-queue-size", 0, "virtio-blk ring depth per device (0 = inherit from snapshot)") @@ -396,7 +398,6 @@ func addReceiveFlags(cmd *cobra.Command) { cmd.Flags().Bool("no-direct-io", false, "disable O_DIRECT on writable disks (inherit from snapshot if not set)") addRestoreModeFlag(cmd) cmd.Flags().Bool("pull", false, "auto-pull base image if not found locally (for cross-node clone)") - cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT") } func addCPUBandwidthFlags(cmd *cobra.Command) { diff --git a/cmd/vm/migrate.go b/cmd/vm/migrate.go index 2c50aef2..854208d4 100644 --- a/cmd/vm/migrate.go +++ b/cmd/vm/migrate.go @@ -7,38 +7,38 @@ import ( "github.com/spf13/cobra" "github.com/cocoonstack/cocoon/cmd/cliutil" - cmdcore "github.com/cocoonstack/cocoon/cmd/core" "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/utils" ) func (h Handler) MigrateCapture(cmd *cobra.Command, args []string) error { - ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate capture", hypervisor.ErrMigrateUnsupported) - if err != nil { - return err - } req := hypervisor.MigrateRequest{Series: cliutil.FlagStr(cmd, "series"), Final: cliutil.FlagBool(cmd, "final")} if req.Final && req.Series == "" { return fmt.Errorf("--final needs --series: the baseline is never the last round") } + ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate capture", hypervisor.ErrMigrateUnsupported) + if err != nil { + return err + } + logger := log.WithFunc("cmd.vm.migrate.capture") round, err := m.MigrateCapture(ctx, args[0], req) if err != nil { return err } return outputOrLog(cmd, round, func() { - log.WithFunc("cmd.vm.migrate.capture").Infof(ctx, "series %s round %d in %s (paused: %t, %d files)", round.Series, round.Round, round.Dir, round.Paused, len(round.Files)) + logger.Infof(ctx, "series %s round %d in %s (paused: %t, %d files)", round.Series, round.Round, round.Dir, round.Paused, len(round.Files)) }) } func (h Handler) MigrateReceive(cmd *cobra.Command, _ []string) error { + id := cliutil.FlagStr(cmd, "id") + if id != "" && !utils.ValidID(id) { + return fmt.Errorf("--id must have the shape of a generated VM ID, got %q", id) + } ctx, conf := h.Init(cmd) logger := log.WithFunc("cmd.vm.migrate.receive") dir := cliutil.FlagStr(cmd, "from-dir") - cfg, err := readVerifiedEnvelope(dir) - if err != nil { - return err - } - conf.UseFirecracker = isFirecracker(cfg.Hypervisor) - hyper, err := cmdcore.InitHypervisor(ctx, conf) + cfg, hyper, err := openSnapshotDir(ctx, conf, dir) if err != nil { return err } @@ -47,7 +47,7 @@ func (h Handler) MigrateReceive(cmd *cobra.Command, _ []string) error { return fmt.Errorf("migrate receive: backend %s: %w", hyper.Type(), hypervisor.ErrMigrateUnsupported) } // A receive killed mid-flight leaves its placeholder and maybe a VMM under this ID. - if id := cliutil.FlagStr(cmd, "id"); id != "" { + if id != "" { if _, err = hyper.ReconcileStaleCreate(ctx, id); err != nil { return fmt.Errorf("reclaim interrupted receive: %w", err) } diff --git a/cmd/vm/run.go b/cmd/vm/run.go index 142a82af..263009d3 100644 --- a/cmd/vm/run.go +++ b/cmd/vm/run.go @@ -231,14 +231,7 @@ func (h Handler) cloneDirect(ctx context.Context, cmd *cobra.Command, conf *conf // cloneFromDir keeps the dir read-only, so concurrent clones of a golden image are safe. func (h Handler) cloneFromDir(ctx context.Context, cmd *cobra.Command, conf *config.Config, dir string, logger *log.Fields) error { - cfg, err := readVerifiedEnvelope(dir) - if err != nil { - return err - } - if cfg.Hypervisor != "" { - conf.UseFirecracker = isFirecracker(cfg.Hypervisor) - } - hyper, err := cmdcore.InitHypervisor(ctx, conf) + cfg, hyper, err := openSnapshotDir(ctx, conf, dir) if err != nil { return err } @@ -286,12 +279,7 @@ func (h Handler) prepareClone(ctx context.Context, cmd *cobra.Command, conf *con if err != nil { return cloneSetup{}, err } - vmID := cliutil.FlagStr(cmd, "id") - if vmID == "" { - vmID = utils.GenerateID() - } else if !utils.ValidID(vmID) { - return cloneSetup{}, fmt.Errorf("--id must have the shape of a generated VM ID, got %q", vmID) - } + vmID := cmp.Or(cliutil.FlagStr(cmd, "id"), utils.GenerateID()) vmCfg.Name = cmp.Or(vmCfg.Name, "cocoon-clone-"+network.VMIDPrefix(vmID)) if err = vmCfg.Validate(); err != nil { return cloneSetup{}, err @@ -525,6 +513,21 @@ func snapshotSource(cmd *cobra.Command, args []string, baseArgs int) (fromDir, s return "", args[baseArgs], nil } +func openSnapshotDir(ctx context.Context, conf *config.Config, dir string) (types.SnapshotConfig, hypervisor.Hypervisor, error) { + cfg, err := readVerifiedEnvelope(dir) + if err != nil { + return types.SnapshotConfig{}, nil, err + } + if cfg.Hypervisor != "" { + conf.UseFirecracker = isFirecracker(cfg.Hypervisor) + } + hyper, err := cmdcore.InitHypervisor(ctx, conf) + if err != nil { + return types.SnapshotConfig{}, nil, err + } + return cfg, hyper, nil +} + func readVerifiedEnvelope(dir string) (types.SnapshotConfig, error) { envelope, err := snapshot.ReadSnapshotEnvelope(dir) if err != nil { diff --git a/docs/cli.md b/docs/cli.md index bd829351..cb04a783 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -260,7 +260,7 @@ See [Live Migration](snapshots.md#live-migration). | `--series` | empty | Series to add a delta to; empty starts a new series | | `--final` | `false` | Leave the guest paused after this delta; needs `--series` | -`cocoon vm migrate receive` takes `--from-dir` (required), `--id` (default: generated, must have the shape of a generated ID) and the [clone flags](#clone-flags) except `--nics` and `--data-disk`, which would resume the guest. With `--output json`, `capture` prints `{"series", "dir", "round", "paused", "files": [{"name", "size"}]}`; round 0 is the baseline. +`cocoon vm migrate receive` takes `--from-dir` (required), `--id` (default: generated, must have the shape of a generated ID) and the [clone flags](#clone-flags) except `--nics` and `--data-disk`: a device op after restore would resume the guest. With `--output json`, `capture` prints `{"series", "dir", "round", "paused", "files": [{"name", "size"}]}`; round 0 is the baseline. `cocoon vm migrate abort` requires `--series`. diff --git a/docs/snapshots.md b/docs/snapshots.md index 670ce988..f40e947d 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -171,7 +171,7 @@ cocoon vm start my-vm # resumes t cocoon vm rm --force my-vm ``` -To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. It can run again safely. +To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. Once the directory is gone, a repeat or an old series ID does nothing, so it cannot resume a guest paused for a newer series. Requirements: @@ -182,8 +182,8 @@ Requirements: Rules for the caller: - Copy every file a round lists, in the listed round order. `state.json` and `config.json` are replaced each round. A `.ref` file never leaves the source. A short or missing disk delta fails the receive. -- A VM has one live series. A new baseline ends the previous one, whose directory stays until `abort` or a `gc` 24 hours later. -- A failed round ends the series, resumes the guest and deletes its directory; start again with a new baseline. This includes a call with an old series ID, which also ends the live series. `snapshot save`, a NIC resize, a device detach, a reboot or a stop ends the series too, and the next delta fails. +- A VM has one series. A new baseline ends the previous one and deletes its directory, along with any `snapshot-*` directory a killed capture left; the directory of a series nobody continues stays until `gc` removes it after 24 hours. A delta with an old series ID fails and leaves the guest alone. +- A failed round ends the series, resumes the guest and deletes its directory; start again with a new baseline. A round after a killed capture fails the same way. `snapshot save`, removing a NIC, a device detach, a reboot or a stop ends the series too, and the next delta fails. - A VM cloned or received seconds ago may still eject its old NICs. If that happens during a series, the next delta fails; start again. - Without `--id` the receive generates an ID. With `--id`, a retry after a killed receive reclaims what it left. A retry after a finished receive fails with `id collision`; `vm inspect` shows the VM. Only `vm migrate receive` reads a series directory: `vm clone --from-dir` would pair the last memory with the baseline disks. diff --git a/hypervisor/cloudhypervisor/cloudhypervisor.go b/hypervisor/cloudhypervisor/cloudhypervisor.go index a03e07fc..eb95610b 100644 --- a/hypervisor/cloudhypervisor/cloudhypervisor.go +++ b/hypervisor/cloudhypervisor/cloudhypervisor.go @@ -22,6 +22,7 @@ var ( _ disk.SnapshotAttacher = (*CloudHypervisor)(nil) _ fs.Attacher = (*CloudHypervisor)(nil) _ fs.Lister = (*CloudHypervisor)(nil) + _ hypervisor.Migrator = (*CloudHypervisor)(nil) _ vfio.Attacher = (*CloudHypervisor)(nil) _ vfio.Lister = (*CloudHypervisor)(nil) ) diff --git a/hypervisor/cloudhypervisor/direct.go b/hypervisor/cloudhypervisor/direct.go index 3e024e41..b8567463 100644 --- a/hypervisor/cloudhypervisor/direct.go +++ b/hypervisor/cloudhypervisor/direct.go @@ -12,19 +12,7 @@ import ( ) func (ch *CloudHypervisor) DirectClone(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) { - // config.json is copied verbatim, so re-parsing the runDir copy would decode identical bytes. - var srcCfg *chVMConfig - spec := hypervisor.CloneSpec{ - VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, - AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { - return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg, false) - }, - } - return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { - var err error - srcCfg, err = cloneSnapshotFiles(ctx, dstDir, srcDir) - return err - }) + return ch.directClone(ctx, vmID, vmCfg, net, snapshotConfig, srcDir, false) } func (ch *CloudHypervisor) DirectRestore(ctx context.Context, vmRef string, vmCfg *types.VMConfig, srcDir, sourceSnapshotID string) (*types.VM, error) { @@ -51,6 +39,26 @@ func (ch *CloudHypervisor) DirectRestore(ctx context.Context, vmRef string, vmCf }) } +func (ch *CloudHypervisor) directClone(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string, migrate bool) (*types.VM, error) { + // config.json is copied verbatim, so re-parsing the runDir copy would decode identical bytes. + var srcCfg *chVMConfig + spec := hypervisor.CloneSpec{ + VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, + AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { + return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg, migrate) + }, + } + return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { + var err error + if migrate { + srcCfg, err = cloneSeriesFiles(ctx, dstDir, srcDir) + } else { + srcCfg, err = cloneSnapshotFiles(ctx, dstDir, srcDir) + } + return err + }) +} + func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig, error) { chCfg, err := parseCHConfig(filepath.Join(srcDir, configJSONName)) if err != nil { @@ -59,7 +67,7 @@ func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig return chCfg, hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, classifySnapshotFile(chCfg)) } -func classifySnapshotFile(chCfg *chVMConfig) func(name string) hypervisor.SnapshotFileKind { +func classifySnapshotFile(chCfg *chVMConfig) hypervisor.SnapshotClassifier { cowFiles := identifyCOWFiles(chCfg) return func(name string) hypervisor.SnapshotFileKind { switch { diff --git a/hypervisor/cloudhypervisor/migrate.go b/hypervisor/cloudhypervisor/migrate.go index 644c60f1..5a66630a 100644 --- a/hypervisor/cloudhypervisor/migrate.go +++ b/hypervisor/cloudhypervisor/migrate.go @@ -19,8 +19,6 @@ const ( memoryDiffPrefix = "memory-ranges.diff." ) -var _ hypervisor.Migrator = (*CloudHypervisor)(nil) - func (ch *CloudHypervisor) MigrateCapture(ctx context.Context, ref string, req hypervisor.MigrateRequest) (*hypervisor.MigrateRound, error) { return ch.MigrateCaptureSequence(ctx, ref, req, ch.migrateSpec(ctx)) } @@ -30,18 +28,7 @@ func (ch *CloudHypervisor) MigrateAbort(ctx context.Context, ref, series string) } func (ch *CloudHypervisor) MigrateReceive(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) { - var srcCfg *chVMConfig - spec := hypervisor.CloneSpec{ - VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, - AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { - return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg, true) - }, - } - return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { - var err error - srcCfg, err = cloneSeriesFiles(ctx, dstDir, srcDir) - return err - }) + return ch.directClone(ctx, vmID, vmCfg, net, snapshotConfig, srcDir, true) } func (ch *CloudHypervisor) migrateSpec(ctx context.Context) hypervisor.MigrateSpec { @@ -57,6 +44,7 @@ func (ch *CloudHypervisor) migrateSpec(ctx context.Context) hypervisor.MigrateSp EndSeries: func(_ *hypervisor.VMRecord, hc *http.Client) error { return endSnapshotSeries(context.WithoutCancel(ctx), hc) }, + Rounds: countMemoryDiffs, } } diff --git a/hypervisor/diskdelta.go b/hypervisor/diskdelta.go index f1551897..b7f5c14f 100644 --- a/hypervisor/diskdelta.go +++ b/hypervisor/diskdelta.go @@ -94,7 +94,7 @@ func diffExtents(ref, cur []utils.Extent, size int64) []deltaRange { curAt, curEnd := extentAt(&cur, pos) end := min(refEnd, curEnd, size) if refAt != curAt { - zero := curAt == (mapping{}) || curAt.unwritten + zero := !curAt.mapped || curAt.unwritten if n := len(out) - 1; n >= 0 && out[n].Zero == zero && out[n].Offset+out[n].Length == pos { out[n].Length += end - pos } else { diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go index 498a18d8..8f280ab7 100644 --- a/hypervisor/migrate.go +++ b/hypervisor/migrate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "maps" + "net/http" "os" "path/filepath" "slices" @@ -48,17 +49,19 @@ type MigrateFile struct { Size int64 `json:"size"` } -// MigrateSpec carries backend hooks for the migrate sequences; SnapshotSpec.Capture writes the guest state of one round. +// MigrateSpec carries backend hooks for the migrate sequences; SnapshotSpec.Capture writes the guest state of one round and Rounds counts those in a series directory. type MigrateSpec struct { SnapshotSpec EndSeries TransitionHook + Rounds func(dir string) (int, error) } -// roundSteps is what a baseline and a delta do differently around the shared pause window; round 0 is the baseline. +// roundSteps is what a baseline and a delta do differently around the shared pause window. type roundSteps struct { series string round int final bool + before func(hc *http.Client) error inPause func() error afterPause func() error } @@ -87,7 +90,7 @@ func (b *Backend) MigrateCaptureSequence(ctx context.Context, ref string, req Mi return round, nil } -// MigrateAbortSequence is idempotent; a VM that no longer runs only loses the directory. +// MigrateAbortSequence acts only while the series directory exists, so a repeat or an old series leaves the guest alone; the directory goes last so a failed resume can retry. func (b *Backend) MigrateAbortSequence(ctx context.Context, ref, series string, spec MigrateSpec) error { vmID, rec, unlock, err := b.lockForCapture(ctx, ref) if err != nil { @@ -98,17 +101,17 @@ func (b *Backend) MigrateAbortSequence(ctx context.Context, ref, series string, if err != nil { return err } + if !utils.FileExists(dir) { + return nil + } hc := utils.NewSocketHTTPClient(SocketPath(rec.RunDir)) runErr := b.WithRunningVM(ctx, &rec, func(_ int) error { return errors.Join(spec.EndSeries(&rec, hc), resumeAfterCapture(spec.Resume, &rec, hc)) }) - if errors.Is(runErr, ErrNotRunning) { - runErr = nil - } - if runErr = errors.Join(runErr, os.RemoveAll(dir)); runErr != nil { + if runErr != nil && !errors.Is(runErr, ErrNotRunning) { return fmt.Errorf("migrate abort VM %s: %w", vmID, runErr) } - return nil + return os.RemoveAll(dir) } func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec MigrateSpec) (*MigrateRound, error) { @@ -116,6 +119,10 @@ func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec Migra if err := utils.CheckReflink(rec.RunDir); err != nil { return nil, fmt.Errorf("live migration needs a reflink filesystem under the run dir: %w", err) } + // The ops lock proves no capture owns these, and one directory per VM keeps an old series ID from reaching the guest. + if errs := utils.RemoveMatching(ctx, rec.RunDir, isCaptureDir); len(errs) > 0 { + return nil, errors.Join(errs...) + } dir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) if err != nil { return nil, fmt.Errorf("create series dir: %w", err) @@ -123,6 +130,7 @@ func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec Migra series := strings.TrimPrefix(filepath.Base(dir), captureDirPrefix) return b.captureRound(ctx, rec, spec, dir, roundSteps{ series: series, + before: func(hc *http.Client) error { return spec.EndSeries(rec, hc) }, inPause: func() error { return CopyWritableDisks(ctx, dir, rec) }, @@ -130,7 +138,7 @@ func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec Migra if err := saveCaptureMeta(rec, spec.SnapshotSpec, dir); err != nil { return err } - return snapshot.WriteSnapshotEnvelope(dir, *b.BuildSnapshotConfig(series, rec), nil, nil) + return b.writeSeriesEnvelope(rec, series, dir) }, }) } @@ -152,6 +160,7 @@ func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRe series: req.Series, round: number, final: req.Final, + before: func(*http.Client) error { return checkRoundsInStep(spec, dir, pairs, number) }, inPause: func() error { if req.Final { return nil @@ -159,10 +168,12 @@ func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRe return reflinkNextRefs(ctx, pairs) }, afterPause: func() error { - _, mapErr := utils.Map(ctx, pairs, func(_ context.Context, _ int, p [2]string) (struct{}, error) { + if _, mapErr := utils.Map(ctx, pairs, func(_ context.Context, _ int, p [2]string) (struct{}, error) { return struct{}{}, writeRoundDelta(p, number, req.Final) - }) - return mapErr + }); mapErr != nil { + return mapErr + } + return b.writeSeriesEnvelope(rec, req.Series, dir) }, } return b.captureRound(ctx, rec, spec, dir, steps) @@ -175,10 +186,8 @@ func (b *Backend) captureRound(ctx context.Context, rec *VMRecord, spec MigrateS } hc := utils.NewSocketHTTPClient(SocketPath(rec.RunDir)) run := func() error { - if steps.round == 0 { - if endErr := spec.EndSeries(rec, hc); endErr != nil { - return fmt.Errorf("end previous series: %w", endErr) - } + if beforeErr := steps.before(hc); beforeErr != nil { + return beforeErr } if pauseErr := spec.Pause(rec, hc); pauseErr != nil { return fmt.Errorf("pause: %w", pauseErr) @@ -211,18 +220,44 @@ func (b *Backend) captureRound(ctx context.Context, rec *VMRecord, spec MigrateS return &MigrateRound{Series: steps.series, Dir: dir, Round: steps.round, Paused: steps.final, Files: files}, nil } +// writeSeriesEnvelope reruns each round because a NIC added mid-series does not end the series. +func (b *Backend) writeSeriesEnvelope(rec *VMRecord, series, dir string) error { + return snapshot.WriteSnapshotEnvelope(dir, *b.BuildSnapshotConfig(series, rec), nil, nil) +} + // IsMigrateDeltaFile reports a disk delta or reference a clone of a series must leave behind. func IsMigrateDeltaFile(name string) bool { return strings.HasSuffix(name, diskRefSuffix) || strings.Contains(name, diskDeltaInfix) } func seriesDir(rec *VMRecord, series string) (string, error) { - if series == "" || !filepath.IsLocal(series) || strings.ContainsRune(series, filepath.Separator) { + if !filepath.IsLocal(series) || strings.ContainsRune(series, filepath.Separator) { return "", fmt.Errorf("invalid series id %q", series) } return filepath.Join(rec.RunDir, captureDirPrefix+series), nil } +func isCaptureDir(e os.DirEntry) bool { + return e.IsDir() && strings.HasPrefix(e.Name(), captureDirPrefix) +} + +// checkRoundsInStep catches a round a killed capture left half written, which would otherwise surface only at the receive. +func checkRoundsInStep(spec MigrateSpec, dir string, pairs [][2]string, round int) error { + rounds, err := spec.Rounds(dir) + if err != nil { + return err + } + if rounds != round-1 { + return fmt.Errorf("series holds %d guest rounds and %d disk rounds: an earlier capture died, start a new series", rounds, round-1) + } + for _, p := range pairs { + if n := nextRound(p[0]); n != round { + return fmt.Errorf("%s is at round %d, want %d: an earlier capture died, start a new series", filepath.Base(p[0]), n-1, round-1) + } + } + return nil +} + func nextRound(disk string) int { n := 1 for utils.FileExists(filepath.Join(filepath.Dir(disk), diskDeltaName(filepath.Base(disk), n))) { diff --git a/hypervisor/migrate_test.go b/hypervisor/migrate_test.go index 95fa5060..741ba4fe 100644 --- a/hypervisor/migrate_test.go +++ b/hypervisor/migrate_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" @@ -43,7 +44,7 @@ func TestMigrateMiddleRoundResumesAndHidesTheReference(t *testing.T) { if round.Round != 1 || round.Paused || calls.resumed != 1 { t.Errorf("round %d paused %t resumed %d, want 1 false 1", round.Round, round.Paused, calls.resumed) } - if want := []string{"cow.raw.delta.1", "mem.1"}; !slices.Equal(fileNames(round), want) { + if want := []string{"cow.raw.delta.1", "mem.1", snapshot.SnapshotJSONName}; !slices.Equal(fileNames(round), want) { t.Errorf("files = %v, want %v", fileNames(round), want) } if !utils.FileExists(filepath.Join(dir, "cow.raw.ref")) { @@ -103,7 +104,7 @@ func TestMigrateUnknownSeriesKeepsTheGuestRunning(t *testing.T) { } } -func TestMigrateAbortIsIdempotent(t *testing.T) { +func TestMigrateAbortRepeatLeavesTheGuestAlone(t *testing.T) { b, id, rec := newMigrateTestVM(t) dir := seedSeries(t, rec) calls := &migrateCalls{} @@ -112,25 +113,136 @@ func TestMigrateAbortIsIdempotent(t *testing.T) { t.Fatalf("abort: %v", err) } } - if calls.ended != 2 || calls.resumed != 2 { - t.Errorf("ended %d resumed %d, want 2 2", calls.ended, calls.resumed) + if calls.ended != 1 || calls.resumed != 1 { + t.Errorf("ended %d resumed %d, want 1 1", calls.ended, calls.resumed) } if utils.FileExists(dir) { t.Error("the series survived the abort") } } -func TestMigrateAbortResumesWhenEndingTheSeriesFails(t *testing.T) { +func TestMigrateAbortOfAnOldSeriesLeavesTheGuestAlone(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + if err := b.MigrateAbortSequence(t.Context(), id, "old", migrateStubSpec(calls)); err != nil { + t.Fatalf("abort: %v", err) + } + if calls.ended != 0 || calls.resumed != 0 || !utils.FileExists(dir) { + t.Errorf("ended %d resumed %d kept %t, want 0 0 true", calls.ended, calls.resumed, utils.FileExists(dir)) + } +} + +func TestMigrateAbortKeepsTheSeriesUntilTheGuestResumes(t *testing.T) { b, id, rec := newMigrateTestVM(t) dir := seedSeries(t, rec) calls := &migrateCalls{} spec := migrateStubSpec(calls) - spec.EndSeries = func(*VMRecord, *http.Client) error { return errors.New("stop dirty log") } + spec.Resume = func(*VMRecord, *http.Client) error { return errors.New("vmm busy") } if err := b.MigrateAbortSequence(t.Context(), id, "s1", spec); err == nil { t.Fatal("abort hid the failure") } + if !utils.FileExists(dir) { + t.Fatal("the series went with the guest still paused, so a retry could not resume it") + } + if err := b.MigrateAbortSequence(t.Context(), id, "s1", migrateStubSpec(calls)); err != nil { + t.Fatalf("retry: %v", err) + } if calls.resumed != 1 || utils.FileExists(dir) { - t.Errorf("resumed %d series kept %t, want 1 false", calls.resumed, utils.FileExists(dir)) + t.Errorf("resumed %d kept %t, want 1 false", calls.resumed, utils.FileExists(dir)) + } +} + +func TestMigrateAbortResumesWhenEndingTheSeriesFails(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + spec.EndSeries = func(*VMRecord, *http.Client) error { return errors.New("stop dirty log") } + if err := b.MigrateAbortSequence(t.Context(), id, "s1", spec); err == nil { + t.Fatal("abort hid the failure") + } + if calls.resumed != 1 { + t.Errorf("resumed %d, want 1", calls.resumed) + } +} + +func TestMigrateRoundOutOfStepDiscardsTheSeries(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, dir string, spec *MigrateSpec) + }{ + { + name: "guest round without a disk round", + setup: func(_ *testing.T, _ string, spec *MigrateSpec) { + spec.Rounds = func(string) (int, error) { return 1, nil } + }, + }, + { + name: "disk round without a guest round", + setup: func(t *testing.T, dir string, _ *MigrateSpec) { + writeSparse(t, filepath.Join(dir, diskDeltaName(types.COWRawFileName, 1)), 8, nil) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + tt.setup(t, dir, &spec) + _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, spec) + if err == nil || !strings.Contains(err.Error(), "start a new series") { + t.Fatalf("capture: %v, want out of step", err) + } + if calls.paused != 0 || calls.ended != 1 || utils.FileExists(dir) { + t.Errorf("paused %d ended %d kept %t, want 0 1 false", calls.paused, calls.ended, utils.FileExists(dir)) + } + }) + } +} + +func TestMigrateRoundRewritesTheEnvelope(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error { + idx.VMs[id].NetworkConfigs = []*types.NetworkConfig{{MTU: 1500}, {MTU: 1500}} + return nil + }); err != nil { + t.Fatalf("seed nics: %v", err) + } + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, migrateStubSpec(&migrateCalls{})); err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("capture: %v", err) + } + envelope, err := snapshot.ReadSnapshotEnvelope(dir) + if err != nil { + t.Fatalf("read envelope: %v", err) + } + if envelope.Config.NICs != 2 || envelope.Config.ID != "s1" { + t.Errorf("envelope nics %d id %q, want 2 s1", envelope.Config.NICs, envelope.Config.ID) + } +} + +func TestIsCaptureDirMatchesSeriesDirsOnly(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, captureDirPrefix+"s1"), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.Mkdir(filepath.Join(dir, restoreStagingName), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + writeSparse(t, filepath.Join(dir, captureDirPrefix+"file"), 1, nil) + if errs := utils.RemoveMatching(t.Context(), dir, isCaptureDir); len(errs) != 0 { + t.Fatalf("remove: %v", errs) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(entries) != 2 || utils.FileExists(filepath.Join(dir, captureDirPrefix+"s1")) { + t.Errorf("left %v", entries) } } @@ -193,10 +305,11 @@ func migrateStubSpec(calls *migrateCalls) MigrateSpec { Resume: func(*VMRecord, *http.Client) error { calls.resumed++; return nil }, Capture: func(_ *VMRecord, _ *http.Client, dir string) error { calls.rounds++ - return os.WriteFile(filepath.Join(dir, "mem."+string(rune('0'+calls.rounds))), []byte("x"), 0o600) + return os.WriteFile(filepath.Join(dir, "mem."+strconv.Itoa(calls.rounds)), []byte("x"), 0o600) }, BuildMeta: func(*VMRecord, string) (*snapshot.SnapshotMeta, error) { return &snapshot.SnapshotMeta{}, nil }, EndSeries: func(*VMRecord, *http.Client) error { calls.ended++; return nil }, + Rounds: func(string) (int, error) { return calls.rounds, nil }, } } diff --git a/hypervisor/utils.go b/hypervisor/utils.go index fb49c84b..e1d3be52 100644 --- a/hypervisor/utils.go +++ b/hypervisor/utils.go @@ -56,6 +56,9 @@ var hostCPUCount = sync.OnceValue(readHostCPUCount) // SnapshotFileKind classifies a snapshot file for CloneSnapshotFiles. type SnapshotFileKind int +// SnapshotClassifier picks the kind of the snapshot file called name. +type SnapshotClassifier func(name string) SnapshotFileKind + // LockVMOps serializes mutating verbs on one VM across processes; the flock dies with the holder so a crash never wedges the VM. func (b *Backend) LockVMOps(ctx context.Context, vmID string) (func(), error) { l, err := vmlock.New(b.Conf.RootDirPath(), vmID) @@ -366,7 +369,7 @@ func VerifyBaseFiles(storageConfigs []*types.StorageConfig, boot *types.BootConf } // CloneSnapshotFiles copies snapshot files by kind: COW-class copies fan out concurrently, links and small meta stay serial. -func CloneSnapshotFiles(ctx context.Context, dstDir, srcDir string, classify func(name string) SnapshotFileKind) error { +func CloneSnapshotFiles(ctx context.Context, dstDir, srcDir string, classify SnapshotClassifier) error { entries, err := os.ReadDir(srcDir) if err != nil { return fmt.Errorf("read srcDir: %w", err) diff --git a/utils/reflink_linux.go b/utils/reflink_linux.go index 573bba40..9f92da61 100644 --- a/utils/reflink_linux.go +++ b/utils/reflink_linux.go @@ -42,18 +42,11 @@ func CheckReflink(dir string) error { if err != nil { return fmt.Errorf("create reflink probe: %w", err) } + _ = src.Close() defer os.Remove(src.Name()) //nolint:errcheck - defer src.Close() //nolint:errcheck - dst, err := os.CreateTemp(dir, ".reflink-probe-*") - if err != nil { - return fmt.Errorf("create reflink probe: %w", err) - } - defer os.Remove(dst.Name()) //nolint:errcheck - defer dst.Close() //nolint:errcheck - if err := unix.IoctlFileClone(int(dst.Fd()), int(src.Fd())); err != nil { - return fmt.Errorf("ficlone in %s: %w", dir, err) - } - return nil + dst := src.Name() + ".clone" + defer os.Remove(dst) //nolint:errcheck + return tryFiclone(dst, src.Name(), NoSync) } func tryFiclone(dst, src string, sync SyncMode) error { From 124f4d643d69602dccc8b4e9d68cf237bd9338cb Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 14:59:21 +0800 Subject: [PATCH 4/8] fix(vm): keep a new baseline away from snapshots being persisted SnapshotSequence releases the ops lock before PersistSnapshotDir reads its directory, so sweeping every snapshot-* dir could delete a snapshot save in progress. Series directories get their own prefix and the baseline removes only those; the shared prefix keeps them under the GC sweep. --- docs/snapshots.md | 2 +- hypervisor/migrate.go | 21 +++++++++++++-------- hypervisor/migrate_test.go | 30 +++++++++++++----------------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/docs/snapshots.md b/docs/snapshots.md index f40e947d..45711195 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -182,7 +182,7 @@ Requirements: Rules for the caller: - Copy every file a round lists, in the listed round order. `state.json` and `config.json` are replaced each round. A `.ref` file never leaves the source. A short or missing disk delta fails the receive. -- A VM has one series. A new baseline ends the previous one and deletes its directory, along with any `snapshot-*` directory a killed capture left; the directory of a series nobody continues stays until `gc` removes it after 24 hours. A delta with an old series ID fails and leaves the guest alone. +- A VM has one series. A new baseline ends the previous one and deletes its directory; the directory of a series nobody continues stays until `gc` removes it after 24 hours. A delta with an old series ID fails and leaves the guest alone. - A failed round ends the series, resumes the guest and deletes its directory; start again with a new baseline. A round after a killed capture fails the same way. `snapshot save`, removing a NIC, a device detach, a reboot or a stop ends the series too, and the next delta fails. - A VM cloned or received seconds ago may still eject its old NICs. If that happens during a series, the next delta fails; start again. - Without `--id` the receive generates an ID. With `--id`, a retry after a killed receive reclaims what it left. A retry after a finished receive fails with `id collision`; `vm inspect` shows the VM. Only `vm migrate receive` reads a series directory: `vm clone --from-dir` would pair the last memory with the baseline disks. diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go index 8f280ab7..cc54fdc2 100644 --- a/hypervisor/migrate.go +++ b/hypervisor/migrate.go @@ -16,7 +16,12 @@ import ( "github.com/cocoonstack/cocoon/utils" ) -const diskRefSuffix = ".ref" +const ( + diskRefSuffix = ".ref" + + // seriesDirPrefix extends captureDirPrefix so the GC sweep still ages out a series. + seriesDirPrefix = captureDirPrefix + "migrate-" +) // Migrator captures a running VM in rounds and restores the result on another node. type Migrator interface { @@ -119,15 +124,15 @@ func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec Migra if err := utils.CheckReflink(rec.RunDir); err != nil { return nil, fmt.Errorf("live migration needs a reflink filesystem under the run dir: %w", err) } - // The ops lock proves no capture owns these, and one directory per VM keeps an old series ID from reaching the guest. - if errs := utils.RemoveMatching(ctx, rec.RunDir, isCaptureDir); len(errs) > 0 { + // One directory per VM keeps an old series ID from reaching the guest. + if errs := utils.RemoveMatching(ctx, rec.RunDir, isSeriesDir); len(errs) > 0 { return nil, errors.Join(errs...) } - dir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) + dir, err := os.MkdirTemp(rec.RunDir, seriesDirPrefix) if err != nil { return nil, fmt.Errorf("create series dir: %w", err) } - series := strings.TrimPrefix(filepath.Base(dir), captureDirPrefix) + series := strings.TrimPrefix(filepath.Base(dir), seriesDirPrefix) return b.captureRound(ctx, rec, spec, dir, roundSteps{ series: series, before: func(hc *http.Client) error { return spec.EndSeries(rec, hc) }, @@ -234,11 +239,11 @@ func seriesDir(rec *VMRecord, series string) (string, error) { if !filepath.IsLocal(series) || strings.ContainsRune(series, filepath.Separator) { return "", fmt.Errorf("invalid series id %q", series) } - return filepath.Join(rec.RunDir, captureDirPrefix+series), nil + return filepath.Join(rec.RunDir, seriesDirPrefix+series), nil } -func isCaptureDir(e os.DirEntry) bool { - return e.IsDir() && strings.HasPrefix(e.Name(), captureDirPrefix) +func isSeriesDir(e os.DirEntry) bool { + return e.IsDir() && strings.HasPrefix(e.Name(), seriesDirPrefix) } // checkRoundsInStep catches a round a killed capture left half written, which would otherwise surface only at the receive. diff --git a/hypervisor/migrate_test.go b/hypervisor/migrate_test.go index 741ba4fe..493acf5a 100644 --- a/hypervisor/migrate_test.go +++ b/hypervisor/migrate_test.go @@ -27,7 +27,7 @@ func TestMigrateBaselineRefusesWithoutReflink(t *testing.T) { if calls.paused != 0 { t.Error("the guest was paused before the refusal") } - if dirs, _ := filepath.Glob(filepath.Join(rec.RunDir, captureDirPrefix+"*")); len(dirs) != 0 { + if dirs, _ := filepath.Glob(filepath.Join(rec.RunDir, seriesDirPrefix+"*")); len(dirs) != 0 { t.Errorf("series dirs left behind: %v", dirs) } } @@ -225,24 +225,20 @@ func TestMigrateRoundRewritesTheEnvelope(t *testing.T) { } } -func TestIsCaptureDirMatchesSeriesDirsOnly(t *testing.T) { +func TestIsSeriesDirLeavesASnapshotBeingPersisted(t *testing.T) { dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, captureDirPrefix+"s1"), 0o700); err != nil { - t.Fatalf("mkdir: %v", err) - } - if err := os.Mkdir(filepath.Join(dir, restoreStagingName), 0o700); err != nil { - t.Fatalf("mkdir: %v", err) + series := filepath.Join(dir, seriesDirPrefix+"s1") + persisting := filepath.Join(dir, captureDirPrefix+"123") + for _, d := range []string{series, persisting} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } } - writeSparse(t, filepath.Join(dir, captureDirPrefix+"file"), 1, nil) - if errs := utils.RemoveMatching(t.Context(), dir, isCaptureDir); len(errs) != 0 { + if errs := utils.RemoveMatching(t.Context(), dir, isSeriesDir); len(errs) != 0 { t.Fatalf("remove: %v", errs) } - entries, err := os.ReadDir(dir) - if err != nil { - t.Fatalf("read: %v", err) - } - if len(entries) != 2 || utils.FileExists(filepath.Join(dir, captureDirPrefix+"s1")) { - t.Errorf("left %v", entries) + if utils.FileExists(series) || !utils.FileExists(persisting) { + t.Errorf("series kept %t, persisting snapshot kept %t, want false true", utils.FileExists(series), utils.FileExists(persisting)) } } @@ -271,7 +267,7 @@ func TestSeriesDirRejectsPathsOutOfTheRunDir(t *testing.T) { } } got, err := seriesDir(rec, "123") - if err != nil || got != "/run/vm/snapshot-123" { + if err != nil || got != "/run/vm/snapshot-migrate-123" { t.Errorf("got %q, %v", got, err) } } @@ -333,7 +329,7 @@ func newMigrateTestVM(t *testing.T) (*Backend, string, *VMRecord) { func seedSeries(t *testing.T, rec *VMRecord) string { t.Helper() - dir := filepath.Join(rec.RunDir, captureDirPrefix+"s1") + dir := filepath.Join(rec.RunDir, seriesDirPrefix+"s1") if err := os.Mkdir(dir, 0o700); err != nil { t.Fatalf("mkdir: %v", err) } From 9371ddceccebfc820a58851751ae4f11f326b18d Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 15:10:00 +0800 Subject: [PATCH 5/8] cut: drop two delta checks a later step already makes A short delta already fails at CopyN, and a missing series directory already fails before the pause at stampFiles. --- hypervisor/diskdelta.go | 35 +++++------------------------------ hypervisor/diskdelta_test.go | 6 +++--- hypervisor/migrate.go | 3 --- 3 files changed, 8 insertions(+), 36 deletions(-) diff --git a/hypervisor/diskdelta.go b/hypervisor/diskdelta.go index b7f5c14f..3798e0d4 100644 --- a/hypervisor/diskdelta.go +++ b/hypervisor/diskdelta.go @@ -15,12 +15,7 @@ import ( "github.com/cocoonstack/cocoon/utils" ) -const ( - diskDeltaInfix = ".delta." - - // deltaLengthSize is the byte count of the big-endian header length that opens a delta file. - deltaLengthSize = 8 -) +const diskDeltaInfix = ".delta." // diskDelta heads a delta file; the bytes of each non-zero range follow in order. type diskDelta struct { @@ -139,7 +134,7 @@ func writeDeltaFile(dst string, header *diskDelta, src *os.File) (err error) { } err = os.Rename(tmp, dst) }() - if err = binary.Write(out, binary.BigEndian, uint64(len(raw))); err != nil { + if err = binary.Write(out, binary.BigEndian, int64(len(raw))); err != nil { return fmt.Errorf("write delta header: %w", err) } if _, err = out.Write(raw); err != nil { @@ -194,35 +189,15 @@ func applyDiskDelta(dst, delta string) error { return nil } -// readDeltaHeader leaves f at the first data byte and rejects a file whose size disagrees with its header. +// readDeltaHeader leaves f at the first data byte. func readDeltaHeader(f *os.File) (*diskDelta, error) { - var length uint64 + var length int64 if err := binary.Read(f, binary.BigEndian, &length); err != nil { return nil, fmt.Errorf("read delta header length: %w", err) } - fi, err := f.Stat() - if err != nil { - return nil, err - } - if length > uint64(fi.Size()) { //nolint:gosec - return nil, fmt.Errorf("delta header length %d exceeds file size %d", length, fi.Size()) - } - raw := make([]byte, length) - if _, err = io.ReadFull(f, raw); err != nil { - return nil, fmt.Errorf("read delta header: %w", err) - } var header diskDelta - if err = json.Unmarshal(raw, &header); err != nil { + if err := json.UnmarshalRead(io.LimitReader(f, length), &header); err != nil { return nil, fmt.Errorf("decode delta header: %w", err) } - want := deltaLengthSize + int64(length) - for _, r := range header.Ranges { - if !r.Zero { - want += r.Length - } - } - if fi.Size() != want { - return nil, fmt.Errorf("delta is %d bytes, its header needs %d: the transfer is incomplete", fi.Size(), want) - } return &header, nil } diff --git a/hypervisor/diskdelta_test.go b/hypervisor/diskdelta_test.go index 2264360e..f7cfe250 100644 --- a/hypervisor/diskdelta_test.go +++ b/hypervisor/diskdelta_test.go @@ -3,10 +3,10 @@ package hypervisor import ( "bytes" "errors" + "io" "os" "path/filepath" "slices" - "strings" "testing" "github.com/cocoonstack/cocoon/utils" @@ -167,8 +167,8 @@ func TestApplyDiskDeltaRejectsATruncatedFile(t *testing.T) { } dst := filepath.Join(dir, "dst") writeSparse(t, dst, 4096, nil) - if err = applyDiskDelta(dst, delta); err == nil || !strings.Contains(err.Error(), "incomplete") { - t.Errorf("applyDiskDelta: %v, want incomplete transfer", err) + if err = applyDiskDelta(dst, delta); !errors.Is(err, io.EOF) { + t.Errorf("applyDiskDelta: %v, want EOF", err) } } diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go index cc54fdc2..3a7935d5 100644 --- a/hypervisor/migrate.go +++ b/hypervisor/migrate.go @@ -153,9 +153,6 @@ func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRe if err != nil { return nil, err } - if _, err = os.Stat(dir); err != nil { - return nil, fmt.Errorf("series %s: %w", req.Series, err) - } pairs, err := writableDiskPairs(dir, rec) if err != nil { return nil, err From ad3285eca70e841f083b9a9f251ee3e0a84f6c6d Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 15:52:41 +0800 Subject: [PATCH 6/8] docs(vm): state which filesystems live migration runs on --- README.md | 2 +- docs/cli.md | 2 +- docs/snapshots.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c041eb21..11f6ad41 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Lightweight MicroVM engine with dual hypervisor backends: [Cloud Hypervisor](htt - **OCI VM images** — pull container-registry images with kernel + rootfs layers, content-addressed EROFS blob cache; **cloud images** from HTTP(S) URLs with automatic qcow2 conversion; local file/stdin import - **Docker-like CLI** — `create`, `run`, `start`, `stop`, `list`, `inspect`, `console`, `exec`, `rm`, `clone`, `status` - **Snapshot & clone** — capture a running VM (memory, disks, config) and clone it into new VMs with fresh network identity; clones restore memory copy-on-write by default (page cache shared across sibling clones) and hold shared source leases so parallel clone bursts survive concurrent source removal; restore and atomic hibernate in place; portable export/import between hosts -- **Live migration** — move a running VM to another node (CH): round-based memory and disk deltas, the guest stops only for the last round; the caller owns transfer and round policy +- **Live migration** — move a running VM to another node (CH, run dir on XFS): round-based memory and disk deltas, the guest stops only for the last round; the caller owns transfer and round policy - **CNI networking** — multi-queue virtio-net TAPs wired via TC redirect (no bridge in the data path), multi-NIC, bridge mode, NIC hot-resize - **Data disks & runtime attach** — extra virtio-blk disks at create/clone time; hot-plug vhost-user-fs shares and VFIO PCI devices (CH); external raw disks and NIC resize on CH or Firecracker `--pci` VMs; a private copy of a snapshot's COW disk on CH - **Windows guests** — UEFI + Hyper-V enlightenments via the cocoonstack CH/firmware forks diff --git a/docs/cli.md b/docs/cli.md index cb04a783..d0ef7e24 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -251,7 +251,7 @@ The dir is read-only across the call, so multiple clones of the same dir (golden ### Migrate Flags -See [Live Migration](snapshots.md#live-migration). +See [Live Migration](snapshots.md#live-migration). The source run dir needs a reflink filesystem; only XFS is tested, and ext4 is refused. `cocoon vm migrate capture`: diff --git a/docs/snapshots.md b/docs/snapshots.md index 45711195..60290361 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -176,7 +176,7 @@ To give up at any point before that, run `cocoon vm migrate abort my-vm --series Requirements: - **Hypervisor build**: the cocoonstack fork at `35b098c65260` or newer. An older VMM answers 404 on the first capture and nothing changes. A VM keeps the VMM it started on, so it needs a restart after an upgrade. -- **Filesystem**: the source run dir must support reflink (XFS, btrfs); capture refuses otherwise before it pauses the guest. On the target, keep the copied directory on the filesystem of the run dir, so memory files hard-link and the directory can go after the receive. +- **Filesystem**: the source run dir needs reflink, because disk deltas come from comparing extents the disk shares with a reflinked copy. XFS is the only filesystem tested. btrfs and bcachefs pass the capture check but are untested. ext4 and tmpfs have no reflink, so capture refuses before it pauses the guest. A target run dir without reflink falls back to a full disk copy (untested), and the VM then cannot migrate away from it. On the target, keep the copied directory on the filesystem of the run dir, so memory files hard-link and the directory can go after the receive. - **Devices**: a hot-attached disk, share or VFIO device refuses the capture, as it does for `snapshot save`. Rules for the caller: From 7395cde6b41483bb4ede075dcef56eea96f93d3b Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 16:18:10 +0800 Subject: [PATCH 7/8] cut: apply the simplify and loc-justify round Fold the series file copy into cloneSnapshotFiles, which drops a classifier type. Replace fileStamp with os.FileInfo. ValidID stops compiling a regexp at every process start for a rarely used flag. The per-round envelope write moves before the pause, so its fsync leaves the downtime window. Remove comments the names and tests already carry, including those fix commits added. vm start resuming a received guest is the normal path, so the orphaned pause warning no longer blames an interrupted capture. --- hypervisor/cloudhypervisor/direct.go | 10 +++---- hypervisor/cloudhypervisor/extend.go | 2 +- hypervisor/cloudhypervisor/migrate.go | 12 +------- hypervisor/diskdelta.go | 1 - hypervisor/diskdelta_test.go | 4 +-- hypervisor/migrate.go | 43 +++++++++++---------------- hypervisor/utils.go | 5 +--- utils/id.go | 8 +++-- utils/reflink_linux.go | 2 +- 9 files changed, 32 insertions(+), 55 deletions(-) diff --git a/hypervisor/cloudhypervisor/direct.go b/hypervisor/cloudhypervisor/direct.go index b8567463..5c81f162 100644 --- a/hypervisor/cloudhypervisor/direct.go +++ b/hypervisor/cloudhypervisor/direct.go @@ -64,21 +64,19 @@ func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig if err != nil { return nil, fmt.Errorf("parse source config: %w", err) } - return chCfg, hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, classifySnapshotFile(chCfg)) -} - -func classifySnapshotFile(chCfg *chVMConfig) hypervisor.SnapshotClassifier { cowFiles := identifyCOWFiles(chCfg) - return func(name string) hypervisor.SnapshotFileKind { + return chCfg, hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, func(name string) hypervisor.SnapshotFileKind { switch { case strings.HasPrefix(name, memoryRangeFile): return hypervisor.SnapshotFileMemory case cowFiles[name]: return hypervisor.SnapshotFileCOW + case hypervisor.IsMigrateDeltaFile(name): + return hypervisor.SnapshotFileSkip default: return hypervisor.SnapshotFileMeta } - } + }) } // cleanSnapshotFiles enumerates by name so stale data-*.raw and cocoon.json from a previous incarnation don't linger; COW files are overwritten anyway. diff --git a/hypervisor/cloudhypervisor/extend.go b/hypervisor/cloudhypervisor/extend.go index ff8b93cc..64969381 100644 --- a/hypervisor/cloudhypervisor/extend.go +++ b/hypervisor/cloudhypervisor/extend.go @@ -341,7 +341,7 @@ func convergeOrphanedPause(ctx context.Context, hc *http.Client, vmID string) (* return info, nil } log.WithFunc("cloudhypervisor.convergeOrphanedPause"). - Warnf(ctx, "vm %s is paused with no capture in flight (interrupted snapshot or hibernate), resuming", vmID) + Warnf(ctx, "vm %s is paused with no capture in flight, resuming", vmID) if err := resumeVM(ctx, hc); err != nil { return nil, fmt.Errorf("resume orphaned pause: %w", err) } diff --git a/hypervisor/cloudhypervisor/migrate.go b/hypervisor/cloudhypervisor/migrate.go index 5a66630a..4d8fe146 100644 --- a/hypervisor/cloudhypervisor/migrate.go +++ b/hypervisor/cloudhypervisor/migrate.go @@ -6,7 +6,6 @@ import ( "maps" "net/http" "os" - "path/filepath" "slices" "strings" @@ -49,17 +48,8 @@ func (ch *CloudHypervisor) migrateSpec(ctx context.Context) hypervisor.MigrateSp } func cloneSeriesFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig, error) { - chCfg, err := parseCHConfig(filepath.Join(srcDir, configJSONName)) + chCfg, err := cloneSnapshotFiles(ctx, dstDir, srcDir) if err != nil { - return nil, fmt.Errorf("parse source config: %w", err) - } - classify := classifySnapshotFile(chCfg) - if err = hypervisor.CloneSnapshotFiles(ctx, dstDir, srcDir, func(name string) hypervisor.SnapshotFileKind { - if hypervisor.IsMigrateDeltaFile(name) { - return hypervisor.SnapshotFileSkip - } - return classify(name) - }); err != nil { return nil, err } rounds, err := countMemoryDiffs(srcDir) diff --git a/hypervisor/diskdelta.go b/hypervisor/diskdelta.go index 3798e0d4..8c6b9a34 100644 --- a/hypervisor/diskdelta.go +++ b/hypervisor/diskdelta.go @@ -29,7 +29,6 @@ type deltaRange struct { Zero bool `json:"zero,omitzero"` } -// mapping is where one logical position lives; the zero value is a hole. type mapping struct { physical int64 unwritten bool diff --git a/hypervisor/diskdelta_test.go b/hypervisor/diskdelta_test.go index f7cfe250..e9145df7 100644 --- a/hypervisor/diskdelta_test.go +++ b/hypervisor/diskdelta_test.go @@ -96,9 +96,7 @@ func TestDiskDeltaRoundTrip(t *testing.T) { writeSparse(t, src, 2<<20, map[int64]string{0: "next", 1 << 19: "kept", 3 << 19: "grown"}) delta := filepath.Join(dir, diskDeltaName("disk", 1)) if err := writeDiskDelta(delta, ref, src); err != nil { - if errors.Is(err, errors.ErrUnsupported) { - t.Skip("needs FIEMAP") - } + skipWithoutFiemap(t, err) t.Fatalf("writeDiskDelta: %v", err) } if err := applyDiskDelta(ref, delta); err != nil { diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go index 3a7935d5..283e1ec9 100644 --- a/hypervisor/migrate.go +++ b/hypervisor/migrate.go @@ -19,7 +19,6 @@ import ( const ( diskRefSuffix = ".ref" - // seriesDirPrefix extends captureDirPrefix so the GC sweep still ages out a series. seriesDirPrefix = captureDirPrefix + "migrate-" ) @@ -48,20 +47,18 @@ type MigrateRound struct { Files []MigrateFile `json:"files"` } -// MigrateFile is one file a round wrote. type MigrateFile struct { Name string `json:"name"` Size int64 `json:"size"` } -// MigrateSpec carries backend hooks for the migrate sequences; SnapshotSpec.Capture writes the guest state of one round and Rounds counts those in a series directory. +// MigrateSpec carries backend hooks for the migrate sequences. type MigrateSpec struct { SnapshotSpec EndSeries TransitionHook Rounds func(dir string) (int, error) } -// roundSteps is what a baseline and a delta do differently around the shared pause window. type roundSteps struct { series string round int @@ -71,11 +68,6 @@ type roundSteps struct { afterPause func() error } -type fileStamp struct { - size int64 - modTime int64 -} - // MigrateCaptureSequence runs one round; any failure ends the series, resumes the guest and deletes the directory. func (b *Backend) MigrateCaptureSequence(ctx context.Context, ref string, req MigrateRequest, spec MigrateSpec) (*MigrateRound, error) { vmID, rec, unlock, err := b.lockForCapture(ctx, ref) @@ -95,7 +87,7 @@ func (b *Backend) MigrateCaptureSequence(ctx context.Context, ref string, req Mi return round, nil } -// MigrateAbortSequence acts only while the series directory exists, so a repeat or an old series leaves the guest alone; the directory goes last so a failed resume can retry. +// MigrateAbortSequence acts only while the series directory exists. func (b *Backend) MigrateAbortSequence(ctx context.Context, ref, series string, spec MigrateSpec) error { vmID, rec, unlock, err := b.lockForCapture(ctx, ref) if err != nil { @@ -124,7 +116,6 @@ func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec Migra if err := utils.CheckReflink(rec.RunDir); err != nil { return nil, fmt.Errorf("live migration needs a reflink filesystem under the run dir: %w", err) } - // One directory per VM keeps an old series ID from reaching the guest. if errs := utils.RemoveMatching(ctx, rec.RunDir, isSeriesDir); len(errs) > 0 { return nil, errors.Join(errs...) } @@ -162,7 +153,12 @@ func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRe series: req.Series, round: number, final: req.Final, - before: func(*http.Client) error { return checkRoundsInStep(spec, dir, pairs, number) }, + before: func(*http.Client) error { + if err := checkRoundsInStep(spec, dir, pairs, number); err != nil { + return err + } + return b.writeSeriesEnvelope(rec, req.Series, dir) + }, inPause: func() error { if req.Final { return nil @@ -170,12 +166,10 @@ func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRe return reflinkNextRefs(ctx, pairs) }, afterPause: func() error { - if _, mapErr := utils.Map(ctx, pairs, func(_ context.Context, _ int, p [2]string) (struct{}, error) { + _, mapErr := utils.Map(ctx, pairs, func(_ context.Context, _ int, p [2]string) (struct{}, error) { return struct{}{}, writeRoundDelta(p, number, req.Final) - }); mapErr != nil { - return mapErr - } - return b.writeSeriesEnvelope(rec, req.Series, dir) + }) + return mapErr }, } return b.captureRound(ctx, rec, spec, dir, steps) @@ -222,7 +216,6 @@ func (b *Backend) captureRound(ctx context.Context, rec *VMRecord, spec MigrateS return &MigrateRound{Series: steps.series, Dir: dir, Round: steps.round, Paused: steps.final, Files: files}, nil } -// writeSeriesEnvelope reruns each round because a NIC added mid-series does not end the series. func (b *Backend) writeSeriesEnvelope(rec *VMRecord, series, dir string) error { return snapshot.WriteSnapshotEnvelope(dir, *b.BuildSnapshotConfig(series, rec), nil, nil) } @@ -243,7 +236,6 @@ func isSeriesDir(e os.DirEntry) bool { return e.IsDir() && strings.HasPrefix(e.Name(), seriesDirPrefix) } -// checkRoundsInStep catches a round a killed capture left half written, which would otherwise surface only at the receive. func checkRoundsInStep(spec MigrateSpec, dir string, pairs [][2]string, round int) error { rounds, err := spec.Rounds(dir) if err != nil { @@ -298,33 +290,34 @@ func nextRefPath(copyPath string) string { return copyPath + ".next" + diskRefSuffix } -func stampFiles(dir string) (map[string]fileStamp, error) { +func stampFiles(dir string) (map[string]os.FileInfo, error) { entries, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("read series dir: %w", err) } - stamps := make(map[string]fileStamp, len(entries)) + stamps := make(map[string]os.FileInfo, len(entries)) for _, e := range entries { info, err := e.Info() if err != nil { return nil, fmt.Errorf("stat %s: %w", e.Name(), err) } if info.Mode().IsRegular() && !strings.HasSuffix(e.Name(), diskRefSuffix) { - stamps[e.Name()] = fileStamp{size: info.Size(), modTime: info.ModTime().UnixNano()} + stamps[e.Name()] = info } } return stamps, nil } -func changedFiles(dir string, before map[string]fileStamp) ([]MigrateFile, error) { +func changedFiles(dir string, before map[string]os.FileInfo) ([]MigrateFile, error) { after, err := stampFiles(dir) if err != nil { return nil, err } files := []MigrateFile{} for _, name := range slices.Sorted(maps.Keys(after)) { - if stamp, ok := before[name]; !ok || stamp != after[name] { - files = append(files, MigrateFile{Name: name, Size: after[name].size}) + info := after[name] + if old, ok := before[name]; !ok || old.Size() != info.Size() || !old.ModTime().Equal(info.ModTime()) { + files = append(files, MigrateFile{Name: name, Size: info.Size()}) } } return files, nil diff --git a/hypervisor/utils.go b/hypervisor/utils.go index e1d3be52..fb49c84b 100644 --- a/hypervisor/utils.go +++ b/hypervisor/utils.go @@ -56,9 +56,6 @@ var hostCPUCount = sync.OnceValue(readHostCPUCount) // SnapshotFileKind classifies a snapshot file for CloneSnapshotFiles. type SnapshotFileKind int -// SnapshotClassifier picks the kind of the snapshot file called name. -type SnapshotClassifier func(name string) SnapshotFileKind - // LockVMOps serializes mutating verbs on one VM across processes; the flock dies with the holder so a crash never wedges the VM. func (b *Backend) LockVMOps(ctx context.Context, vmID string) (func(), error) { l, err := vmlock.New(b.Conf.RootDirPath(), vmID) @@ -369,7 +366,7 @@ func VerifyBaseFiles(storageConfigs []*types.StorageConfig, boot *types.BootConf } // CloneSnapshotFiles copies snapshot files by kind: COW-class copies fan out concurrently, links and small meta stay serial. -func CloneSnapshotFiles(ctx context.Context, dstDir, srcDir string, classify SnapshotClassifier) error { +func CloneSnapshotFiles(ctx context.Context, dstDir, srcDir string, classify func(name string) SnapshotFileKind) error { entries, err := os.ReadDir(srcDir) if err != nil { return fmt.Errorf("read srcDir: %w", err) diff --git a/utils/id.go b/utils/id.go index a137c553..fab46f20 100644 --- a/utils/id.go +++ b/utils/id.go @@ -2,10 +2,10 @@ package utils import ( "crypto/rand" - "regexp" + "strings" ) -var validID = regexp.MustCompile(`^[A-Z2-7]{26}$`) +const idLength = 26 // GenerateID returns a random 26-character crypto-secure base32 ID. func GenerateID() string { @@ -14,5 +14,7 @@ func GenerateID() string { // ValidID reports whether id has the shape GenerateID produces. func ValidID(id string) bool { - return validID.MatchString(id) + return len(id) == idLength && !strings.ContainsFunc(id, func(r rune) bool { + return (r < 'A' || r > 'Z') && (r < '2' || r > '7') + }) } diff --git a/utils/reflink_linux.go b/utils/reflink_linux.go index 9f92da61..23fb335c 100644 --- a/utils/reflink_linux.go +++ b/utils/reflink_linux.go @@ -36,7 +36,7 @@ func ReflinkCopy(ctx context.Context, dst, src string, sync SyncMode) error { return SparseCopy(dst, src, sync) } -// CheckReflink reports whether the filesystem holding dir supports FICLONE; cloning an empty file costs no extent walk. +// CheckReflink reports whether the filesystem holding dir supports FICLONE. func CheckReflink(dir string) error { src, err := os.CreateTemp(dir, ".reflink-probe-*") if err != nil { From 95921b858569df93095517b127f149ece2078400 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 4 Nov 2026 17:55:39 +0800 Subject: [PATCH 8/8] fix(vm): refuse a delta after a round killed before its reference moved writeRoundDelta commits the delta and then renames the next reference. A kill between the two leaves rounds in step on a stale reference, so a range that went hole, data, hole compares equal and the replay keeps the old bytes. The residual next reference marks that window; the check runs before the pause and takes the existing path: end the series, resume the guest, delete the directory. A delta after a final round has the same stale reference. It takes a vm start outside the documented flow, so the docs state the rule. --- docs/snapshots.md | 2 +- hypervisor/migrate.go | 3 +++ hypervisor/migrate_test.go | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/snapshots.md b/docs/snapshots.md index 60290361..1ebf8daf 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -171,7 +171,7 @@ cocoon vm start my-vm # resumes t cocoon vm rm --force my-vm ``` -To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. Once the directory is gone, a repeat or an old series ID does nothing, so it cannot resume a guest paused for a newer series. +To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. Once the directory is gone, a repeat or an old series ID does nothing, so it cannot resume a guest paused for a newer series. After `--final`, receive the series or abort it: resuming the guest with `vm start` and capturing again leaves a delta that misses blocks the guest discarded. Requirements: diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go index 283e1ec9..c57b2742 100644 --- a/hypervisor/migrate.go +++ b/hypervisor/migrate.go @@ -248,6 +248,9 @@ func checkRoundsInStep(spec MigrateSpec, dir string, pairs [][2]string, round in if n := nextRound(p[0]); n != round { return fmt.Errorf("%s is at round %d, want %d: an earlier capture died, start a new series", filepath.Base(p[0]), n-1, round-1) } + if utils.FileExists(nextRefPath(p[0])) { + return fmt.Errorf("%s has an unmoved reference: an earlier capture died, start a new series", filepath.Base(p[0])) + } } return nil } diff --git a/hypervisor/migrate_test.go b/hypervisor/migrate_test.go index 493acf5a..31ad5529 100644 --- a/hypervisor/migrate_test.go +++ b/hypervisor/migrate_test.go @@ -184,6 +184,15 @@ func TestMigrateRoundOutOfStepDiscardsTheSeries(t *testing.T) { writeSparse(t, filepath.Join(dir, diskDeltaName(types.COWRawFileName, 1)), 8, nil) }, }, + { + name: "round killed before its reference moved", + setup: func(t *testing.T, dir string, spec *MigrateSpec) { + spec.Rounds = func(string) (int, error) { return 1, nil } + disk := filepath.Join(dir, types.COWRawFileName) + writeSparse(t, filepath.Join(dir, diskDeltaName(types.COWRawFileName, 1)), 8, nil) + writeSparse(t, nextRefPath(disk), 8, nil) + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {