feat(vm): live migration for Cloud Hypervisor VMs

README.md

@@ -9,6 +9,7 @@ Lightweight MicroVM engine with dual hypervisor backends: [Cloud Hypervisor](htt
 - **OCI VM images** — pull container-registry images with kernel + rootfs layers, content-addressed EROFS blob cache; **cloud images** from HTTP(S) URLs with automatic qcow2 conversion; local file/stdin import
 - **Docker-like CLI** — `create`, `run`, `start`, `stop`, `list`, `inspect`, `console`, `exec`, `rm`, `clone`, `status`
 - **Snapshot & clone** — capture a running VM (memory, disks, config) and clone it into new VMs with fresh network identity; clones restore memory copy-on-write by default (page cache shared across sibling clones) and hold shared source leases so parallel clone bursts survive concurrent source removal; restore and atomic hibernate in place; portable export/import between hosts
+- **Live migration** — move a running VM to another node (CH, run dir on XFS): round-based memory and disk deltas, the guest stops only for the last round; the caller owns transfer and round policy
 - **CNI networking** — multi-queue virtio-net TAPs wired via TC redirect (no bridge in the data path), multi-NIC, bridge mode, NIC hot-resize
 - **Data disks & runtime attach** — extra virtio-blk disks at create/clone time; hot-plug vhost-user-fs shares and VFIO PCI devices (CH); external raw disks and NIC resize on CH or Firecracker `--pci` VMs; a private copy of a snapshot's COW disk on CH
 - **Windows guests** — UEFI + Hyper-V enlightenments via the cocoonstack CH/firmware forks

cmd/vm/commands.go

@@ -198,6 +198,7 @@ func Command(h Handler) *cobra.Command {
 		buildDeviceCommand(h),
 		buildDiskCommand(h),
 		buildNetCommand(h),
+		buildMigrateCommand(h),
 	)
 	return vmCmd
 }
@@ -215,6 +216,44 @@ func buildNetCommand(h Handler) *cobra.Command {
 	return cmd
 }
 
+func buildMigrateCommand(h Handler) *cobra.Command {
+	parent := cliutil.GroupCommand("migrate", "Capture a running VM in rounds and restore it on another node (CH only)")
+
+	capture := &cobra.Command{
+		Use:   "capture VM",
+		Short: "Start a series with a full baseline, or add a delta to one with --series",
+		Args:  cobra.ExactArgs(1),
+		RunE:  h.MigrateCapture,
+	}
+	capture.Flags().String("series", "", "series to add a delta to (empty = start a new series)")
+	capture.Flags().Bool("final", false, "leave the guest paused after this delta; needs --series")
+	cliutil.AddOutputFlag(capture)
+
+	receive := &cobra.Command{
+		Use:   "receive --from-dir DIR",
+		Short: "Restore a transferred series into a new VM with its guest paused; vm start resumes it",
+		Args:  cobra.NoArgs,
+		RunE:  h.MigrateReceive,
+	}
+	addCloneBaseFlags(receive)
+	receive.Flags().String("from-dir", "", "directory holding the transferred series (required)")
+	receive.Flags().String("id", "", "VM ID (default: generated)")
+	_ = receive.MarkFlagRequired("from-dir")
+	cliutil.AddOutputFlag(receive)
+
+	abort := &cobra.Command{
+		Use:   "abort VM",
+		Short: "End a series, resume the guest and delete the series directory",
+		Args:  cobra.ExactArgs(1),
+		RunE:  h.MigrateAbort,
+	}
+	abort.Flags().String("series", "", "series to abort (required)")
+	_ = abort.MarkFlagRequired("series")
+
+	parent.AddCommand(capture, receive, abort)
+	return parent
+}
+
 func buildDiskCommand(h Handler) *cobra.Command {
 	parent := cliutil.GroupCommand("disk", "Attach/detach an extra raw data disk to a running VM (CH, or FC created with --pci)")
 
@@ -339,8 +378,15 @@ func addVMFlags(cmd *cobra.Command) {
 }
 
 func addCloneFlags(cmd *cobra.Command) {
-	cmd.Flags().String("name", "", "VM name (default: cocoon-clone-<id>)")
+	addCloneBaseFlags(cmd)
+	cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT")
 	cmd.Flags().Int("nics", 0, "override NIC count (omit to inherit from snapshot; on Firecracker only for --pci snapshots)")
+	cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)")
+}
+
+// addCloneBaseFlags holds the clone flags that need no device op after restore: a device op resumes a received guest.
+func addCloneBaseFlags(cmd *cobra.Command) {
+	cmd.Flags().String("name", "", "VM name (default: cocoon-clone-<id>)")
 	cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)")
 	cmd.Flags().Int("disk-queue-size", 0, "virtio-blk ring depth per device (0 = inherit from snapshot)")
 	cmd.Flags().Int("cpu-weight", 0, "cgroup cpu.weight, 1..10000 (0 = vCPU count; snapshot knobs are never inherited)")
@@ -352,8 +398,6 @@ func addCloneFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("no-direct-io", false, "disable O_DIRECT on writable disks (inherit from snapshot if not set)")
 	addRestoreModeFlag(cmd)
 	cmd.Flags().Bool("pull", false, "auto-pull base image if not found locally (for cross-node clone)")
-	cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)")
-	cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT")
 }
 
 func addCPUBandwidthFlags(cmd *cobra.Command) {

cmd/vm/migrate.go

@@ -0,0 +1,77 @@
+package vm
+
+import (
+	"fmt"
+
+	"github.com/projecteru2/core/log"
+	"github.com/spf13/cobra"
+
+	"github.com/cocoonstack/cocoon/cmd/cliutil"
+	"github.com/cocoonstack/cocoon/hypervisor"
+	"github.com/cocoonstack/cocoon/utils"
+)
+
+func (h Handler) MigrateCapture(cmd *cobra.Command, args []string) error {
+	req := hypervisor.MigrateRequest{Series: cliutil.FlagStr(cmd, "series"), Final: cliutil.FlagBool(cmd, "final")}
+	if req.Final && req.Series == "" {
+		return fmt.Errorf("--final needs --series: the baseline is never the last round")
+	}
+	ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate capture", hypervisor.ErrMigrateUnsupported)
+	if err != nil {
+		return err
+	}
+	logger := log.WithFunc("cmd.vm.migrate.capture")
+	round, err := m.MigrateCapture(ctx, args[0], req)
+	if err != nil {
+		return err
+	}
+	return outputOrLog(cmd, round, func() {
+		logger.Infof(ctx, "series %s round %d in %s (paused: %t, %d files)", round.Series, round.Round, round.Dir, round.Paused, len(round.Files))
+	})
+}
+
+func (h Handler) MigrateReceive(cmd *cobra.Command, _ []string) error {
+	id := cliutil.FlagStr(cmd, "id")
+	if id != "" && !utils.ValidID(id) {
+		return fmt.Errorf("--id must have the shape of a generated VM ID, got %q", id)
+	}
+	ctx, conf := h.Init(cmd)
+	logger := log.WithFunc("cmd.vm.migrate.receive")
+	dir := cliutil.FlagStr(cmd, "from-dir")
+	cfg, hyper, err := openSnapshotDir(ctx, conf, dir)
+	if err != nil {
+		return err
+	}
+	m, ok := hyper.(hypervisor.Migrator)
+	if !ok {
+		return fmt.Errorf("migrate receive: backend %s: %w", hyper.Type(), hypervisor.ErrMigrateUnsupported)
+	}
+	// A receive killed mid-flight leaves its placeholder and maybe a VMM under this ID.
+	if id != "" {
+		if _, err = hyper.ReconcileStaleCreate(ctx, id); err != nil {
+			return fmt.Errorf("reclaim interrupted receive: %w", err)
+		}
+	}
+	cs, err := h.prepareClone(ctx, cmd, conf, hyper, cfg)
+	if err != nil {
+		return err
+	}
+	defer cs.unlock()
+	vm, err := m.MigrateReceive(ctx, cs.vmID, cs.vmCfg, cs.netSetup, &cfg, dir)
+	if err != nil {
+		rollbackNetwork(ctx, cs.netProvider, cs.vmID)
+		cs.rollback()
+		return fmt.Errorf("migrate receive: %w", err)
+	}
+	return outputOrLog(cmd, vm, func() {
+		logger.Infof(ctx, "VM received: %s (name: %s), guest paused; resume with: cocoon vm start %s", vm.ID, vm.Config.Name, vm.ID)
+	})
+}
+
+func (h Handler) MigrateAbort(cmd *cobra.Command, args []string) error {
+	ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate abort", hypervisor.ErrMigrateUnsupported)
+	if err != nil {
+		return err
+	}
+	return m.MigrateAbort(ctx, args[0], cliutil.FlagStr(cmd, "series"))
+}

cmd/vm/run.go

@@ -231,14 +231,7 @@ func (h Handler) cloneDirect(ctx context.Context, cmd *cobra.Command, conf *conf
 
 // cloneFromDir keeps the dir read-only, so concurrent clones of a golden image are safe.
 func (h Handler) cloneFromDir(ctx context.Context, cmd *cobra.Command, conf *config.Config, dir string, logger *log.Fields) error {
-	cfg, err := readVerifiedEnvelope(dir)
-	if err != nil {
-		return err
-	}
-	if cfg.Hypervisor != "" {
-		conf.UseFirecracker = isFirecracker(cfg.Hypervisor)
-	}
-	hyper, err := cmdcore.InitHypervisor(ctx, conf)
+	cfg, hyper, err := openSnapshotDir(ctx, conf, dir)
 	if err != nil {
 		return err
 	}
@@ -286,7 +279,7 @@ func (h Handler) prepareClone(ctx context.Context, cmd *cobra.Command, conf *con
 	if err != nil {
 		return cloneSetup{}, err
 	}
-	vmID := utils.GenerateID()
+	vmID := cmp.Or(cliutil.FlagStr(cmd, "id"), utils.GenerateID())
 	vmCfg.Name = cmp.Or(vmCfg.Name, "cocoon-clone-"+network.VMIDPrefix(vmID))
 	if err = vmCfg.Validate(); err != nil {
 		return cloneSetup{}, err
@@ -500,7 +493,6 @@ func prereserveVM(ctx context.Context, hyper hypervisor.Hypervisor, vmID string,
 		return nil, nil, err
 	}
 	if err := hyper.PrereserveVM(ctx, vmID, vmCfg, blobIDs); err != nil {
-		hyper.RollbackCreate(ctx, vmID, vmCfg.Name)
 		unlock()
 		return nil, nil, fmt.Errorf("reserve VM record: %w", err)
 	}
@@ -521,6 +513,21 @@ func snapshotSource(cmd *cobra.Command, args []string, baseArgs int) (fromDir, s
 	return "", args[baseArgs], nil
 }
 
+func openSnapshotDir(ctx context.Context, conf *config.Config, dir string) (types.SnapshotConfig, hypervisor.Hypervisor, error) {
+	cfg, err := readVerifiedEnvelope(dir)
+	if err != nil {
+		return types.SnapshotConfig{}, nil, err
+	}
+	if cfg.Hypervisor != "" {
+		conf.UseFirecracker = isFirecracker(cfg.Hypervisor)
+	}
+	hyper, err := cmdcore.InitHypervisor(ctx, conf)
+	if err != nil {
+		return types.SnapshotConfig{}, nil, err
+	}
+	return cfg, hyper, nil
+}
+
 func readVerifiedEnvelope(dir string) (types.SnapshotConfig, error) {
 	envelope, err := snapshot.ReadSnapshotEnvelope(dir)
 	if err != nil {

docs/cli.md

@@ -39,6 +39,10 @@ cocoon
 │   │   ├── attach [flags] VM     Hot-attach a raw disk file (CH, or FC with --pci) or a snapshot's COW disk copy (CH)
 │   │   └── detach [flags] VM     Detach a hot-attached disk by --name (keeps a --path file, removes a snapshot copy)
 │   ├── net [flags] VM             Resize NIC count on a running VM (CH, or FC with --pci)
+│   ├── migrate
+│   │   ├── capture [flags] VM     Start a migration series, or add a delta with --series (CH only)
+│   │   ├── receive [flags]        Restore a transferred series into a new VM, guest paused
+│   │   └── abort --series ID VM   End a series, resume the guest, delete the directory
 │   └── debug [flags] IMAGE        Generate hypervisor launch command (dry run)
 ├── snapshot
 │   ├── save [flags] VM            Create a snapshot from a running VM
@@ -245,6 +249,21 @@ cocoon vm restore my-vm --from-dir /sync/from-host-b --force
 
 The dir is read-only across the call, so multiple clones of the same dir (golden image use case) are safe. Pass `--pull` if the base image may not be present locally — `EnsureImage` pulls the envelope's image by digest when it is missing, and `image_blob_ids` pin its blobs against GC for the clone's lifetime.
 
+### Migrate Flags
+
+See [Live Migration](snapshots.md#live-migration). The source run dir needs a reflink filesystem; only XFS is tested, and ext4 is refused.
+
+`cocoon vm migrate capture`:
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--series` | empty | Series to add a delta to; empty starts a new series |
+| `--final` | `false` | Leave the guest paused after this delta; needs `--series` |
+
+`cocoon vm migrate receive` takes `--from-dir` (required), `--id` (default: generated, must have the shape of a generated ID) and the [clone flags](#clone-flags) except `--nics` and `--data-disk`: a device op after restore would resume the guest. With `--output json`, `capture` prints `{"series", "dir", "round", "paused", "files": [{"name", "size"}]}`; round 0 is the baseline.
+
+`cocoon vm migrate abort` requires `--series`.
+
 ### Reconcile Stale Create
 
 `cocoon vm reconcile-stale-create VM` reclaims a `creating` placeholder whose owning create or clone died mid-flight, freeing the record and its name. A free VM ops lock is the proof of ownerlessness — create and clone hold it from prereserve through the final record commit — so the verb never races an in-flight operation and needs no age heuristics (contrast `vm rm --force`, which queues behind a live clone and then deletes the freshly created VM). With `--output json` it prints `{"id": "...", "outcome": "..."}`, omitting `id` for `not-found`:

docs/daemon.md

@@ -63,7 +63,7 @@ The `file` backend writes one JSON object per line to
 `kind` is one of `vm.compute.start`, `vm.compute.stop`, `vm.storage.start`,
 `vm.storage.stop`, `snap.storage.start`, `snap.storage.stop` — downstream pairs
 each `*.start` with its `*.stop` by id. `reason` is one of `boot`, `restart`,
-`clone`, `restore`, `stop-user`, `stop-crash`, `vm-rm`, `snap-rm`.
+`clone`, `restore`, `migrate`, `stop-user`, `stop-crash`, `vm-rm`, `snap-rm`.
 
 One operational note: the daemon is the first long-lived writer of the metering
 ledger, and the `file` backend holds that file open for its whole lifetime. If

docs/index.md

@@ -26,7 +26,7 @@ cocoon CLI ──► images: OCI (EROFS layers, direct boot) | cloudimg (qcow2,
 - [Networking](networking.md) — CNI with TC redirect, bridge mode,
   multi-NIC, NIC hot-resize
 - [Snapshots, clone & restore](snapshots.md) — capture, clone, restore,
-  hibernate, export/import, cross-node moves
+  hibernate, export/import, cross-node moves, live migration
 - [Runtime device attach](devices.md) — vhost-user-fs shares, VFIO PCI
   passthrough, external disk hot-attach
 - [Windows guests](windows.md) — the `--windows` path and the patched

docs/snapshots.md

@@ -152,6 +152,45 @@ Pause, capture, persist, and VMM termination share one pause window: the snapsho
 
 A canceled or timed-out `vm hibernate` or `snapshot save` waits for the hypervisor to end its memory dump, up to 10 minutes, then resumes the guest, so SIGTERM can take that long to end the command. SIGKILL skips the resume; see [known issues](known-issues.md#a-killed-capture-leaves-the-guest-paused).
 
+### Live Migration
+
+`vm migrate` captures a running VM in rounds and restores it on another node (Cloud Hypervisor only). The guest stops only for the last round, the copy of its files and the restore, and its processes and memory carry over. Cocoon captures and restores; moving the files, choosing how many rounds to run, and deciding which node owns the VM stay with the caller.
+
+```bash
+# Source: a full baseline, then deltas. The guest runs again after each one.
+cocoon vm migrate capture my-vm --output json                        # prints series, dir, files
+cocoon vm migrate capture my-vm --series 123 --output json           # optional, repeatable
+# Source: the last delta leaves the guest paused.
+cocoon vm migrate capture my-vm --series 123 --final --output json
+
+# Target: after copying every file each round listed into one directory
+cocoon vm migrate receive --from-dir /xfer/123 --id <source-vm-id> --name my-vm --pull
+cocoon vm start my-vm                                                # resumes the guest
+
+# Source: only after the target runs
+cocoon vm rm --force my-vm
+```
+
+To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. Once the directory is gone, a repeat or an old series ID does nothing, so it cannot resume a guest paused for a newer series. After `--final`, receive the series or abort it: resuming the guest with `vm start` and capturing again leaves a delta that misses blocks the guest discarded.
+
+Requirements:
+
+- **Hypervisor build**: the cocoonstack fork at `35b098c65260` or newer. An older VMM answers 404 on the first capture and nothing changes. A VM keeps the VMM it started on, so it needs a restart after an upgrade.
+- **Filesystem**: the source run dir needs reflink, because disk deltas come from comparing extents the disk shares with a reflinked copy. XFS is the only filesystem tested. btrfs and bcachefs pass the capture check but are untested. ext4 and tmpfs have no reflink, so capture refuses before it pauses the guest. A target run dir without reflink falls back to a full disk copy (untested), and the VM then cannot migrate away from it. On the target, keep the copied directory on the filesystem of the run dir, so memory files hard-link and the directory can go after the receive.
+- **Devices**: a hot-attached disk, share or VFIO device refuses the capture, as it does for `snapshot save`.
+
+Rules for the caller:
+
+- Copy every file a round lists, in the listed round order. `state.json` and `config.json` are replaced each round. A `.ref` file never leaves the source. A short or missing disk delta fails the receive.
+- A VM has one series. A new baseline ends the previous one and deletes its directory; the directory of a series nobody continues stays until `gc` removes it after 24 hours. A delta with an old series ID fails and leaves the guest alone.
+- A failed round ends the series, resumes the guest and deletes its directory; start again with a new baseline. A round after a killed capture fails the same way. `snapshot save`, removing a NIC, a device detach, a reboot or a stop ends the series too, and the next delta fails.
+- A VM cloned or received seconds ago may still eject its old NICs. If that happens during a series, the next delta fails; start again.
+- Without `--id` the receive generates an ID. With `--id`, a retry after a killed receive reclaims what it left. A retry after a finished receive fails with `id collision`; `vm inspect` shows the VM. Only `vm migrate receive` reads a series directory: `vm clone --from-dir` would pair the last memory with the baseline disks.
+
+The received VM gets fresh NICs like a clone, with new MACs, so the guest needs the [post-clone network setup](#post-clone-guest-setup). No reseed runs: the guest state is the same one, not a copy. `--restore-mode ondemand` is rejected by the hypervisor for a series with deltas. Metering opens the new intervals with reason `migrate`.
+
+A qcow2 overlay is open when captured, so its dirty bit is set and the target VMM rebuilds the refcounts when it opens the file. The overlay hash therefore differs after the receive; compare it before the VMM opens it.
+
 ### Restore Constraints
 
 - **VM must be running, stopped, or error.** Restore cold-spawns a fresh hypervisor process from the snapshot; a stopped target (e.g. hibernated) gets the `vm start` network self-heal first. Error-state VMs are admitted because restore rebuilds the run dir — it is the recovery path for a crashed or interrupted restore (which `vm start` refuses).

hypervisor/clone.go

@@ -57,8 +57,8 @@ func (b *Backend) RunningCloneRecord(rec *VMRecord, vmCfg *types.VMConfig, stora
 	return info
 }
 
-// FinalizeClone persists the record and emits the clone open-interval pair.
-func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string) error {
+// FinalizeClone persists the record and emits the open-interval pair.
+func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string, reason metering.Reason) error {
 	if err := b.UpdateRecord(ctx, vmID, func(r *VMRecord) error {
 		r.VM = *info
 		// The subpackage building info cannot reach markTransition; without this the clone commits at generation zero.
@@ -72,7 +72,7 @@ func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM
 	}); err != nil {
 		return err
 	}
-	b.emitOpenInterval(ctx, info, metering.ReasonClone, sourceSnapshotID, timeNow())
+	b.emitOpenInterval(ctx, info, reason, sourceSnapshotID, timeNow())
 	return nil
 }