diff --git a/README.md b/README.md index a7e1d61b..11f6ad41 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Lightweight MicroVM engine with dual hypervisor backends: [Cloud Hypervisor](htt - **OCI VM images** — pull container-registry images with kernel + rootfs layers, content-addressed EROFS blob cache; **cloud images** from HTTP(S) URLs with automatic qcow2 conversion; local file/stdin import - **Docker-like CLI** — `create`, `run`, `start`, `stop`, `list`, `inspect`, `console`, `exec`, `rm`, `clone`, `status` - **Snapshot & clone** — capture a running VM (memory, disks, config) and clone it into new VMs with fresh network identity; clones restore memory copy-on-write by default (page cache shared across sibling clones) and hold shared source leases so parallel clone bursts survive concurrent source removal; restore and atomic hibernate in place; portable export/import between hosts +- **Live migration** — move a running VM to another node (CH, run dir on XFS): round-based memory and disk deltas, the guest stops only for the last round; the caller owns transfer and round policy - **CNI networking** — multi-queue virtio-net TAPs wired via TC redirect (no bridge in the data path), multi-NIC, bridge mode, NIC hot-resize - **Data disks & runtime attach** — extra virtio-blk disks at create/clone time; hot-plug vhost-user-fs shares and VFIO PCI devices (CH); external raw disks and NIC resize on CH or Firecracker `--pci` VMs; a private copy of a snapshot's COW disk on CH - **Windows guests** — UEFI + Hyper-V enlightenments via the cocoonstack CH/firmware forks diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index 1c61523b..6a9a20f9 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -198,6 +198,7 @@ func Command(h Handler) *cobra.Command { buildDeviceCommand(h), buildDiskCommand(h), buildNetCommand(h), + buildMigrateCommand(h), ) return vmCmd } @@ -215,6 +216,44 @@ func buildNetCommand(h Handler) *cobra.Command { return cmd } +func buildMigrateCommand(h Handler) *cobra.Command { + parent := cliutil.GroupCommand("migrate", "Capture a running VM in rounds and restore it on another node (CH only)") + + capture := &cobra.Command{ + Use: "capture VM", + Short: "Start a series with a full baseline, or add a delta to one with --series", + Args: cobra.ExactArgs(1), + RunE: h.MigrateCapture, + } + capture.Flags().String("series", "", "series to add a delta to (empty = start a new series)") + capture.Flags().Bool("final", false, "leave the guest paused after this delta; needs --series") + cliutil.AddOutputFlag(capture) + + receive := &cobra.Command{ + Use: "receive --from-dir DIR", + Short: "Restore a transferred series into a new VM with its guest paused; vm start resumes it", + Args: cobra.NoArgs, + RunE: h.MigrateReceive, + } + addCloneBaseFlags(receive) + receive.Flags().String("from-dir", "", "directory holding the transferred series (required)") + receive.Flags().String("id", "", "VM ID (default: generated)") + _ = receive.MarkFlagRequired("from-dir") + cliutil.AddOutputFlag(receive) + + abort := &cobra.Command{ + Use: "abort VM", + Short: "End a series, resume the guest and delete the series directory", + Args: cobra.ExactArgs(1), + RunE: h.MigrateAbort, + } + abort.Flags().String("series", "", "series to abort (required)") + _ = abort.MarkFlagRequired("series") + + parent.AddCommand(capture, receive, abort) + return parent +} + func buildDiskCommand(h Handler) *cobra.Command { parent := cliutil.GroupCommand("disk", "Attach/detach an extra raw data disk to a running VM (CH, or FC created with --pci)") @@ -339,8 +378,15 @@ func addVMFlags(cmd *cobra.Command) { } func addCloneFlags(cmd *cobra.Command) { - cmd.Flags().String("name", "", "VM name (default: cocoon-clone-)") + addCloneBaseFlags(cmd) + cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT") cmd.Flags().Int("nics", 0, "override NIC count (omit to inherit from snapshot; on Firecracker only for --pci snapshots)") + cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)") +} + +// addCloneBaseFlags holds the clone flags that need no device op after restore: a device op resumes a received guest. +func addCloneBaseFlags(cmd *cobra.Command) { + cmd.Flags().String("name", "", "VM name (default: cocoon-clone-)") cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)") cmd.Flags().Int("disk-queue-size", 0, "virtio-blk ring depth per device (0 = inherit from snapshot)") cmd.Flags().Int("cpu-weight", 0, "cgroup cpu.weight, 1..10000 (0 = vCPU count; snapshot knobs are never inherited)") @@ -352,8 +398,6 @@ func addCloneFlags(cmd *cobra.Command) { cmd.Flags().Bool("no-direct-io", false, "disable O_DIRECT on writable disks (inherit from snapshot if not set)") addRestoreModeFlag(cmd) cmd.Flags().Bool("pull", false, "auto-pull base image if not found locally (for cross-node clone)") - cmd.Flags().StringArray("data-disk", nil, "create and hot-add an extra data disk to the clone: size=20G[,name=...][,fstype=ext4|none]; repeatable (CH, or FC --pci snapshots)") - cmd.Flags().String("from-dir", "", "clone from a snapshot directory (must contain snapshot.json) instead of the local snapshot DB; mutually exclusive with positional SNAPSHOT") } func addCPUBandwidthFlags(cmd *cobra.Command) { diff --git a/cmd/vm/migrate.go b/cmd/vm/migrate.go new file mode 100644 index 00000000..854208d4 --- /dev/null +++ b/cmd/vm/migrate.go @@ -0,0 +1,77 @@ +package vm + +import ( + "fmt" + + "github.com/projecteru2/core/log" + "github.com/spf13/cobra" + + "github.com/cocoonstack/cocoon/cmd/cliutil" + "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/utils" +) + +func (h Handler) MigrateCapture(cmd *cobra.Command, args []string) error { + req := hypervisor.MigrateRequest{Series: cliutil.FlagStr(cmd, "series"), Final: cliutil.FlagBool(cmd, "final")} + if req.Final && req.Series == "" { + return fmt.Errorf("--final needs --series: the baseline is never the last round") + } + ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate capture", hypervisor.ErrMigrateUnsupported) + if err != nil { + return err + } + logger := log.WithFunc("cmd.vm.migrate.capture") + round, err := m.MigrateCapture(ctx, args[0], req) + if err != nil { + return err + } + return outputOrLog(cmd, round, func() { + logger.Infof(ctx, "series %s round %d in %s (paused: %t, %d files)", round.Series, round.Round, round.Dir, round.Paused, len(round.Files)) + }) +} + +func (h Handler) MigrateReceive(cmd *cobra.Command, _ []string) error { + id := cliutil.FlagStr(cmd, "id") + if id != "" && !utils.ValidID(id) { + return fmt.Errorf("--id must have the shape of a generated VM ID, got %q", id) + } + ctx, conf := h.Init(cmd) + logger := log.WithFunc("cmd.vm.migrate.receive") + dir := cliutil.FlagStr(cmd, "from-dir") + cfg, hyper, err := openSnapshotDir(ctx, conf, dir) + if err != nil { + return err + } + m, ok := hyper.(hypervisor.Migrator) + if !ok { + return fmt.Errorf("migrate receive: backend %s: %w", hyper.Type(), hypervisor.ErrMigrateUnsupported) + } + // A receive killed mid-flight leaves its placeholder and maybe a VMM under this ID. + if id != "" { + if _, err = hyper.ReconcileStaleCreate(ctx, id); err != nil { + return fmt.Errorf("reclaim interrupted receive: %w", err) + } + } + cs, err := h.prepareClone(ctx, cmd, conf, hyper, cfg) + if err != nil { + return err + } + defer cs.unlock() + vm, err := m.MigrateReceive(ctx, cs.vmID, cs.vmCfg, cs.netSetup, &cfg, dir) + if err != nil { + rollbackNetwork(ctx, cs.netProvider, cs.vmID) + cs.rollback() + return fmt.Errorf("migrate receive: %w", err) + } + return outputOrLog(cmd, vm, func() { + logger.Infof(ctx, "VM received: %s (name: %s), guest paused; resume with: cocoon vm start %s", vm.ID, vm.Config.Name, vm.ID) + }) +} + +func (h Handler) MigrateAbort(cmd *cobra.Command, args []string) error { + ctx, _, _, m, err := h.resolveAttacher[hypervisor.Migrator](cmd, args, "migrate abort", hypervisor.ErrMigrateUnsupported) + if err != nil { + return err + } + return m.MigrateAbort(ctx, args[0], cliutil.FlagStr(cmd, "series")) +} diff --git a/cmd/vm/run.go b/cmd/vm/run.go index 1f2ca5ad..263009d3 100644 --- a/cmd/vm/run.go +++ b/cmd/vm/run.go @@ -231,14 +231,7 @@ func (h Handler) cloneDirect(ctx context.Context, cmd *cobra.Command, conf *conf // cloneFromDir keeps the dir read-only, so concurrent clones of a golden image are safe. func (h Handler) cloneFromDir(ctx context.Context, cmd *cobra.Command, conf *config.Config, dir string, logger *log.Fields) error { - cfg, err := readVerifiedEnvelope(dir) - if err != nil { - return err - } - if cfg.Hypervisor != "" { - conf.UseFirecracker = isFirecracker(cfg.Hypervisor) - } - hyper, err := cmdcore.InitHypervisor(ctx, conf) + cfg, hyper, err := openSnapshotDir(ctx, conf, dir) if err != nil { return err } @@ -286,7 +279,7 @@ func (h Handler) prepareClone(ctx context.Context, cmd *cobra.Command, conf *con if err != nil { return cloneSetup{}, err } - vmID := utils.GenerateID() + vmID := cmp.Or(cliutil.FlagStr(cmd, "id"), utils.GenerateID()) vmCfg.Name = cmp.Or(vmCfg.Name, "cocoon-clone-"+network.VMIDPrefix(vmID)) if err = vmCfg.Validate(); err != nil { return cloneSetup{}, err @@ -500,7 +493,6 @@ func prereserveVM(ctx context.Context, hyper hypervisor.Hypervisor, vmID string, return nil, nil, err } if err := hyper.PrereserveVM(ctx, vmID, vmCfg, blobIDs); err != nil { - hyper.RollbackCreate(ctx, vmID, vmCfg.Name) unlock() return nil, nil, fmt.Errorf("reserve VM record: %w", err) } @@ -521,6 +513,21 @@ func snapshotSource(cmd *cobra.Command, args []string, baseArgs int) (fromDir, s return "", args[baseArgs], nil } +func openSnapshotDir(ctx context.Context, conf *config.Config, dir string) (types.SnapshotConfig, hypervisor.Hypervisor, error) { + cfg, err := readVerifiedEnvelope(dir) + if err != nil { + return types.SnapshotConfig{}, nil, err + } + if cfg.Hypervisor != "" { + conf.UseFirecracker = isFirecracker(cfg.Hypervisor) + } + hyper, err := cmdcore.InitHypervisor(ctx, conf) + if err != nil { + return types.SnapshotConfig{}, nil, err + } + return cfg, hyper, nil +} + func readVerifiedEnvelope(dir string) (types.SnapshotConfig, error) { envelope, err := snapshot.ReadSnapshotEnvelope(dir) if err != nil { diff --git a/docs/cli.md b/docs/cli.md index 451ee88b..d0ef7e24 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -39,6 +39,10 @@ cocoon │ │ ├── attach [flags] VM Hot-attach a raw disk file (CH, or FC with --pci) or a snapshot's COW disk copy (CH) │ │ └── detach [flags] VM Detach a hot-attached disk by --name (keeps a --path file, removes a snapshot copy) │ ├── net [flags] VM Resize NIC count on a running VM (CH, or FC with --pci) +│ ├── migrate +│ │ ├── capture [flags] VM Start a migration series, or add a delta with --series (CH only) +│ │ ├── receive [flags] Restore a transferred series into a new VM, guest paused +│ │ └── abort --series ID VM End a series, resume the guest, delete the directory │ └── debug [flags] IMAGE Generate hypervisor launch command (dry run) ├── snapshot │ ├── save [flags] VM Create a snapshot from a running VM @@ -245,6 +249,21 @@ cocoon vm restore my-vm --from-dir /sync/from-host-b --force The dir is read-only across the call, so multiple clones of the same dir (golden image use case) are safe. Pass `--pull` if the base image may not be present locally — `EnsureImage` pulls the envelope's image by digest when it is missing, and `image_blob_ids` pin its blobs against GC for the clone's lifetime. +### Migrate Flags + +See [Live Migration](snapshots.md#live-migration). The source run dir needs a reflink filesystem; only XFS is tested, and ext4 is refused. + +`cocoon vm migrate capture`: + +| Flag | Default | Description | +|------|---------|-------------| +| `--series` | empty | Series to add a delta to; empty starts a new series | +| `--final` | `false` | Leave the guest paused after this delta; needs `--series` | + +`cocoon vm migrate receive` takes `--from-dir` (required), `--id` (default: generated, must have the shape of a generated ID) and the [clone flags](#clone-flags) except `--nics` and `--data-disk`: a device op after restore would resume the guest. With `--output json`, `capture` prints `{"series", "dir", "round", "paused", "files": [{"name", "size"}]}`; round 0 is the baseline. + +`cocoon vm migrate abort` requires `--series`. + ### Reconcile Stale Create `cocoon vm reconcile-stale-create VM` reclaims a `creating` placeholder whose owning create or clone died mid-flight, freeing the record and its name. A free VM ops lock is the proof of ownerlessness — create and clone hold it from prereserve through the final record commit — so the verb never races an in-flight operation and needs no age heuristics (contrast `vm rm --force`, which queues behind a live clone and then deletes the freshly created VM). With `--output json` it prints `{"id": "...", "outcome": "..."}`, omitting `id` for `not-found`: diff --git a/docs/daemon.md b/docs/daemon.md index cab0112b..6c36aaca 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -63,7 +63,7 @@ The `file` backend writes one JSON object per line to `kind` is one of `vm.compute.start`, `vm.compute.stop`, `vm.storage.start`, `vm.storage.stop`, `snap.storage.start`, `snap.storage.stop` — downstream pairs each `*.start` with its `*.stop` by id. `reason` is one of `boot`, `restart`, -`clone`, `restore`, `stop-user`, `stop-crash`, `vm-rm`, `snap-rm`. +`clone`, `restore`, `migrate`, `stop-user`, `stop-crash`, `vm-rm`, `snap-rm`. One operational note: the daemon is the first long-lived writer of the metering ledger, and the `file` backend holds that file open for its whole lifetime. If diff --git a/docs/index.md b/docs/index.md index b6456ec7..0c93389c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,7 @@ cocoon CLI ──► images: OCI (EROFS layers, direct boot) | cloudimg (qcow2, - [Networking](networking.md) — CNI with TC redirect, bridge mode, multi-NIC, NIC hot-resize - [Snapshots, clone & restore](snapshots.md) — capture, clone, restore, - hibernate, export/import, cross-node moves + hibernate, export/import, cross-node moves, live migration - [Runtime device attach](devices.md) — vhost-user-fs shares, VFIO PCI passthrough, external disk hot-attach - [Windows guests](windows.md) — the `--windows` path and the patched diff --git a/docs/snapshots.md b/docs/snapshots.md index 8863e697..1ebf8daf 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -152,6 +152,45 @@ Pause, capture, persist, and VMM termination share one pause window: the snapsho A canceled or timed-out `vm hibernate` or `snapshot save` waits for the hypervisor to end its memory dump, up to 10 minutes, then resumes the guest, so SIGTERM can take that long to end the command. SIGKILL skips the resume; see [known issues](known-issues.md#a-killed-capture-leaves-the-guest-paused). +### Live Migration + +`vm migrate` captures a running VM in rounds and restores it on another node (Cloud Hypervisor only). The guest stops only for the last round, the copy of its files and the restore, and its processes and memory carry over. Cocoon captures and restores; moving the files, choosing how many rounds to run, and deciding which node owns the VM stay with the caller. + +```bash +# Source: a full baseline, then deltas. The guest runs again after each one. +cocoon vm migrate capture my-vm --output json # prints series, dir, files +cocoon vm migrate capture my-vm --series 123 --output json # optional, repeatable +# Source: the last delta leaves the guest paused. +cocoon vm migrate capture my-vm --series 123 --final --output json + +# Target: after copying every file each round listed into one directory +cocoon vm migrate receive --from-dir /xfer/123 --id --name my-vm --pull +cocoon vm start my-vm # resumes the guest + +# Source: only after the target runs +cocoon vm rm --force my-vm +``` + +To give up at any point before that, run `cocoon vm migrate abort my-vm --series 123`: it ends the series, resumes the guest and deletes the directory. Once the directory is gone, a repeat or an old series ID does nothing, so it cannot resume a guest paused for a newer series. After `--final`, receive the series or abort it: resuming the guest with `vm start` and capturing again leaves a delta that misses blocks the guest discarded. + +Requirements: + +- **Hypervisor build**: the cocoonstack fork at `35b098c65260` or newer. An older VMM answers 404 on the first capture and nothing changes. A VM keeps the VMM it started on, so it needs a restart after an upgrade. +- **Filesystem**: the source run dir needs reflink, because disk deltas come from comparing extents the disk shares with a reflinked copy. XFS is the only filesystem tested. btrfs and bcachefs pass the capture check but are untested. ext4 and tmpfs have no reflink, so capture refuses before it pauses the guest. A target run dir without reflink falls back to a full disk copy (untested), and the VM then cannot migrate away from it. On the target, keep the copied directory on the filesystem of the run dir, so memory files hard-link and the directory can go after the receive. +- **Devices**: a hot-attached disk, share or VFIO device refuses the capture, as it does for `snapshot save`. + +Rules for the caller: + +- Copy every file a round lists, in the listed round order. `state.json` and `config.json` are replaced each round. A `.ref` file never leaves the source. A short or missing disk delta fails the receive. +- A VM has one series. A new baseline ends the previous one and deletes its directory; the directory of a series nobody continues stays until `gc` removes it after 24 hours. A delta with an old series ID fails and leaves the guest alone. +- A failed round ends the series, resumes the guest and deletes its directory; start again with a new baseline. A round after a killed capture fails the same way. `snapshot save`, removing a NIC, a device detach, a reboot or a stop ends the series too, and the next delta fails. +- A VM cloned or received seconds ago may still eject its old NICs. If that happens during a series, the next delta fails; start again. +- Without `--id` the receive generates an ID. With `--id`, a retry after a killed receive reclaims what it left. A retry after a finished receive fails with `id collision`; `vm inspect` shows the VM. Only `vm migrate receive` reads a series directory: `vm clone --from-dir` would pair the last memory with the baseline disks. + +The received VM gets fresh NICs like a clone, with new MACs, so the guest needs the [post-clone network setup](#post-clone-guest-setup). No reseed runs: the guest state is the same one, not a copy. `--restore-mode ondemand` is rejected by the hypervisor for a series with deltas. Metering opens the new intervals with reason `migrate`. + +A qcow2 overlay is open when captured, so its dirty bit is set and the target VMM rebuilds the refcounts when it opens the file. The overlay hash therefore differs after the receive; compare it before the VMM opens it. + ### Restore Constraints - **VM must be running, stopped, or error.** Restore cold-spawns a fresh hypervisor process from the snapshot; a stopped target (e.g. hibernated) gets the `vm start` network self-heal first. Error-state VMs are admitted because restore rebuilds the run dir — it is the recovery path for a crashed or interrupted restore (which `vm start` refuses). diff --git a/hypervisor/clone.go b/hypervisor/clone.go index 75a75ab2..a16614fd 100644 --- a/hypervisor/clone.go +++ b/hypervisor/clone.go @@ -57,8 +57,8 @@ func (b *Backend) RunningCloneRecord(rec *VMRecord, vmCfg *types.VMConfig, stora return info } -// FinalizeClone persists the record and emits the clone open-interval pair. -func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string) error { +// FinalizeClone persists the record and emits the open-interval pair. +func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string, reason metering.Reason) error { if err := b.UpdateRecord(ctx, vmID, func(r *VMRecord) error { r.VM = *info // The subpackage building info cannot reach markTransition; without this the clone commits at generation zero. @@ -72,7 +72,7 @@ func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM }); err != nil { return err } - b.emitOpenInterval(ctx, info, metering.ReasonClone, sourceSnapshotID, timeNow()) + b.emitOpenInterval(ctx, info, reason, sourceSnapshotID, timeNow()) return nil } diff --git a/hypervisor/cloudhypervisor/clone.go b/hypervisor/cloudhypervisor/clone.go index 608e0ba6..81992a61 100644 --- a/hypervisor/cloudhypervisor/clone.go +++ b/hypervisor/cloudhypervisor/clone.go @@ -13,6 +13,7 @@ import ( "github.com/cocoonstack/cocoon/config" "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/metering" "github.com/cocoonstack/cocoon/network" "github.com/cocoonstack/cocoon/types" "github.com/cocoonstack/cocoon/utils" @@ -28,6 +29,7 @@ type cloneResumeOpts struct { networkConfigs []*types.NetworkConfig snapshotCfg *chVMConfig queueCPUs []int + keepPaused bool } func (ch *CloudHypervisor) Clone(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, snapshot io.Reader) (*types.VM, error) { @@ -39,10 +41,10 @@ func (ch *CloudHypervisor) cloneAfterExtract(ctx context.Context, rec *hyperviso if err != nil { return nil, fmt.Errorf("parse CH config: %w", err) } - return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, chCfg) + return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, chCfg, false) } -func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string, chCfg *chVMConfig) (*types.VM, error) { +func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string, chCfg *chVMConfig, migrate bool) (*types.VM, error) { vmID, runDir := rec.ID, rec.RunDir networkConfigs := net.NetworkConfigs logger := log.WithFunc("cloudhypervisor.cloneAfterExtractParsed") @@ -136,13 +138,18 @@ func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, rec *hyp networkConfigs: networkConfigs, snapshotCfg: chCfg, queueCPUs: queueCPUs, + keepPaused: migrate, }); err != nil { return nil, err } saveConsolePTY(ctx, vmID, runDir, sockPath, directBoot) info := ch.RunningCloneRecord(rec, vmCfg, storageConfigs, net) - if err := ch.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID); err != nil { + reason := metering.ReasonClone + if migrate { + reason = metering.ReasonMigrate + } + if err := ch.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID, reason); err != nil { ch.AbortLaunch(ctx, pid, sockPath, runDir, runtimeFiles) return nil, fmt.Errorf("finalize VM record: %w", err) } @@ -186,6 +193,9 @@ func (ch *CloudHypervisor) restoreAndResumeClone(ctx context.Context, pid int, s if err = ch.ArmCPUQuota(opts.vmID, &opts.vmCfg.Config); err != nil { return err } + if opts.keepPaused { + return nil + } if err = resumeVM(ctx, hc); err != nil { return fmt.Errorf("vm.resume: %w", err) } diff --git a/hypervisor/cloudhypervisor/cloudhypervisor.go b/hypervisor/cloudhypervisor/cloudhypervisor.go index a03e07fc..eb95610b 100644 --- a/hypervisor/cloudhypervisor/cloudhypervisor.go +++ b/hypervisor/cloudhypervisor/cloudhypervisor.go @@ -22,6 +22,7 @@ var ( _ disk.SnapshotAttacher = (*CloudHypervisor)(nil) _ fs.Attacher = (*CloudHypervisor)(nil) _ fs.Lister = (*CloudHypervisor)(nil) + _ hypervisor.Migrator = (*CloudHypervisor)(nil) _ vfio.Attacher = (*CloudHypervisor)(nil) _ vfio.Lister = (*CloudHypervisor)(nil) ) diff --git a/hypervisor/cloudhypervisor/direct.go b/hypervisor/cloudhypervisor/direct.go index 8849df2c..5c81f162 100644 --- a/hypervisor/cloudhypervisor/direct.go +++ b/hypervisor/cloudhypervisor/direct.go @@ -12,19 +12,7 @@ import ( ) func (ch *CloudHypervisor) DirectClone(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) { - // config.json is copied verbatim, so re-parsing the runDir copy would decode identical bytes. - var srcCfg *chVMConfig - spec := hypervisor.CloneSpec{ - VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, - AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { - return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg) - }, - } - return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { - var err error - srcCfg, err = cloneSnapshotFiles(ctx, dstDir, srcDir) - return err - }) + return ch.directClone(ctx, vmID, vmCfg, net, snapshotConfig, srcDir, false) } func (ch *CloudHypervisor) DirectRestore(ctx context.Context, vmRef string, vmCfg *types.VMConfig, srcDir, sourceSnapshotID string) (*types.VM, error) { @@ -51,6 +39,26 @@ func (ch *CloudHypervisor) DirectRestore(ctx context.Context, vmRef string, vmCf }) } +func (ch *CloudHypervisor) directClone(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string, migrate bool) (*types.VM, error) { + // config.json is copied verbatim, so re-parsing the runDir copy would decode identical bytes. + var srcCfg *chVMConfig + spec := hypervisor.CloneSpec{ + VMCfg: vmCfg, Net: net, SnapshotConfig: snapshotConfig, + AfterExtract: func(ctx context.Context, rec *hypervisor.VMRecord, vmCfg *types.VMConfig, net types.NetSetup, sourceSnapshotID string) (*types.VM, error) { + return ch.cloneAfterExtractParsed(ctx, rec, vmCfg, net, sourceSnapshotID, srcCfg, migrate) + }, + } + return ch.DirectCloneBase(ctx, vmID, spec, srcDir, func(dstDir, srcDir string) error { + var err error + if migrate { + srcCfg, err = cloneSeriesFiles(ctx, dstDir, srcDir) + } else { + srcCfg, err = cloneSnapshotFiles(ctx, dstDir, srcDir) + } + return err + }) +} + func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig, error) { chCfg, err := parseCHConfig(filepath.Join(srcDir, configJSONName)) if err != nil { @@ -63,6 +71,8 @@ func cloneSnapshotFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig return hypervisor.SnapshotFileMemory case cowFiles[name]: return hypervisor.SnapshotFileCOW + case hypervisor.IsMigrateDeltaFile(name): + return hypervisor.SnapshotFileSkip default: return hypervisor.SnapshotFileMeta } diff --git a/hypervisor/cloudhypervisor/extend.go b/hypervisor/cloudhypervisor/extend.go index ff8b93cc..64969381 100644 --- a/hypervisor/cloudhypervisor/extend.go +++ b/hypervisor/cloudhypervisor/extend.go @@ -341,7 +341,7 @@ func convergeOrphanedPause(ctx context.Context, hc *http.Client, vmID string) (* return info, nil } log.WithFunc("cloudhypervisor.convergeOrphanedPause"). - Warnf(ctx, "vm %s is paused with no capture in flight (interrupted snapshot or hibernate), resuming", vmID) + Warnf(ctx, "vm %s is paused with no capture in flight, resuming", vmID) if err := resumeVM(ctx, hc); err != nil { return nil, fmt.Errorf("resume orphaned pause: %w", err) } diff --git a/hypervisor/cloudhypervisor/migrate.go b/hypervisor/cloudhypervisor/migrate.go new file mode 100644 index 00000000..4d8fe146 --- /dev/null +++ b/hypervisor/cloudhypervisor/migrate.go @@ -0,0 +1,75 @@ +package cloudhypervisor + +import ( + "context" + "fmt" + "maps" + "net/http" + "os" + "slices" + "strings" + + "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/types" +) + +const ( + snapshotTypeDiff = "diff" + memoryDiffPrefix = "memory-ranges.diff." +) + +func (ch *CloudHypervisor) MigrateCapture(ctx context.Context, ref string, req hypervisor.MigrateRequest) (*hypervisor.MigrateRound, error) { + return ch.MigrateCaptureSequence(ctx, ref, req, ch.migrateSpec(ctx)) +} + +func (ch *CloudHypervisor) MigrateAbort(ctx context.Context, ref, series string) error { + return ch.MigrateAbortSequence(ctx, ref, series, ch.migrateSpec(ctx)) +} + +func (ch *CloudHypervisor) MigrateReceive(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) { + return ch.directClone(ctx, vmID, vmCfg, net, snapshotConfig, srcDir, true) +} + +func (ch *CloudHypervisor) migrateSpec(ctx context.Context) hypervisor.MigrateSpec { + spec := ch.snapshotSpec(ctx) + spec.Capture = func(_ *hypervisor.VMRecord, hc *http.Client, dir string) error { + if err := snapshotVM(ctx, hc, dir, snapshotTypeDiff); err != nil { + return fmt.Errorf("snapshot: %w", err) + } + return nil + } + return hypervisor.MigrateSpec{ + SnapshotSpec: spec, + EndSeries: func(_ *hypervisor.VMRecord, hc *http.Client) error { + return endSnapshotSeries(context.WithoutCancel(ctx), hc) + }, + Rounds: countMemoryDiffs, + } +} + +func cloneSeriesFiles(ctx context.Context, dstDir, srcDir string) (*chVMConfig, error) { + chCfg, err := cloneSnapshotFiles(ctx, dstDir, srcDir) + if err != nil { + return nil, err + } + rounds, err := countMemoryDiffs(srcDir) + if err != nil { + return nil, err + } + disks := slices.Collect(maps.Keys(identifyCOWFiles(chCfg))) + return chCfg, hypervisor.ApplyDiskDeltas(ctx, dstDir, srcDir, disks, rounds) +} + +func countMemoryDiffs(dir string) (int, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return 0, fmt.Errorf("read series dir: %w", err) + } + rounds := 0 + for _, e := range entries { + if strings.HasPrefix(e.Name(), memoryDiffPrefix) { + rounds++ + } + } + return rounds, nil +} diff --git a/hypervisor/cloudhypervisor/migrate_test.go b/hypervisor/cloudhypervisor/migrate_test.go new file mode 100644 index 00000000..c6c64278 --- /dev/null +++ b/hypervisor/cloudhypervisor/migrate_test.go @@ -0,0 +1,128 @@ +package cloudhypervisor + +import ( + "encoding/json/v2" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + + "github.com/cocoonstack/cocoon/config" + "github.com/cocoonstack/cocoon/hypervisor" + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +func TestSnapshotVMSendsTheType(t *testing.T) { + tests := []struct { + name string + snapshotType string + want string + }{ + {name: "plain snapshot omits it", want: ""}, + {name: "diff", snapshotType: snapshotTypeDiff, want: "diff"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got map[string]string + sock := serveCHHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.UnmarshalRead(r.Body, &got) + w.WriteHeader(http.StatusNoContent) + })) + if err := snapshotVM(t.Context(), utils.NewSocketHTTPClient(sock), "/d", tt.snapshotType); err != nil { + t.Fatalf("snapshotVM: %v", err) + } + if got["destination_url"] != "file:///d" || got["snapshot_type"] != tt.want { + t.Errorf("payload = %v", got) + } + }) + } +} + +func TestEndSnapshotSeriesCallsTheEndpoint(t *testing.T) { + var path, method string + sock := serveCHHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path, method = r.URL.Path, r.Method + w.WriteHeader(http.StatusNoContent) + })) + if err := endSnapshotSeries(t.Context(), utils.NewSocketHTTPClient(sock)); err != nil { + t.Fatalf("endSnapshotSeries: %v", err) + } + if method != http.MethodPut || !strings.HasSuffix(path, "/vm.end-snapshot-series") { + t.Errorf("%s %s", method, path) + } +} + +func TestRestoreAndResumeCloneKeepPausedSkipsResume(t *testing.T) { + var ( + mu sync.Mutex + paths []string + ) + sock := serveCHHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, filepath.Base(r.URL.Path)) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + scopeParent := t.TempDir() + if err := os.Mkdir(filepath.Join(scopeParent, "vm-T.scope"), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + ch := &CloudHypervisor{ + Backend: &hypervisor.Backend{Conf: tmpScopeConf{Config: NewConfig(&config.Config{}), parent: scopeParent}}, + conf: NewConfig(&config.Config{}), + } + if err := ch.restoreAndResumeClone(t.Context(), 0, sock, t.TempDir(), &cloneResumeOpts{ + vmID: "T", + vmCfg: &types.VMConfig{CPU: 2}, + directBoot: true, + snapshotCfg: &chVMConfig{}, + keepPaused: true, + }); err != nil { + t.Fatalf("restoreAndResumeClone: %v", err) + } + mu.Lock() + defer mu.Unlock() + if slices.Contains(paths, "vm.resume") { + t.Errorf("calls = %v, want no vm.resume", paths) + } +} + +func TestCloneSeriesFilesLeavesDeltasBehind(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + writeMigrateFile(t, src, configJSONName, `{"disks":[{"path":"/run/cow.raw"}]}`) + for _, name := range []string{"cow.raw", memoryRangeFile + "s", "cow.raw.ref", stateJSONName} { + writeMigrateFile(t, src, name, "x") + } + if _, err := cloneSeriesFiles(t.Context(), dst, src); err != nil { + t.Fatalf("cloneSeriesFiles: %v", err) + } + if utils.FileExists(filepath.Join(dst, "cow.raw.ref")) { + t.Error("the reference reached the run dir") + } + for _, name := range []string{"cow.raw", memoryRangeFile + "s", stateJSONName} { + if !utils.FileExists(filepath.Join(dst, name)) { + t.Errorf("%s missing from the run dir", name) + } + } +} + +func TestCloneSeriesFilesFailsWithoutTheDiskDeltaOfARound(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + writeMigrateFile(t, src, configJSONName, `{"disks":[{"path":"/run/cow.raw"}]}`) + writeMigrateFile(t, src, "cow.raw", "x") + writeMigrateFile(t, src, memoryDiffPrefix+"1", "x") + if _, err := cloneSeriesFiles(t.Context(), dst, src); err == nil { + t.Fatal("cloneSeriesFiles accepted a round without its disk delta") + } +} + +func writeMigrateFile(t *testing.T, dir, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } +} diff --git a/hypervisor/cloudhypervisor/snapshot.go b/hypervisor/cloudhypervisor/snapshot.go index eadda364..d90d73eb 100644 --- a/hypervisor/cloudhypervisor/snapshot.go +++ b/hypervisor/cloudhypervisor/snapshot.go @@ -53,7 +53,7 @@ func (ch *CloudHypervisor) snapshotSpec(ctx context.Context) hypervisor.Snapshot }, Resume: func(_ *hypervisor.VMRecord, hc *http.Client) error { return resumeVM(context.WithoutCancel(ctx), hc) }, Capture: func(rec *hypervisor.VMRecord, hc *http.Client, tmpDir string) error { - if err := snapshotVM(ctx, hc, tmpDir); err != nil { + if err := snapshotVM(ctx, hc, tmpDir, ""); err != nil { return fmt.Errorf("snapshot: %w", err) } return hypervisor.CopyWritableDisks(ctx, tmpDir, rec) diff --git a/hypervisor/cloudhypervisor/utils.go b/hypervisor/cloudhypervisor/utils.go index 1c562e89..e7e04f84 100644 --- a/hypervisor/cloudhypervisor/utils.go +++ b/hypervisor/cloudhypervisor/utils.go @@ -48,6 +48,11 @@ var runtimeFiles = []string{hypervisor.APISocketName, pidFileName, hypervisor.Co type chMemoryRestoreMode string +type chSnapshotConfig struct { + DestinationURL string `json:"destination_url"` + SnapshotType string `json:"snapshot_type,omitempty"` +} + type chRestoreConfig struct { SourceURL string `json:"source_url"` MemoryRestoreMode chMemoryRestoreMode `json:"memory_restore_mode,omitempty"` @@ -134,10 +139,16 @@ func isAlreadyInStateError(err error, state string) bool { return strings.Contains(ae.Message, fmt.Sprintf("Invalid transition: InvalidStateTransition(%s, %s)", state, state)) } -func snapshotVM(ctx context.Context, hc *http.Client, destDir string) error { +func snapshotVM(ctx context.Context, hc *http.Client, destDir, snapshotType string) error { hc.Timeout = hypervisor.VMMemTransferTimeout defer func() { hc.Timeout = utils.HTTPTimeout }() - return vmPutJSON(ctx, hc, "vm.snapshot", "snapshot request", map[string]string{"destination_url": "file://" + destDir}, http.StatusNoContent) + cfg := chSnapshotConfig{DestinationURL: "file://" + destDir, SnapshotType: snapshotType} + return vmPutJSON(ctx, hc, "vm.snapshot", "snapshot request", cfg, http.StatusNoContent) +} + +func endSnapshotSeries(ctx context.Context, hc *http.Client) error { + _, err := vmAPIOnce(ctx, hc, "vm.end-snapshot-series") + return err } func restoreVM(ctx context.Context, hc *http.Client, sourceDir, restoreMode string) error { diff --git a/hypervisor/diskdelta.go b/hypervisor/diskdelta.go new file mode 100644 index 00000000..8c6b9a34 --- /dev/null +++ b/hypervisor/diskdelta.go @@ -0,0 +1,202 @@ +package hypervisor + +import ( + "context" + "encoding/binary" + "encoding/json/v2" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strconv" + + "github.com/cocoonstack/cocoon/utils" +) + +const diskDeltaInfix = ".delta." + +// diskDelta heads a delta file; the bytes of each non-zero range follow in order. +type diskDelta struct { + Size int64 `json:"size"` + Ranges []deltaRange `json:"ranges"` +} + +type deltaRange struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Zero bool `json:"zero,omitzero"` +} + +type mapping struct { + physical int64 + unwritten bool + mapped bool +} + +// ApplyDiskDeltas replays rounds 1..rounds of each named disk from srcDir onto its copy in dstDir. +func ApplyDiskDeltas(ctx context.Context, dstDir, srcDir string, disks []string, rounds int) error { + _, err := utils.Map(ctx, disks, func(_ context.Context, _ int, name string) (struct{}, error) { + for n := 1; n <= rounds; n++ { + if err := applyDiskDelta(filepath.Join(dstDir, name), filepath.Join(srcDir, diskDeltaName(name, n))); err != nil { + return struct{}{}, fmt.Errorf("apply %s round %d: %w", name, n, err) + } + } + return struct{}{}, nil + }) + return err +} + +func diskDeltaName(disk string, round int) string { + return disk + diskDeltaInfix + strconv.Itoa(round) +} + +// writeDiskDelta records how src differs from ref; the two must share extents through reflink, so an extent both map holds the same bytes. +func writeDiskDelta(dst, ref, src string) error { + srcFile, err := os.Open(src) //nolint:gosec + if err != nil { + return fmt.Errorf("open delta source: %w", err) + } + defer srcFile.Close() //nolint:errcheck + fi, err := srcFile.Stat() + if err != nil { + return fmt.Errorf("stat delta source: %w", err) + } + srcExtents, err := utils.FileExtents(srcFile) + if err != nil { + return err + } + refFile, err := os.Open(ref) //nolint:gosec + if err != nil { + return fmt.Errorf("open delta reference: %w", err) + } + defer refFile.Close() //nolint:errcheck + refExtents, err := utils.FileExtents(refFile) + if err != nil { + return err + } + header := diskDelta{Size: fi.Size(), Ranges: diffExtents(refExtents, srcExtents, fi.Size())} + return writeDeltaFile(dst, &header, srcFile) +} + +// diffExtents returns the ranges below size where cur and ref map different storage; a range cur reads as zeros is marked Zero. +func diffExtents(ref, cur []utils.Extent, size int64) []deltaRange { + var out []deltaRange + for pos := int64(0); pos < size; { + refAt, refEnd := extentAt(&ref, pos) + curAt, curEnd := extentAt(&cur, pos) + end := min(refEnd, curEnd, size) + if refAt != curAt { + zero := !curAt.mapped || curAt.unwritten + if n := len(out) - 1; n >= 0 && out[n].Zero == zero && out[n].Offset+out[n].Length == pos { + out[n].Length += end - pos + } else { + out = append(out, deltaRange{Offset: pos, Length: end - pos, Zero: zero}) + } + } + pos = end + } + return out +} + +// extentAt advances *extents past pos and returns the mapping at pos plus the offset where it stops holding. +func extentAt(extents *[]utils.Extent, pos int64) (mapping, int64) { + for len(*extents) > 0 && (*extents)[0].Logical+(*extents)[0].Length <= pos { + *extents = (*extents)[1:] + } + if len(*extents) == 0 { + return mapping{}, math.MaxInt64 + } + e := (*extents)[0] + if pos < e.Logical { + return mapping{}, e.Logical + } + return mapping{physical: e.Physical + pos - e.Logical, unwritten: e.Unwritten, mapped: true}, e.Logical + e.Length +} + +func writeDeltaFile(dst string, header *diskDelta, src *os.File) (err error) { + raw, err := json.Marshal(header) + if err != nil { + return fmt.Errorf("encode delta header: %w", err) + } + tmp := dst + ".tmp" + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) //nolint:gosec + if err != nil { + return fmt.Errorf("create delta: %w", err) + } + defer func() { + err = errors.Join(err, out.Close()) + if err != nil { + _ = os.Remove(tmp) + return + } + err = os.Rename(tmp, dst) + }() + if err = binary.Write(out, binary.BigEndian, int64(len(raw))); err != nil { + return fmt.Errorf("write delta header: %w", err) + } + if _, err = out.Write(raw); err != nil { + return fmt.Errorf("write delta header: %w", err) + } + for _, r := range header.Ranges { + if r.Zero { + continue + } + if _, err = src.Seek(r.Offset, io.SeekStart); err != nil { + return fmt.Errorf("seek delta source to %d: %w", r.Offset, err) + } + if _, err = io.CopyN(out, src, r.Length); err != nil { + return fmt.Errorf("copy range at %d len %d: %w", r.Offset, r.Length, err) + } + } + return nil +} + +func applyDiskDelta(dst, delta string) error { + in, err := os.Open(delta) //nolint:gosec + if err != nil { + return err + } + defer in.Close() //nolint:errcheck + header, err := readDeltaHeader(in) + if err != nil { + return err + } + out, err := os.OpenFile(dst, os.O_WRONLY, 0) //nolint:gosec + if err != nil { + return err + } + defer out.Close() //nolint:errcheck + if err = out.Truncate(header.Size); err != nil { + return fmt.Errorf("resize %s: %w", dst, err) + } + for _, r := range header.Ranges { + if r.Zero { + if err = utils.PunchHole(out, r.Offset, r.Length); err != nil { + return err + } + continue + } + if _, err = out.Seek(r.Offset, io.SeekStart); err != nil { + return fmt.Errorf("seek %s to %d: %w", dst, r.Offset, err) + } + if _, err = io.CopyN(out, in, r.Length); err != nil { + return fmt.Errorf("write range at %d len %d: %w", r.Offset, r.Length, err) + } + } + return nil +} + +// readDeltaHeader leaves f at the first data byte. +func readDeltaHeader(f *os.File) (*diskDelta, error) { + var length int64 + if err := binary.Read(f, binary.BigEndian, &length); err != nil { + return nil, fmt.Errorf("read delta header length: %w", err) + } + var header diskDelta + if err := json.UnmarshalRead(io.LimitReader(f, length), &header); err != nil { + return nil, fmt.Errorf("decode delta header: %w", err) + } + return &header, nil +} diff --git a/hypervisor/diskdelta_test.go b/hypervisor/diskdelta_test.go new file mode 100644 index 00000000..e9145df7 --- /dev/null +++ b/hypervisor/diskdelta_test.go @@ -0,0 +1,197 @@ +package hypervisor + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/cocoon/utils" +) + +func TestDiffExtents(t *testing.T) { + tests := []struct { + name string + ref []utils.Extent + cur []utils.Extent + size int64 + want []deltaRange + }{ + { + name: "shared extents differ nowhere", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + size: 10, + }, + { + name: "split extent with the same storage differs nowhere", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}, {Logical: 4, Physical: 104, Length: 6}}, + size: 10, + }, + { + name: "rewritten middle", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 10}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}, {Logical: 4, Physical: 900, Length: 2}, {Logical: 6, Physical: 106, Length: 4}}, + size: 10, + want: []deltaRange{{Offset: 4, Length: 2}}, + }, + { + name: "new data in a hole", + cur: []utils.Extent{{Logical: 2, Physical: 900, Length: 2}}, + size: 10, + want: []deltaRange{{Offset: 2, Length: 2}}, + }, + { + name: "discarded data", + ref: []utils.Extent{{Logical: 2, Physical: 100, Length: 2}}, + size: 10, + want: []deltaRange{{Offset: 2, Length: 2, Zero: true}}, + }, + { + name: "unwritten extent reads as zeros", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}}, + cur: []utils.Extent{{Logical: 0, Physical: 900, Length: 4, Unwritten: true}}, + size: 4, + want: []deltaRange{{Offset: 0, Length: 4, Zero: true}}, + }, + { + name: "written over a shared unwritten extent", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 4, Unwritten: true}}, + cur: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}}, + size: 4, + want: []deltaRange{{Offset: 0, Length: 4}}, + }, + { + name: "adjacent changes of one kind merge", + ref: []utils.Extent{{Logical: 0, Physical: 100, Length: 4}}, + cur: []utils.Extent{{Logical: 0, Physical: 900, Length: 2}, {Logical: 2, Physical: 500, Length: 2}, {Logical: 4, Physical: 700, Length: 2}}, + size: 6, + want: []deltaRange{{Offset: 0, Length: 6}}, + }, + { + name: "extent past the end of the file is cut", + cur: []utils.Extent{{Logical: 0, Physical: 900, Length: 100}}, + size: 10, + want: []deltaRange{{Offset: 0, Length: 10}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := diffExtents(tt.ref, tt.cur, tt.size); !slices.Equal(got, tt.want) { + t.Errorf("got %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestDiskDeltaRoundTrip(t *testing.T) { + dir := t.TempDir() + ref := filepath.Join(dir, "ref") + src := filepath.Join(dir, "src") + writeSparse(t, ref, 1<<20, map[int64]string{0: "base", 1 << 19: "kept"}) + writeSparse(t, src, 2<<20, map[int64]string{0: "next", 1 << 19: "kept", 3 << 19: "grown"}) + delta := filepath.Join(dir, diskDeltaName("disk", 1)) + if err := writeDiskDelta(delta, ref, src); err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("writeDiskDelta: %v", err) + } + if err := applyDiskDelta(ref, delta); err != nil { + t.Fatalf("applyDiskDelta: %v", err) + } + got, err := os.ReadFile(ref) + if err != nil { + t.Fatalf("read: %v", err) + } + want, err := os.ReadFile(src) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("applied disk differs from the source") + } +} + +func TestApplyDiskDeltaPunchesZeroRanges(t *testing.T) { + dir := t.TempDir() + dst := filepath.Join(dir, "dst") + writeSparse(t, dst, 1<<20, map[int64]string{0: "stale"}) + src, err := os.Create(filepath.Join(dir, "src")) + if err != nil { + t.Fatalf("create: %v", err) + } + defer src.Close() + delta := filepath.Join(dir, "delta") + if err = writeDeltaFile(delta, &diskDelta{Size: 1 << 20, Ranges: []deltaRange{{Offset: 0, Length: 4096, Zero: true}}}, src); err != nil { + t.Fatalf("writeDeltaFile: %v", err) + } + if err = applyDiskDelta(dst, delta); err != nil { + if errors.Is(err, errors.ErrUnsupported) { + t.Skip("needs fallocate") + } + t.Fatalf("applyDiskDelta: %v", err) + } + got, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got[:5], make([]byte, 5)) { + t.Errorf("range reads %q, want zeros", got[:5]) + } +} + +func TestApplyDiskDeltaRejectsATruncatedFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + writeSparse(t, src, 4096, map[int64]string{0: "data"}) + srcFile, err := os.Open(src) + if err != nil { + t.Fatalf("open: %v", err) + } + defer srcFile.Close() + delta := filepath.Join(dir, "delta") + if err = writeDeltaFile(delta, &diskDelta{Size: 4096, Ranges: []deltaRange{{Offset: 0, Length: 4096}}}, srcFile); err != nil { + t.Fatalf("writeDeltaFile: %v", err) + } + fi, err := os.Stat(delta) + if err != nil { + t.Fatalf("stat: %v", err) + } + if err = os.Truncate(delta, fi.Size()-1); err != nil { + t.Fatalf("truncate: %v", err) + } + dst := filepath.Join(dir, "dst") + writeSparse(t, dst, 4096, nil) + if err = applyDiskDelta(dst, delta); !errors.Is(err, io.EOF) { + t.Errorf("applyDiskDelta: %v, want EOF", err) + } +} + +func TestApplyDiskDeltasFailsOnAMissingRound(t *testing.T) { + dst, src := t.TempDir(), t.TempDir() + writeSparse(t, filepath.Join(dst, "disk"), 4096, nil) + err := ApplyDiskDeltas(t.Context(), dst, src, []string{"disk"}, 1) + if !errors.Is(err, os.ErrNotExist) { + t.Errorf("ApplyDiskDeltas: %v, want not exist", err) + } +} + +func writeSparse(t *testing.T, path string, size int64, chunks map[int64]string) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer f.Close() + if err = f.Truncate(size); err != nil { + t.Fatalf("truncate: %v", err) + } + for off, data := range chunks { + if _, err = f.WriteAt([]byte(data), off); err != nil { + t.Fatalf("write: %v", err) + } + } +} diff --git a/hypervisor/disks.go b/hypervisor/disks.go index 098d9579..7314c1a2 100644 --- a/hypervisor/disks.go +++ b/hypervisor/disks.go @@ -85,16 +85,10 @@ func RecordedCOWPath(rec *VMRecord) (string, error) { // CopyWritableDisks copies concurrently; durability is paid at persist. func CopyWritableDisks(ctx context.Context, dstDir string, rec *VMRecord) error { - cowPath, err := RecordedCOWPath(rec) + pairs, err := writableDiskPairs(dstDir, rec) if err != nil { return err } - pairs := [][2]string{{filepath.Join(dstDir, filepath.Base(cowPath)), cowPath}} - for _, sc := range rec.StorageConfigs { - if sc.Role == types.StorageRoleData { - pairs = append(pairs, [2]string{filepath.Join(dstDir, DataDiskBaseName(sc.Serial)), sc.Path}) - } - } return copyPairs(ctx, pairs) } @@ -127,6 +121,21 @@ func PrepareDataDisks(ctx context.Context, baseDir string, specs []types.DataDis return out, nil } +// writableDiskPairs returns {copy under dstDir, live disk} for the COW and every data disk. +func writableDiskPairs(dstDir string, rec *VMRecord) ([][2]string, error) { + cowPath, err := RecordedCOWPath(rec) + if err != nil { + return nil, err + } + pairs := [][2]string{{filepath.Join(dstDir, filepath.Base(cowPath)), cowPath}} + for _, sc := range rec.StorageConfigs { + if sc.Role == types.StorageRoleData { + pairs = append(pairs, [2]string{filepath.Join(dstDir, DataDiskBaseName(sc.Serial)), sc.Path}) + } + } + return pairs, nil +} + func copyPairs(ctx context.Context, pairs [][2]string) error { _, err := utils.Map(ctx, pairs, func(ctx context.Context, _ int, p [2]string) (struct{}, error) { if err := utils.ReflinkCopy(ctx, p[0], p[1], utils.NoSync); err != nil { diff --git a/hypervisor/firecracker/clone.go b/hypervisor/firecracker/clone.go index f85dca26..2eb0c9e2 100644 --- a/hypervisor/firecracker/clone.go +++ b/hypervisor/firecracker/clone.go @@ -18,6 +18,7 @@ import ( "github.com/cocoonstack/cocoon/hypervisor" "github.com/cocoonstack/cocoon/lock/flock" "github.com/cocoonstack/cocoon/lock/vmlock" + "github.com/cocoonstack/cocoon/metering" "github.com/cocoonstack/cocoon/snapshot" "github.com/cocoonstack/cocoon/types" "github.com/cocoonstack/cocoon/utils" @@ -109,7 +110,7 @@ func (fc *Firecracker) cloneAfterExtract(ctx context.Context, rec *hypervisor.VM storageConfigs = append(storageConfigs, dataDisks...) info := fc.RunningCloneRecord(rec, vmCfg, storageConfigs, net) - if err := fc.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID); err != nil { + if err := fc.FinalizeClone(ctx, vmID, info, bootCfg, blobIDs, sourceSnapshotID, metering.ReasonClone); err != nil { leaseControl.close() fc.AbortLaunch(ctx, pid, sockPath, runDir, runtimeFiles) return nil, fmt.Errorf("finalize VM record: %w", err) diff --git a/hypervisor/firecracker/migrate_test.go b/hypervisor/firecracker/migrate_test.go new file mode 100644 index 00000000..8b369417 --- /dev/null +++ b/hypervisor/firecracker/migrate_test.go @@ -0,0 +1,13 @@ +package firecracker + +import ( + "testing" + + "github.com/cocoonstack/cocoon/hypervisor" +) + +func TestFirecrackerIsNotAMigrator(t *testing.T) { + if _, ok := any(&Firecracker{}).(hypervisor.Migrator); ok { + t.Error("Firecracker claims live migration; vm migrate would stop returning ErrMigrateUnsupported") + } +} diff --git a/hypervisor/hypervisor.go b/hypervisor/hypervisor.go index e3a16d8c..12812ed3 100644 --- a/hypervisor/hypervisor.go +++ b/hypervisor/hypervisor.go @@ -18,6 +18,8 @@ var ( ErrAmbiguous = errors.New("vm ref resolves to multiple backends") ErrHotAttached = errors.New("detach before snapshot or hibernate") + + ErrMigrateUnsupported = errors.New("backend does not support live migration") ) // Hypervisor manages VM lifecycle. diff --git a/hypervisor/migrate.go b/hypervisor/migrate.go new file mode 100644 index 00000000..c57b2742 --- /dev/null +++ b/hypervisor/migrate.go @@ -0,0 +1,327 @@ +package hypervisor + +import ( + "context" + "errors" + "fmt" + "maps" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/cocoonstack/cocoon/snapshot" + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +const ( + diskRefSuffix = ".ref" + + seriesDirPrefix = captureDirPrefix + "migrate-" +) + +// Migrator captures a running VM in rounds and restores the result on another node. +type Migrator interface { + // MigrateCapture starts a new series when req.Series is empty and adds a round to it otherwise. + MigrateCapture(ctx context.Context, ref string, req MigrateRequest) (*MigrateRound, error) + // MigrateReceive restores a transferred series into a new VM and leaves its guest paused; srcDir stays read-only. + MigrateReceive(ctx context.Context, vmID string, vmCfg *types.VMConfig, net types.NetSetup, snapshotConfig *types.SnapshotConfig, srcDir string) (*types.VM, error) + // MigrateAbort ends the series, resumes the guest and deletes the series directory. + MigrateAbort(ctx context.Context, ref, series string) error +} + +// MigrateRequest selects the round to capture; Final leaves the guest paused. +type MigrateRequest struct { + Series string + Final bool +} + +// MigrateRound reports one captured round; round 0 is the baseline and Files lists what the round wrote. +type MigrateRound struct { + Series string `json:"series"` + Dir string `json:"dir"` + Round int `json:"round"` + Paused bool `json:"paused"` + Files []MigrateFile `json:"files"` +} + +type MigrateFile struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +// MigrateSpec carries backend hooks for the migrate sequences. +type MigrateSpec struct { + SnapshotSpec + EndSeries TransitionHook + Rounds func(dir string) (int, error) +} + +type roundSteps struct { + series string + round int + final bool + before func(hc *http.Client) error + inPause func() error + afterPause func() error +} + +// MigrateCaptureSequence runs one round; any failure ends the series, resumes the guest and deletes the directory. +func (b *Backend) MigrateCaptureSequence(ctx context.Context, ref string, req MigrateRequest, spec MigrateSpec) (*MigrateRound, error) { + vmID, rec, unlock, err := b.lockForCapture(ctx, ref) + if err != nil { + return nil, err + } + defer unlock() + var round *MigrateRound + if req.Series == "" { + round, err = b.captureBaseline(ctx, &rec, spec) + } else { + round, err = b.captureDelta(ctx, &rec, req, spec) + } + if err != nil { + return nil, fmt.Errorf("migrate capture VM %s: %w", vmID, err) + } + return round, nil +} + +// MigrateAbortSequence acts only while the series directory exists. +func (b *Backend) MigrateAbortSequence(ctx context.Context, ref, series string, spec MigrateSpec) error { + vmID, rec, unlock, err := b.lockForCapture(ctx, ref) + if err != nil { + return err + } + defer unlock() + dir, err := seriesDir(&rec, series) + if err != nil { + return err + } + if !utils.FileExists(dir) { + return nil + } + hc := utils.NewSocketHTTPClient(SocketPath(rec.RunDir)) + runErr := b.WithRunningVM(ctx, &rec, func(_ int) error { + return errors.Join(spec.EndSeries(&rec, hc), resumeAfterCapture(spec.Resume, &rec, hc)) + }) + if runErr != nil && !errors.Is(runErr, ErrNotRunning) { + return fmt.Errorf("migrate abort VM %s: %w", vmID, runErr) + } + return os.RemoveAll(dir) +} + +func (b *Backend) captureBaseline(ctx context.Context, rec *VMRecord, spec MigrateSpec) (*MigrateRound, error) { + // A fallback copy of the disks would run inside the pause and share no extents with the live disk. + if err := utils.CheckReflink(rec.RunDir); err != nil { + return nil, fmt.Errorf("live migration needs a reflink filesystem under the run dir: %w", err) + } + if errs := utils.RemoveMatching(ctx, rec.RunDir, isSeriesDir); len(errs) > 0 { + return nil, errors.Join(errs...) + } + dir, err := os.MkdirTemp(rec.RunDir, seriesDirPrefix) + if err != nil { + return nil, fmt.Errorf("create series dir: %w", err) + } + series := strings.TrimPrefix(filepath.Base(dir), seriesDirPrefix) + return b.captureRound(ctx, rec, spec, dir, roundSteps{ + series: series, + before: func(hc *http.Client) error { return spec.EndSeries(rec, hc) }, + inPause: func() error { + return CopyWritableDisks(ctx, dir, rec) + }, + afterPause: func() error { + if err := saveCaptureMeta(rec, spec.SnapshotSpec, dir); err != nil { + return err + } + return b.writeSeriesEnvelope(rec, series, dir) + }, + }) +} + +func (b *Backend) captureDelta(ctx context.Context, rec *VMRecord, req MigrateRequest, spec MigrateSpec) (*MigrateRound, error) { + dir, err := seriesDir(rec, req.Series) + if err != nil { + return nil, err + } + pairs, err := writableDiskPairs(dir, rec) + if err != nil { + return nil, err + } + number := nextRound(pairs[0][0]) + steps := roundSteps{ + series: req.Series, + round: number, + final: req.Final, + before: func(*http.Client) error { + if err := checkRoundsInStep(spec, dir, pairs, number); err != nil { + return err + } + return b.writeSeriesEnvelope(rec, req.Series, dir) + }, + inPause: func() error { + if req.Final { + return nil + } + return reflinkNextRefs(ctx, pairs) + }, + afterPause: func() error { + _, mapErr := utils.Map(ctx, pairs, func(_ context.Context, _ int, p [2]string) (struct{}, error) { + return struct{}{}, writeRoundDelta(p, number, req.Final) + }) + return mapErr + }, + } + return b.captureRound(ctx, rec, spec, dir, steps) +} + +func (b *Backend) captureRound(ctx context.Context, rec *VMRecord, spec MigrateSpec, dir string, steps roundSteps) (*MigrateRound, error) { + before, err := stampFiles(dir) + if err != nil { + return nil, errors.Join(err, os.RemoveAll(dir)) + } + hc := utils.NewSocketHTTPClient(SocketPath(rec.RunDir)) + run := func() error { + if beforeErr := steps.before(hc); beforeErr != nil { + return beforeErr + } + if pauseErr := spec.Pause(rec, hc); pauseErr != nil { + return fmt.Errorf("pause: %w", pauseErr) + } + if captureErr := spec.Capture(rec, hc, dir); captureErr != nil { + return captureErr + } + if stepErr := steps.inPause(); stepErr != nil { + return stepErr + } + if !steps.final { + if resumeErr := resumeAfterCapture(spec.Resume, rec, hc); resumeErr != nil { + return resumeErr + } + } + return steps.afterPause() + } + if err = b.WithRunningVM(ctx, rec, func(_ int) error { + if runErr := run(); runErr != nil { + return errors.Join(runErr, spec.EndSeries(rec, hc), resumeAfterCapture(spec.Resume, rec, hc)) + } + return nil + }); err != nil { + return nil, errors.Join(err, os.RemoveAll(dir)) + } + files, err := changedFiles(dir, before) + if err != nil { + return nil, err + } + return &MigrateRound{Series: steps.series, Dir: dir, Round: steps.round, Paused: steps.final, Files: files}, nil +} + +func (b *Backend) writeSeriesEnvelope(rec *VMRecord, series, dir string) error { + return snapshot.WriteSnapshotEnvelope(dir, *b.BuildSnapshotConfig(series, rec), nil, nil) +} + +// IsMigrateDeltaFile reports a disk delta or reference a clone of a series must leave behind. +func IsMigrateDeltaFile(name string) bool { + return strings.HasSuffix(name, diskRefSuffix) || strings.Contains(name, diskDeltaInfix) +} + +func seriesDir(rec *VMRecord, series string) (string, error) { + if !filepath.IsLocal(series) || strings.ContainsRune(series, filepath.Separator) { + return "", fmt.Errorf("invalid series id %q", series) + } + return filepath.Join(rec.RunDir, seriesDirPrefix+series), nil +} + +func isSeriesDir(e os.DirEntry) bool { + return e.IsDir() && strings.HasPrefix(e.Name(), seriesDirPrefix) +} + +func checkRoundsInStep(spec MigrateSpec, dir string, pairs [][2]string, round int) error { + rounds, err := spec.Rounds(dir) + if err != nil { + return err + } + if rounds != round-1 { + return fmt.Errorf("series holds %d guest rounds and %d disk rounds: an earlier capture died, start a new series", rounds, round-1) + } + for _, p := range pairs { + if n := nextRound(p[0]); n != round { + return fmt.Errorf("%s is at round %d, want %d: an earlier capture died, start a new series", filepath.Base(p[0]), n-1, round-1) + } + if utils.FileExists(nextRefPath(p[0])) { + return fmt.Errorf("%s has an unmoved reference: an earlier capture died, start a new series", filepath.Base(p[0])) + } + } + return nil +} + +func nextRound(disk string) int { + n := 1 + for utils.FileExists(filepath.Join(filepath.Dir(disk), diskDeltaName(filepath.Base(disk), n))) { + n++ + } + return n +} + +// reflinkNextRefs freezes each disk in the pause window so the delta can be written after the guest resumes. +func reflinkNextRefs(ctx context.Context, pairs [][2]string) error { + refs := make([][2]string, len(pairs)) + for i, p := range pairs { + refs[i] = [2]string{nextRefPath(p[0]), p[1]} + } + return copyPairs(ctx, refs) +} + +func writeRoundDelta(pair [2]string, round int, final bool) error { + copyPath, live := pair[0], pair[1] + ref := copyPath + diskRefSuffix + if !utils.FileExists(ref) { + ref = copyPath + } + dst := filepath.Join(filepath.Dir(copyPath), diskDeltaName(filepath.Base(copyPath), round)) + if final { + return writeDiskDelta(dst, ref, live) + } + next := nextRefPath(copyPath) + if err := writeDiskDelta(dst, ref, next); err != nil { + return err + } + return os.Rename(next, copyPath+diskRefSuffix) +} + +func nextRefPath(copyPath string) string { + return copyPath + ".next" + diskRefSuffix +} + +func stampFiles(dir string) (map[string]os.FileInfo, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read series dir: %w", err) + } + stamps := make(map[string]os.FileInfo, len(entries)) + for _, e := range entries { + info, err := e.Info() + if err != nil { + return nil, fmt.Errorf("stat %s: %w", e.Name(), err) + } + if info.Mode().IsRegular() && !strings.HasSuffix(e.Name(), diskRefSuffix) { + stamps[e.Name()] = info + } + } + return stamps, nil +} + +func changedFiles(dir string, before map[string]os.FileInfo) ([]MigrateFile, error) { + after, err := stampFiles(dir) + if err != nil { + return nil, err + } + files := []MigrateFile{} + for _, name := range slices.Sorted(maps.Keys(after)) { + info := after[name] + if old, ok := before[name]; !ok || old.Size() != info.Size() || !old.ModTime().Equal(info.ModTime()) { + files = append(files, MigrateFile{Name: name, Size: info.Size()}) + } + } + return files, nil +} diff --git a/hypervisor/migrate_test.go b/hypervisor/migrate_test.go new file mode 100644 index 00000000..31ad5529 --- /dev/null +++ b/hypervisor/migrate_test.go @@ -0,0 +1,362 @@ +package hypervisor + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/cocoonstack/cocoon/snapshot" + "github.com/cocoonstack/cocoon/types" + "github.com/cocoonstack/cocoon/utils" +) + +func TestMigrateBaselineRefusesWithoutReflink(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + if utils.CheckReflink(rec.RunDir) == nil { + t.Skip("the test filesystem supports reflink") + } + calls := &migrateCalls{} + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{}, migrateStubSpec(calls)); err == nil { + t.Fatal("capture succeeded without reflink") + } + if calls.paused != 0 { + t.Error("the guest was paused before the refusal") + } + if dirs, _ := filepath.Glob(filepath.Join(rec.RunDir, seriesDirPrefix+"*")); len(dirs) != 0 { + t.Errorf("series dirs left behind: %v", dirs) + } +} + +func TestMigrateMiddleRoundResumesAndHidesTheReference(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + round, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, migrateStubSpec(calls)) + if err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("capture: %v", err) + } + if round.Round != 1 || round.Paused || calls.resumed != 1 { + t.Errorf("round %d paused %t resumed %d, want 1 false 1", round.Round, round.Paused, calls.resumed) + } + if want := []string{"cow.raw.delta.1", "mem.1", snapshot.SnapshotJSONName}; !slices.Equal(fileNames(round), want) { + t.Errorf("files = %v, want %v", fileNames(round), want) + } + if !utils.FileExists(filepath.Join(dir, "cow.raw.ref")) { + t.Error("the next round has no reference") + } + round, err = b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, migrateStubSpec(calls)) + if err != nil { + t.Fatalf("second capture: %v", err) + } + if round.Round != 2 { + t.Errorf("round = %d, want 2", round.Round) + } +} + +func TestMigrateFinalRoundLeavesTheGuestPaused(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + round, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1", Final: true}, migrateStubSpec(calls)) + if err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("capture: %v", err) + } + if !round.Paused || calls.resumed != 0 || calls.ended != 0 { + t.Errorf("paused %t resumed %d ended %d, want true 0 0", round.Paused, calls.resumed, calls.ended) + } + if utils.FileExists(filepath.Join(dir, "cow.raw.ref")) { + t.Error("the final round reflinked a reference inside the pause") + } +} + +func TestMigrateFailedRoundResumesAndDiscardsTheSeries(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + spec.Capture = func(*VMRecord, *http.Client, string) error { return errors.New("dump failed") } + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1", Final: true}, spec); err == nil || !strings.Contains(err.Error(), "dump failed") { + t.Fatalf("capture: %v, want dump failed", err) + } + if calls.ended != 1 || calls.resumed != 1 { + t.Errorf("ended %d resumed %d, want 1 1", calls.ended, calls.resumed) + } + if utils.FileExists(dir) { + t.Error("the failed series survived") + } +} + +func TestMigrateUnknownSeriesKeepsTheGuestRunning(t *testing.T) { + b, id, _ := newMigrateTestVM(t) + calls := &migrateCalls{} + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "gone"}, migrateStubSpec(calls)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("capture: %v, want not exist", err) + } + if calls.paused != 0 || calls.ended != 0 { + t.Errorf("paused %d ended %d, want 0 0", calls.paused, calls.ended) + } +} + +func TestMigrateAbortRepeatLeavesTheGuestAlone(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + for range 2 { + if err := b.MigrateAbortSequence(t.Context(), id, "s1", migrateStubSpec(calls)); err != nil { + t.Fatalf("abort: %v", err) + } + } + if calls.ended != 1 || calls.resumed != 1 { + t.Errorf("ended %d resumed %d, want 1 1", calls.ended, calls.resumed) + } + if utils.FileExists(dir) { + t.Error("the series survived the abort") + } +} + +func TestMigrateAbortOfAnOldSeriesLeavesTheGuestAlone(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + if err := b.MigrateAbortSequence(t.Context(), id, "old", migrateStubSpec(calls)); err != nil { + t.Fatalf("abort: %v", err) + } + if calls.ended != 0 || calls.resumed != 0 || !utils.FileExists(dir) { + t.Errorf("ended %d resumed %d kept %t, want 0 0 true", calls.ended, calls.resumed, utils.FileExists(dir)) + } +} + +func TestMigrateAbortKeepsTheSeriesUntilTheGuestResumes(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + spec.Resume = func(*VMRecord, *http.Client) error { return errors.New("vmm busy") } + if err := b.MigrateAbortSequence(t.Context(), id, "s1", spec); err == nil { + t.Fatal("abort hid the failure") + } + if !utils.FileExists(dir) { + t.Fatal("the series went with the guest still paused, so a retry could not resume it") + } + if err := b.MigrateAbortSequence(t.Context(), id, "s1", migrateStubSpec(calls)); err != nil { + t.Fatalf("retry: %v", err) + } + if calls.resumed != 1 || utils.FileExists(dir) { + t.Errorf("resumed %d kept %t, want 1 false", calls.resumed, utils.FileExists(dir)) + } +} + +func TestMigrateAbortResumesWhenEndingTheSeriesFails(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + spec.EndSeries = func(*VMRecord, *http.Client) error { return errors.New("stop dirty log") } + if err := b.MigrateAbortSequence(t.Context(), id, "s1", spec); err == nil { + t.Fatal("abort hid the failure") + } + if calls.resumed != 1 { + t.Errorf("resumed %d, want 1", calls.resumed) + } +} + +func TestMigrateRoundOutOfStepDiscardsTheSeries(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, dir string, spec *MigrateSpec) + }{ + { + name: "guest round without a disk round", + setup: func(_ *testing.T, _ string, spec *MigrateSpec) { + spec.Rounds = func(string) (int, error) { return 1, nil } + }, + }, + { + name: "disk round without a guest round", + setup: func(t *testing.T, dir string, _ *MigrateSpec) { + writeSparse(t, filepath.Join(dir, diskDeltaName(types.COWRawFileName, 1)), 8, nil) + }, + }, + { + name: "round killed before its reference moved", + setup: func(t *testing.T, dir string, spec *MigrateSpec) { + spec.Rounds = func(string) (int, error) { return 1, nil } + disk := filepath.Join(dir, types.COWRawFileName) + writeSparse(t, filepath.Join(dir, diskDeltaName(types.COWRawFileName, 1)), 8, nil) + writeSparse(t, nextRefPath(disk), 8, nil) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + calls := &migrateCalls{} + spec := migrateStubSpec(calls) + tt.setup(t, dir, &spec) + _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, spec) + if err == nil || !strings.Contains(err.Error(), "start a new series") { + t.Fatalf("capture: %v, want out of step", err) + } + if calls.paused != 0 || calls.ended != 1 || utils.FileExists(dir) { + t.Errorf("paused %d ended %d kept %t, want 0 1 false", calls.paused, calls.ended, utils.FileExists(dir)) + } + }) + } +} + +func TestMigrateRoundRewritesTheEnvelope(t *testing.T) { + b, id, rec := newMigrateTestVM(t) + dir := seedSeries(t, rec) + if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error { + idx.VMs[id].NetworkConfigs = []*types.NetworkConfig{{MTU: 1500}, {MTU: 1500}} + return nil + }); err != nil { + t.Fatalf("seed nics: %v", err) + } + if _, err := b.MigrateCaptureSequence(t.Context(), id, MigrateRequest{Series: "s1"}, migrateStubSpec(&migrateCalls{})); err != nil { + skipWithoutFiemap(t, err) + t.Fatalf("capture: %v", err) + } + envelope, err := snapshot.ReadSnapshotEnvelope(dir) + if err != nil { + t.Fatalf("read envelope: %v", err) + } + if envelope.Config.NICs != 2 || envelope.Config.ID != "s1" { + t.Errorf("envelope nics %d id %q, want 2 s1", envelope.Config.NICs, envelope.Config.ID) + } +} + +func TestIsSeriesDirLeavesASnapshotBeingPersisted(t *testing.T) { + dir := t.TempDir() + series := filepath.Join(dir, seriesDirPrefix+"s1") + persisting := filepath.Join(dir, captureDirPrefix+"123") + for _, d := range []string{series, persisting} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + } + if errs := utils.RemoveMatching(t.Context(), dir, isSeriesDir); len(errs) != 0 { + t.Fatalf("remove: %v", errs) + } + if utils.FileExists(series) || !utils.FileExists(persisting) { + t.Errorf("series kept %t, persisting snapshot kept %t, want false true", utils.FileExists(series), utils.FileExists(persisting)) + } +} + +func TestMigrateAbortOfAStoppedVMDeletesTheSeries(t *testing.T) { + b, _ := newMeteringTestBackend(t) + seedVMRecord(t, b, "stopped", 1, 512, 1024, true) + rec, err := b.LoadRecord(t.Context(), "stopped") + if err != nil { + t.Fatalf("load: %v", err) + } + dir := seedSeries(t, &rec) + calls := &migrateCalls{} + if err = b.MigrateAbortSequence(t.Context(), "stopped", "s1", migrateStubSpec(calls)); err != nil { + t.Fatalf("abort: %v", err) + } + if calls.resumed != 0 || utils.FileExists(dir) { + t.Errorf("resumed %d series kept %t, want 0 false", calls.resumed, utils.FileExists(dir)) + } +} + +func TestSeriesDirRejectsPathsOutOfTheRunDir(t *testing.T) { + rec := &VMRecord{RunDir: "/run/vm"} + for _, series := range []string{"", "..", "../x", "a/b", "/abs"} { + if _, err := seriesDir(rec, series); err == nil { + t.Errorf("series %q accepted", series) + } + } + got, err := seriesDir(rec, "123") + if err != nil || got != "/run/vm/snapshot-migrate-123" { + t.Errorf("got %q, %v", got, err) + } +} + +func TestIsMigrateDeltaFile(t *testing.T) { + for name, want := range map[string]bool{ + "cow.raw.delta.1": true, + "data-x.raw.delta.12": true, + "cow.raw.ref": true, + "cow.raw.next.ref": true, + "cow.raw": false, + "memory-ranges.diff.1": false, + "state.json": false, + } { + if got := IsMigrateDeltaFile(name); got != want { + t.Errorf("%s = %t, want %t", name, got, want) + } + } +} + +type migrateCalls struct { + paused int + resumed int + ended int + rounds int +} + +func migrateStubSpec(calls *migrateCalls) MigrateSpec { + return MigrateSpec{ + Pause: func(*VMRecord, *http.Client) error { calls.paused++; return nil }, + Resume: func(*VMRecord, *http.Client) error { calls.resumed++; return nil }, + Capture: func(_ *VMRecord, _ *http.Client, dir string) error { + calls.rounds++ + return os.WriteFile(filepath.Join(dir, "mem."+strconv.Itoa(calls.rounds)), []byte("x"), 0o600) + }, + BuildMeta: func(*VMRecord, string) (*snapshot.SnapshotMeta, error) { return &snapshot.SnapshotMeta{}, nil }, + EndSeries: func(*VMRecord, *http.Client) error { calls.ended++; return nil }, + Rounds: func(string) (int, error) { return calls.rounds, nil }, + } +} + +func newMigrateTestVM(t *testing.T) (*Backend, string, *VMRecord) { + t.Helper() + b, id := newHibernateTestVM(t) + rec, err := b.LoadRecord(t.Context(), id) + if err != nil { + t.Fatalf("load: %v", err) + } + cow := filepath.Join(rec.RunDir, types.COWRawFileName) + writeSparse(t, cow, 1<<20, map[int64]string{0: "live"}) + if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error { + idx.VMs[id].StorageConfigs = []*types.StorageConfig{{Path: cow, Serial: CowSerial, Role: types.StorageRoleCOW}} + return nil + }); err != nil { + t.Fatalf("seed disks: %v", err) + } + return b, id, &rec +} + +func seedSeries(t *testing.T, rec *VMRecord) string { + t.Helper() + dir := filepath.Join(rec.RunDir, seriesDirPrefix+"s1") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + writeSparse(t, filepath.Join(dir, types.COWRawFileName), 1<<20, map[int64]string{0: "base"}) + return dir +} + +func fileNames(round *MigrateRound) []string { + names := make([]string, len(round.Files)) + for i, f := range round.Files { + names[i] = f.Name + } + return names +} + +func skipWithoutFiemap(t *testing.T, err error) { + t.Helper() + if errors.Is(err, errors.ErrUnsupported) { + t.Skip("needs FIEMAP") + } +} diff --git a/hypervisor/snapshot.go b/hypervisor/snapshot.go index 9736f9df..753ea31b 100644 --- a/hypervisor/snapshot.go +++ b/hypervisor/snapshot.go @@ -147,44 +147,41 @@ func (b *Backend) HibernateSequence(ctx context.Context, ref string, spec Hibern } func (b *Backend) prepareSnapshot(ctx context.Context, ref string) (string, VMRecord, string, func(), error) { - vmID, err := b.ResolveRef(ctx, ref) + vmID, rec, unlock, err := b.lockForCapture(ctx, ref) if err != nil { return "", VMRecord{}, "", nil, err } - unlock, err := b.LockVMOps(ctx, vmID) + tmpDir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) if err != nil { - return "", VMRecord{}, "", nil, err - } - fail := func(err error) (string, VMRecord, string, func(), error) { unlock() - return "", VMRecord{}, "", nil, err + return "", VMRecord{}, "", nil, fmt.Errorf("create temp dir: %w", err) } - rec, err := b.EntryGuardLoad(ctx, vmID) + return vmID, rec, tmpDir, unlock, nil +} + +func (b *Backend) lockForCapture(ctx context.Context, ref string) (string, VMRecord, func(), error) { + vmID, err := b.ResolveRef(ctx, ref) if err != nil { - return fail(err) + return "", VMRecord{}, nil, err } - if vErr := validateRecordInvariants(&rec); vErr != nil { - return fail(vErr) + unlock, err := b.LockVMOps(ctx, vmID) + if err != nil { + return "", VMRecord{}, nil, err + } + rec, err := b.EntryGuardLoad(ctx, vmID) + if err == nil { + err = validateRecordInvariants(&rec) } - tmpDir, err := os.MkdirTemp(rec.RunDir, captureDirPrefix) if err != nil { - return fail(fmt.Errorf("create temp dir: %w", err)) + unlock() + return "", VMRecord{}, nil, err } - return vmID, rec, tmpDir, unlock, nil + return vmID, rec, unlock, nil } func (b *Backend) finalizeSnapshot(ctx context.Context, vmID string, rec *VMRecord, spec SnapshotSpec, tmpDir string) (*types.SnapshotConfig, error) { - if spec.AfterCapture != nil { - if err := spec.AfterCapture(rec, tmpDir); err != nil { - return nil, err - } - } - meta, err := spec.BuildMeta(rec, tmpDir) - if err != nil { - return nil, fmt.Errorf("build snapshot metadata: %w", err) - } - if err = snapshot.SaveSnapshotMeta(tmpDir, meta); err != nil { - return nil, fmt.Errorf("save snapshot metadata: %w", err) + if err := saveCaptureMeta(rec, spec, tmpDir); err != nil { + return nil, err } snapID, err := b.RecordSnapshot(ctx, vmID) if err != nil { @@ -260,6 +257,22 @@ func ReverseLayers[T any](storageConfigs []*types.StorageConfig, project func(id return out } +func saveCaptureMeta(rec *VMRecord, spec SnapshotSpec, tmpDir string) error { + if spec.AfterCapture != nil { + if err := spec.AfterCapture(rec, tmpDir); err != nil { + return err + } + } + meta, err := spec.BuildMeta(rec, tmpDir) + if err != nil { + return fmt.Errorf("build snapshot metadata: %w", err) + } + if err = snapshot.SaveSnapshotMeta(tmpDir, meta); err != nil { + return fmt.Errorf("save snapshot metadata: %w", err) + } + return nil +} + func resumeAfterCapture(resume TransitionHook, rec *VMRecord, hc *http.Client) error { hc.Timeout = VMMemTransferTimeout if err := resume(rec, hc); err != nil { diff --git a/hypervisor/state_test.go b/hypervisor/state_test.go index 71a8a961..b4597447 100644 --- a/hypervisor/state_test.go +++ b/hypervisor/state_test.go @@ -215,7 +215,7 @@ func TestFinalizeCloneEmitsCloneEntries(t *testing.T) { State: types.VMStateRunning, Config: types.VMConfig{CPU: 2, Memory: 2 << 30, Storage: 20 << 30}, } - if err := b.FinalizeClone(ctx, "vm1", info, nil, nil, "snap-source"); err != nil { + if err := b.FinalizeClone(ctx, "vm1", info, nil, nil, "snap-source", metering.ReasonClone); err != nil { t.Fatalf("FinalizeClone: %v", err) } entries := rec.Entries() diff --git a/hypervisor/utils.go b/hypervisor/utils.go index bc388463..fb49c84b 100644 --- a/hypervisor/utils.go +++ b/hypervisor/utils.go @@ -34,6 +34,8 @@ const ( SnapshotFileCOW // SnapshotFileMeta is small metadata that is plain-copied. SnapshotFileMeta + // SnapshotFileSkip is left out of the clone. + SnapshotFileSkip // CloneLocksDirName holds FC clone locks under the backend run root, outside any VM dir — a clone must be able to lock a source whose dir is already gone. CloneLocksDirName = "clone-locks" diff --git a/metering/metering.go b/metering/metering.go index 09e6aba9..f98a542b 100644 --- a/metering/metering.go +++ b/metering/metering.go @@ -20,6 +20,7 @@ const ( ReasonRestart Reason = "restart" ReasonClone Reason = "clone" ReasonRestore Reason = "restore" + ReasonMigrate Reason = "migrate" ReasonStopUser Reason = "stop-user" ReasonStopCrash Reason = "stop-crash" ReasonVMRemove Reason = "vm-rm" diff --git a/utils/extents.go b/utils/extents.go new file mode 100644 index 00000000..d98d05f4 --- /dev/null +++ b/utils/extents.go @@ -0,0 +1,9 @@ +package utils + +// Extent maps a file range to its on-disk location. +type Extent struct { + Logical int64 + Physical int64 + Length int64 + Unwritten bool +} diff --git a/utils/extents_linux.go b/utils/extents_linux.go new file mode 100644 index 00000000..03ca329f --- /dev/null +++ b/utils/extents_linux.go @@ -0,0 +1,78 @@ +//go:build linux + +package utils + +import ( + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + // fsIocFiemap is FS_IOC_FIEMAP; x/sys/unix does not define it. + fsIocFiemap uintptr = 0xC020660B + + fiemapFlagSync uint32 = 0x1 + fiemapExtentLast uint32 = 0x1 + fiemapExtentUnwritten uint32 = 0x800 + + fiemapBatch = 512 +) + +type fiemapExtent struct { + Logical uint64 + Physical uint64 + Length uint64 + reserved64 [2]uint64 + Flags uint32 + reserved [3]uint32 +} + +type fiemapRequest struct { + Start uint64 + Length uint64 + Flags uint32 + Mapped uint32 + Count uint32 + _ uint32 + Extents [fiemapBatch]fiemapExtent +} + +// FileExtents flushes f and returns its extents in logical order. +func FileExtents(f *os.File) ([]Extent, error) { + var out []Extent + req := &fiemapRequest{Length: ^uint64(0), Flags: fiemapFlagSync, Count: fiemapBatch} + for { + req.Mapped = 0 + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), fsIocFiemap, uintptr(unsafe.Pointer(req))); errno != 0 { //nolint:gosec + return nil, fmt.Errorf("fiemap %s: %w", f.Name(), errno) + } + if req.Mapped == 0 { + return out, nil + } + for _, e := range req.Extents[:req.Mapped] { + out = append(out, Extent{ + Logical: int64(e.Logical), //nolint:gosec + Physical: int64(e.Physical), //nolint:gosec + Length: int64(e.Length), //nolint:gosec + Unwritten: e.Flags&fiemapExtentUnwritten != 0, + }) + } + last := req.Extents[req.Mapped-1] + if last.Flags&fiemapExtentLast != 0 { + return out, nil + } + req.Start = last.Logical + last.Length + req.Length = ^uint64(0) - req.Start + } +} + +// PunchHole deallocates a range of f and keeps its size. +func PunchHole(f *os.File, offset, length int64) error { + if err := unix.Fallocate(int(f.Fd()), unix.FALLOC_FL_PUNCH_HOLE|unix.FALLOC_FL_KEEP_SIZE, offset, length); err != nil { + return fmt.Errorf("punch hole in %s at %d len %d: %w", f.Name(), offset, length, err) + } + return nil +} diff --git a/utils/extents_linux_test.go b/utils/extents_linux_test.go new file mode 100644 index 00000000..7c46c4da --- /dev/null +++ b/utils/extents_linux_test.go @@ -0,0 +1,57 @@ +//go:build linux + +package utils + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFileExtentsReportsDataAndSkipsHoles(t *testing.T) { + path := filepath.Join(t.TempDir(), "f") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer f.Close() + if err = f.Truncate(1 << 22); err != nil { + t.Fatalf("truncate: %v", err) + } + if _, err = f.WriteAt(make([]byte, 4096), 1<<21); err != nil { + t.Fatalf("write: %v", err) + } + extents, err := FileExtents(f) + if err != nil { + t.Skipf("filesystem has no FIEMAP: %v", err) + } + if len(extents) == 0 { + t.Fatal("no extents after a flushed write") + } + first, last := extents[0], extents[len(extents)-1] + if first.Logical > 1<<21 || last.Logical+last.Length < 1<<21+4096 { + t.Errorf("extents %+v do not cover the write", extents) + } +} + +func TestPunchHoleKeepsSize(t *testing.T) { + path := filepath.Join(t.TempDir(), "f") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer f.Close() + if _, err = f.WriteAt([]byte{1, 2, 3, 4}, 0); err != nil { + t.Fatalf("write: %v", err) + } + if err = PunchHole(f, 0, 2); err != nil { + t.Skipf("filesystem has no hole punch: %v", err) + } + fi, err := f.Stat() + if err != nil { + t.Fatalf("stat: %v", err) + } + if fi.Size() != 4 { + t.Errorf("size = %d, want 4", fi.Size()) + } +} diff --git a/utils/extents_other.go b/utils/extents_other.go new file mode 100644 index 00000000..dcb38b22 --- /dev/null +++ b/utils/extents_other.go @@ -0,0 +1,18 @@ +//go:build !linux + +package utils + +import ( + "errors" + "os" +) + +// FileExtents needs FIEMAP, so it fails off Linux. +func FileExtents(*os.File) ([]Extent, error) { + return nil, errors.ErrUnsupported +} + +// PunchHole needs fallocate, so it fails off Linux. +func PunchHole(*os.File, int64, int64) error { + return errors.ErrUnsupported +} diff --git a/utils/id.go b/utils/id.go index 11ed6ad5..fab46f20 100644 --- a/utils/id.go +++ b/utils/id.go @@ -2,9 +2,19 @@ package utils import ( "crypto/rand" + "strings" ) +const idLength = 26 + // GenerateID returns a random 26-character crypto-secure base32 ID. func GenerateID() string { return rand.Text() } + +// ValidID reports whether id has the shape GenerateID produces. +func ValidID(id string) bool { + return len(id) == idLength && !strings.ContainsFunc(id, func(r rune) bool { + return (r < 'A' || r > 'Z') && (r < '2' || r > '7') + }) +} diff --git a/utils/id_test.go b/utils/id_test.go index 4489827b..36c638c5 100644 --- a/utils/id_test.go +++ b/utils/id_test.go @@ -4,6 +4,17 @@ import ( "testing" ) +func TestValidID(t *testing.T) { + if !ValidID(GenerateID()) { + t.Error("a generated ID is invalid") + } + for _, id := range []string{"", "short", "../../../../../../../etc/pw", "abcdefghijklmnopqrstuvwxyz", "AAAAAAAAAAAAAAAAAAAAAAAAA1"} { + if ValidID(id) { + t.Errorf("%q accepted", id) + } + } +} + func TestGenerateID_Length(t *testing.T) { id := GenerateID() if len(id) != 26 { diff --git a/utils/reflink_linux.go b/utils/reflink_linux.go index d40f17eb..23fb335c 100644 --- a/utils/reflink_linux.go +++ b/utils/reflink_linux.go @@ -36,6 +36,19 @@ func ReflinkCopy(ctx context.Context, dst, src string, sync SyncMode) error { return SparseCopy(dst, src, sync) } +// CheckReflink reports whether the filesystem holding dir supports FICLONE. +func CheckReflink(dir string) error { + src, err := os.CreateTemp(dir, ".reflink-probe-*") + if err != nil { + return fmt.Errorf("create reflink probe: %w", err) + } + _ = src.Close() + defer os.Remove(src.Name()) //nolint:errcheck + dst := src.Name() + ".clone" + defer os.Remove(dst) //nolint:errcheck + return tryFiclone(dst, src.Name(), NoSync) +} + func tryFiclone(dst, src string, sync SyncMode) error { return CopyWithCleanup(dst, src, func(srcFile, dstFile *os.File) error { if err := unix.IoctlFileClone(int(dstFile.Fd()), int(srcFile.Fd())); err != nil { diff --git a/utils/reflink_other.go b/utils/reflink_other.go index 70bb2f8d..29dfdbb8 100644 --- a/utils/reflink_other.go +++ b/utils/reflink_other.go @@ -2,9 +2,17 @@ package utils -import "context" +import ( + "context" + "errors" +) // ReflinkCopy copies a single file. On non-Linux, falls back to SparseCopy. func ReflinkCopy(_ context.Context, dst, src string, sync SyncMode) error { return SparseCopy(dst, src, sync) } + +// CheckReflink needs FICLONE, so it fails off Linux. +func CheckReflink(string) error { + return errors.ErrUnsupported +}