From 4ab69ee5b072fcedf2659a0ae36162d5db9d0d6a Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:46:36 +0000 Subject: [PATCH 01/29] Add Windows hypervisor primitives --- .github/workflows/test.yml | 12 +- lib/hypervisor/cloudhypervisor/process.go | 7 +- lib/hypervisor/config.go | 52 +++++++- lib/hypervisor/config_validation.go | 57 ++++++++ lib/hypervisor/config_validation_test.go | 51 ++++++++ lib/hypervisor/firecracker/process.go | 7 +- lib/hypervisor/qemu/config.go | 29 ++++- lib/hypervisor/qemu/config_test.go | 39 ++++++ lib/hypervisor/qemu/process.go | 23 +++- lib/hypervisor/qemu/profile.go | 17 ++- lib/hypervisor/qemu/swtpm.go | 87 +++++++++++++ .../windows_config_integration_linux_test.go | 123 ++++++++++++++++++ lib/hypervisor/vz/starter.go | 7 +- 13 files changed, 497 insertions(+), 14 deletions(-) create mode 100644 lib/hypervisor/config_validation.go create mode 100644 lib/hypervisor/config_validation_test.go create mode 100644 lib/hypervisor/qemu/swtpm.go create mode 100644 lib/hypervisor/qemu/windows_config_integration_linux_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 261148f24..83cad318e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,9 +79,13 @@ jobs: ! command -v mkfs.ext4 &> /dev/null || \ ! command -v iptables &> /dev/null || \ ! command -v qemu-system-x86_64 &> /dev/null || \ - ! qemu-system-x86_64 --version >/dev/null 2>&1; then + ! qemu-system-x86_64 --version >/dev/null 2>&1 || \ + ! command -v qemu-img &> /dev/null || \ + ! command -v swtpm &> /dev/null || \ + ! test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd || \ + ! test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd; then apt_update_with_retry - timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables qemu-system-x86 qemu-utils + timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables ovmf qemu-system-x86 qemu-utils swtpm fi go mod download @@ -89,13 +93,15 @@ jobs: run: | set -euo pipefail TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" - for bin in mkfs.erofs mkfs.ext4 iptables qemu-system-x86_64; do + for bin in mkfs.erofs mkfs.ext4 iptables qemu-img qemu-system-x86_64 swtpm; do if ! sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin' >/dev/null"; then echo "missing required binary under sudo PATH: $bin" exit 1 fi sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin'" done + test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd + test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index 14b306be1..824b19160 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -72,7 +72,9 @@ func NewStarter() *Starter { // Verify Starter implements the interface var _ hypervisor.VMStarter = (*Starter)(nil) -func (s *Starter) ValidateConfig(hypervisor.VMConfig) error { return nil } +func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { + return hypervisor.ValidateDirectRawConfig("cloud-hypervisor", config) +} // SocketName returns the socket filename for Cloud Hypervisor. func (s *Starter) SocketName() string { @@ -108,6 +110,9 @@ func (s *Starter) ResolveVersion(p *paths.Paths, requested string) (string, erro // StartVM launches Cloud Hypervisor, configures the VM, and boots it. // Returns the process ID and a Hypervisor client for subsequent operations. func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { + if err := s.ValidateConfig(config); err != nil { + return 0, nil, fmt.Errorf("validate cloud-hypervisor config: %w", err) + } log := logger.FromContext(ctx) // Validate version diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index 2562868da..07f392e8d 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -26,7 +26,12 @@ type VMConfig struct { // PCI device passthrough (GPU, etc.) PCIDevices []string - // Boot configuration + // Boot configuration. Empty BootMode preserves the existing direct-kernel + // behavior for Linux callers. + BootMode BootMode + Firmware *FirmwareConfig + TPM *TPMConfig + KernelPath string InitrdPath string KernelArgs string @@ -54,9 +59,54 @@ type CPUTopology struct { Packages int } +type BootMode string + +const ( + BootModeDirect BootMode = "direct" + BootModeUEFI BootMode = "uefi" +) + +// EffectiveBootMode preserves direct Linux kernel boot for existing callers. +func (c VMConfig) EffectiveBootMode() BootMode { + if c.BootMode == "" { + return BootModeDirect + } + return c.BootMode +} + +// FirmwareConfig describes UEFI firmware files. CodePath is immutable firmware; +// VarsPath is per-instance writable variable storage. +type FirmwareConfig struct { + CodePath string + VarsPath string + SecureBoot bool +} + +// TPMConfig describes a per-instance software TPM 2.0 endpoint. +type TPMConfig struct { + SocketPath string + StateDir string +} + +type DiskFormat string + +const ( + DiskFormatRaw DiskFormat = "raw" + DiskFormatQCOW2 DiskFormat = "qcow2" +) + +// EffectiveFormat preserves raw disks for existing callers. +func (d DiskConfig) EffectiveFormat() DiskFormat { + if d.Format == "" { + return DiskFormatRaw + } + return d.Format +} + // DiskConfig represents a disk attached to the VM type DiskConfig struct { Path string + Format DiskFormat Readonly bool IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited) IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps) diff --git a/lib/hypervisor/config_validation.go b/lib/hypervisor/config_validation.go new file mode 100644 index 000000000..4419e2a01 --- /dev/null +++ b/lib/hypervisor/config_validation.go @@ -0,0 +1,57 @@ +package hypervisor + +import "fmt" + +// ValidateBootConfig validates boot and disk fields shared by hypervisor backends. +func ValidateBootConfig(cfg VMConfig) error { + switch cfg.EffectiveBootMode() { + case BootModeDirect: + if cfg.Firmware != nil { + return fmt.Errorf("direct boot cannot specify firmware") + } + if cfg.TPM != nil { + return fmt.Errorf("direct boot cannot specify a TPM") + } + case BootModeUEFI: + if cfg.Firmware == nil { + return fmt.Errorf("UEFI boot requires firmware") + } + if cfg.Firmware.CodePath == "" || cfg.Firmware.VarsPath == "" { + return fmt.Errorf("UEFI boot requires firmware code and variable storage paths") + } + if cfg.KernelPath != "" || cfg.InitrdPath != "" || cfg.KernelArgs != "" { + return fmt.Errorf("UEFI boot cannot specify a direct kernel, initrd, or kernel arguments") + } + if cfg.TPM != nil && (cfg.TPM.SocketPath == "" || cfg.TPM.StateDir == "") { + return fmt.Errorf("TPM requires socket and state directory paths") + } + default: + return fmt.Errorf("unsupported boot mode %q", cfg.BootMode) + } + + for i, disk := range cfg.Disks { + switch disk.EffectiveFormat() { + case DiskFormatRaw, DiskFormatQCOW2: + default: + return fmt.Errorf("disk %d has unsupported format %q", i, disk.Format) + } + } + return nil +} + +// ValidateDirectRawConfig preserves the Linux-only contract of backends that +// do not implement firmware boot or qcow2 disks. +func ValidateDirectRawConfig(backend string, cfg VMConfig) error { + if err := ValidateBootConfig(cfg); err != nil { + return err + } + if cfg.EffectiveBootMode() != BootModeDirect { + return fmt.Errorf("%s does not support %s boot", backend, cfg.EffectiveBootMode()) + } + for i, disk := range cfg.Disks { + if disk.EffectiveFormat() != DiskFormatRaw { + return fmt.Errorf("%s does not support disk %d format %q", backend, i, disk.EffectiveFormat()) + } + } + return nil +} diff --git a/lib/hypervisor/config_validation_test.go b/lib/hypervisor/config_validation_test.go new file mode 100644 index 000000000..8bfa71916 --- /dev/null +++ b/lib/hypervisor/config_validation_test.go @@ -0,0 +1,51 @@ +package hypervisor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateBootConfigPreservesDirectRawDefaults(t *testing.T) { + cfg := VMConfig{ + KernelPath: "/kernel", + Disks: []DiskConfig{{Path: "/rootfs"}}, + } + require.NoError(t, ValidateBootConfig(cfg)) + assert.Equal(t, BootModeDirect, cfg.EffectiveBootMode()) + assert.Equal(t, DiskFormatRaw, cfg.Disks[0].EffectiveFormat()) +} + +func TestValidateBootConfigUEFI(t *testing.T) { + valid := VMConfig{ + BootMode: BootModeUEFI, + Firmware: &FirmwareConfig{CodePath: "/ovmf/code", VarsPath: "/instance/vars"}, + TPM: &TPMConfig{SocketPath: "/instance/swtpm.sock", StateDir: "/instance/tpm"}, + Disks: []DiskConfig{{Path: "/instance/disk", Format: DiskFormatQCOW2}}, + } + require.NoError(t, ValidateBootConfig(valid)) + + tests := []struct { + name string + cfg VMConfig + }{ + {name: "missing firmware", cfg: VMConfig{BootMode: BootModeUEFI}}, + {name: "direct kernel", cfg: VMConfig{BootMode: BootModeUEFI, Firmware: valid.Firmware, KernelPath: "/kernel"}}, + {name: "incomplete TPM", cfg: VMConfig{BootMode: BootModeUEFI, Firmware: valid.Firmware, TPM: &TPMConfig{StateDir: "/state"}}}, + {name: "unknown disk", cfg: VMConfig{Disks: []DiskConfig{{Path: "/disk", Format: "vhdx"}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Error(t, ValidateBootConfig(tt.cfg)) + }) + } +} + +func TestValidateDirectRawConfigRejectsFirmwareAndQCOW2(t *testing.T) { + uefi := VMConfig{BootMode: BootModeUEFI, Firmware: &FirmwareConfig{CodePath: "/code", VarsPath: "/vars"}} + assert.ErrorContains(t, ValidateDirectRawConfig("backend", uefi), "does not support uefi boot") + + qcow := VMConfig{Disks: []DiskConfig{{Path: "/disk", Format: DiskFormatQCOW2}}} + assert.ErrorContains(t, ValidateDirectRawConfig("backend", qcow), "does not support disk 0 format") +} diff --git a/lib/hypervisor/firecracker/process.go b/lib/hypervisor/firecracker/process.go index 01fd8ce2e..28c9268a6 100644 --- a/lib/hypervisor/firecracker/process.go +++ b/lib/hypervisor/firecracker/process.go @@ -58,7 +58,9 @@ func WithUFFDClient(client UFFDClient) StarterOption { var _ hypervisor.VMStarter = (*Starter)(nil) -func (s *Starter) ValidateConfig(hypervisor.VMConfig) error { return nil } +func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { + return hypervisor.ValidateDirectRawConfig("firecracker", config) +} func (s *Starter) SocketName() string { return "fc.sock" @@ -90,6 +92,9 @@ func (s *Starter) ResolveVersion(p *paths.Paths, requested string) (string, erro } func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { + if err := s.ValidateConfig(config); err != nil { + return 0, nil, fmt.Errorf("validate firecracker config: %w", err) + } processCtx, processSpan := hypervisor.StartProcessSpan(ctx, hypervisor.TypeFirecracker) pid, err := s.startProcess(processCtx, p, version, socketPath) hypervisor.FinishTraceSpan(processSpan, err) diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index 648763c22..b3959a29a 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -20,7 +20,11 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { microvm := machine == MachineTypeMicroVM // Machine type with KVM acceleration (arch-specific when omitted). - args = append(args, "-machine", string(machine)+",accel=kvm") + machineArg := string(machine) + ",accel=kvm" + if cfg.Firmware != nil && cfg.Firmware.SecureBoot { + machineArg += ",smm=on" + } + args = append(args, "-machine", machineArg) if microvm { // Do not allow a host qemu.conf to add devices outside microvm's // documented eight virtio-mmio-device limit. @@ -51,6 +55,18 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { args = append(args, "-device", strings.Join(balloonOpts, ",")) } + // Firmware boot. The code image is shared and immutable; variable storage is + // a per-instance writable copy. + if cfg.EffectiveBootMode() == hypervisor.BootModeUEFI { + args = append(args, + "-drive", fmt.Sprintf("if=pflash,format=raw,unit=0,file=%s,readonly=on", cfg.Firmware.CodePath), + "-drive", fmt.Sprintf("if=pflash,format=raw,unit=1,file=%s", cfg.Firmware.VarsPath), + ) + if cfg.Firmware.SecureBoot { + args = append(args, "-global", "driver=cfi.pflash01,property=secure,value=on") + } + } + // Kernel and initrd if cfg.KernelPath != "" { args = append(args, "-kernel", cfg.KernelPath) @@ -64,7 +80,7 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { // Disk configuration for i, disk := range cfg.Disks { - driveOpts := fmt.Sprintf("file=%s,format=raw,if=none,id=drive%d", disk.Path, i) + driveOpts := fmt.Sprintf("file=%s,format=%s,if=none,id=drive%d", disk.Path, disk.EffectiveFormat(), i) if disk.Readonly { // Disable host-side file locking for shared readonly bases so multiple // VMs can boot concurrently from the same image without lock contention. @@ -80,6 +96,15 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { args = append(args, "-device", fmt.Sprintf("%s,drive=drive%d", virtioDevice(microvm, "virtio-blk"), i)) } + // Software TPM 2.0. The swtpm process is started by Starter before QEMU. + if cfg.TPM != nil { + args = append(args, + "-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s", cfg.TPM.SocketPath), + "-tpmdev", "emulator,id=tpm0,chardev=chrtpm", + "-device", "tpm-crb,tpmdev=tpm0", + ) + } + // Network configuration for i, net := range cfg.Networks { netdevOpts := fmt.Sprintf("tap,id=net%d,ifname=%s,script=no,downscript=no", i, net.TAPDevice) diff --git a/lib/hypervisor/qemu/config_test.go b/lib/hypervisor/qemu/config_test.go index c8d25584f..890a76993 100644 --- a/lib/hypervisor/qemu/config_test.go +++ b/lib/hypervisor/qemu/config_test.go @@ -81,6 +81,35 @@ func TestBuildArgs_Disks(t *testing.T) { assert.Contains(t, args, "virtio-blk-pci,drive=drive1") } +func TestBuildArgs_UEFISecureBootTPMAndQCOW2(t *testing.T) { + cfg := hypervisor.VMConfig{ + VCPUs: 2, + MemoryBytes: 1024 * 1024 * 1024, + BootMode: hypervisor.BootModeUEFI, + Firmware: &hypervisor.FirmwareConfig{ + CodePath: "/firmware/OVMF_CODE.fd", + VarsPath: "/instance/OVMF_VARS.fd", + SecureBoot: true, + }, + TPM: &hypervisor.TPMConfig{ + SocketPath: "/instance/swtpm.sock", + StateDir: "/instance/tpm", + }, + Disks: []hypervisor.DiskConfig{{Path: "/instance/windows.qcow2", Format: hypervisor.DiskFormatQCOW2}}, + } + + args := buildArgs(cfg, MachineTypeQ35) + assert.Contains(t, args, "q35,accel=kvm,smm=on") + assert.Contains(t, args, "if=pflash,format=raw,unit=0,file=/firmware/OVMF_CODE.fd,readonly=on") + assert.Contains(t, args, "if=pflash,format=raw,unit=1,file=/instance/OVMF_VARS.fd") + assert.Contains(t, args, "driver=cfi.pflash01,property=secure,value=on") + assert.Contains(t, args, "file=/instance/windows.qcow2,format=qcow2,if=none,id=drive0") + assert.Contains(t, args, "socket,id=chrtpm,path=/instance/swtpm.sock") + assert.Contains(t, args, "emulator,id=tpm0,chardev=chrtpm") + assert.Contains(t, args, "tpm-crb,tpmdev=tpm0") + assert.NotContains(t, args, "-kernel") +} + func TestBuildArgs_Network(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, @@ -187,6 +216,16 @@ func TestBuildArgs_MicroVM(t *testing.T) { } } +func TestProfilesValidateFirmwareAndDiskFormats(t *testing.T) { + uefi := hypervisor.VMConfig{ + BootMode: hypervisor.BootModeUEFI, + Firmware: &hypervisor.FirmwareConfig{CodePath: "/code", VarsPath: "/vars"}, + Disks: []hypervisor.DiskConfig{{Path: "/disk", Format: hypervisor.DiskFormatQCOW2}}, + } + assert.NoError(t, StandardProfile{}.validateConfig(uefi)) + assert.ErrorContains(t, MicroVMProfile{}.validateConfig(uefi), "does not support uefi boot") +} + func TestBuildArgs_GuestMemoryBalloon(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c02c5beef..aa85455cd 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -425,6 +425,25 @@ func (s *Starter) validateSnapshotMachineType(stored MachineType) (MachineType, return expected, nil } +func (s *Starter) startConfiguredProcess(ctx context.Context, p *paths.Paths, version, socketPath string, config hypervisor.VMConfig, args []string) (int, *QEMU, *cleanup.Cleanup, error) { + tpmProcess, err := startSWTPM(config.TPM, filepath.Dir(socketPath)) + if err != nil { + return 0, nil, nil, err + } + + pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + if err != nil { + if tpmProcess != nil { + tpmProcess.cleanup() + } + return 0, nil, nil, err + } + if tpmProcess != nil { + cu.Add(tpmProcess.cleanup) + } + return pid, hv, cu, nil +} + // StartVM launches QEMU with the VM configuration and returns a Hypervisor client. // QEMU receives all configuration via command-line arguments at process start. func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { @@ -474,7 +493,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) - pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err = s.startConfiguredProcess(ctx, p, version, socketPath, attempt, args) if err == nil { booted = attempt started = true @@ -609,7 +628,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, incomingURI := "exec:cat < " + memoryFile args = append(args, "-incoming", incomingURI) - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err := s.startConfiguredProcess(ctx, p, version, socketPath, config, args) if err != nil { return 0, nil, err } diff --git a/lib/hypervisor/qemu/profile.go b/lib/hypervisor/qemu/profile.go index 2ac3a2cbf..565ab3a70 100644 --- a/lib/hypervisor/qemu/profile.go +++ b/lib/hypervisor/qemu/profile.go @@ -27,9 +27,17 @@ func (StandardProfile) machineType() (MachineType, error) { func (StandardProfile) capabilities() hypervisor.Capabilities { return qemuCapabilities(true) } -func (StandardProfile) validateConfig(hypervisor.VMConfig) error { return nil } -func (StandardProfile) requiresStoredMachineType() bool { return false } -func (StandardProfile) requiresStoredVersion() bool { return false } +func (StandardProfile) validateConfig(cfg hypervisor.VMConfig) error { + if err := hypervisor.ValidateBootConfig(cfg); err != nil { + return err + } + if cfg.EffectiveBootMode() == hypervisor.BootModeUEFI && standardMachineType() != MachineTypeQ35 { + return fmt.Errorf("UEFI boot is currently supported only by qemu/q35 on amd64") + } + return nil +} +func (StandardProfile) requiresStoredMachineType() bool { return false } +func (StandardProfile) requiresStoredVersion() bool { return false } // MicroVMProfile selects QEMU's minimal x86 microvm board and enforces its // virtio-mmio device contract. @@ -43,6 +51,9 @@ func (MicroVMProfile) capabilities() hypervisor.Capabilities { return qemuCapabilities(false) } func (MicroVMProfile) validateConfig(cfg hypervisor.VMConfig) error { + if err := hypervisor.ValidateDirectRawConfig("qemu-microvm", cfg); err != nil { + return err + } if cfg.HotplugBytes > 0 { return fmt.Errorf("microvm does not support hotplug memory") } diff --git a/lib/hypervisor/qemu/swtpm.go b/lib/hypervisor/qemu/swtpm.go new file mode 100644 index 000000000..e85147c02 --- /dev/null +++ b/lib/hypervisor/qemu/swtpm.go @@ -0,0 +1,87 @@ +package qemu + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +func startSWTPM(config *hypervisor.TPMConfig, instanceDir string) (*startedProcess, error) { + if config == nil { + return nil, nil + } + + binary, err := exec.LookPath("swtpm") + if err != nil { + return nil, fmt.Errorf("find swtpm: %w", err) + } + if err := os.MkdirAll(config.StateDir, 0700); err != nil { + return nil, fmt.Errorf("create swtpm state directory: %w", err) + } + if err := os.MkdirAll(filepath.Dir(config.SocketPath), 0755); err != nil { + return nil, fmt.Errorf("create swtpm socket directory: %w", err) + } + if err := os.Remove(config.SocketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("remove stale swtpm socket: %w", err) + } + + logsDir := filepath.Join(instanceDir, "logs") + if err := os.MkdirAll(logsDir, 0755); err != nil { + return nil, fmt.Errorf("create swtpm logs directory: %w", err) + } + logPath := filepath.Join(logsDir, "swtpm.log") + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return nil, fmt.Errorf("create swtpm log: %w", err) + } + defer logFile.Close() + + cmd := exec.Command(binary, + "socket", + "--tpm2", + "--tpmstate", "dir="+config.StateDir, + "--ctrl", "type=unixio,path="+config.SocketPath, + "--terminate", + ) + cmd.Stdout = logFile + cmd.Stderr = logFile + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + proc, err := startManagedProcess(cmd, config.SocketPath) + if err != nil { + return nil, fmt.Errorf("start swtpm: %w", err) + } + if err := waitForSocketFileOrExit(config.SocketPath, socketWaitTimeout, proc); err != nil { + proc.cleanup() + if logData, readErr := os.ReadFile(logPath); readErr == nil && len(logData) > 0 { + const maxLogBytes = 32 << 10 + if len(logData) > maxLogBytes { + logData = logData[len(logData)-maxLogBytes:] + } + return nil, fmt.Errorf("wait for swtpm: %w; swtpm.log: %s", err, logData) + } + return nil, fmt.Errorf("wait for swtpm: %w", err) + } + return proc, nil +} + +// A connect probe would make swtpm treat the probe as its control client; with +// --terminate, closing that probe would stop swtpm before QEMU connects. +func waitForSocketFileOrExit(socketPath string, timeout time.Duration, proc *startedProcess) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if info, err := os.Stat(socketPath); err == nil && info.Mode()&os.ModeSocket != 0 { + return nil + } + if waitErr, exited := proc.checkExited(); exited { + return fmt.Errorf("swtpm exited early: %w", waitErr) + } + time.Sleep(socketPollInterval) + } + return fmt.Errorf("timeout waiting for socket") +} diff --git a/lib/hypervisor/qemu/windows_config_integration_linux_test.go b/lib/hypervisor/qemu/windows_config_integration_linux_test.go new file mode 100644 index 000000000..2eab0ef14 --- /dev/null +++ b/lib/hypervisor/qemu/windows_config_integration_linux_test.go @@ -0,0 +1,123 @@ +//go:build linux && amd64 + +package qemu + +import ( + "context" + "io/fs" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func requireWindowsConfigDependency(t *testing.T, path, description string) string { + t.Helper() + if path != "" { + if _, err := os.Stat(path); err == nil { + return path + } + } + if os.Getenv("CI") == "true" { + t.Fatalf("required Windows config integration dependency is missing: %s (%s)", description, path) + } + t.Skipf("%s is unavailable", description) + return "" +} + +func TestWindowsConfigIntegration(t *testing.T) { + requireWindowsConfigDependency(t, "/dev/kvm", "KVM") + if _, err := exec.LookPath("qemu-system-x86_64"); err != nil { + requireWindowsConfigDependency(t, "", "qemu-system-x86_64") + } + if _, err := exec.LookPath("qemu-img"); err != nil { + requireWindowsConfigDependency(t, "", "qemu-img") + } + if _, err := exec.LookPath("swtpm"); err != nil { + requireWindowsConfigDependency(t, "", "swtpm") + } + + codePath := os.Getenv("HYPEMAN_WINDOWS_OVMF_CODE") + if codePath == "" { + codePath = "/usr/share/OVMF/OVMF_CODE_4M.secboot.fd" + } + varsTemplate := os.Getenv("HYPEMAN_WINDOWS_OVMF_VARS") + if varsTemplate == "" { + varsTemplate = "/usr/share/OVMF/OVMF_VARS_4M.ms.fd" + } + requireWindowsConfigDependency(t, codePath, "Secure Boot OVMF code") + requireWindowsConfigDependency(t, varsTemplate, "Microsoft-enrolled OVMF variables") + + dir, err := os.MkdirTemp("/tmp", "hypeman-win-config-") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + varsData, err := os.ReadFile(varsTemplate) + require.NoError(t, err) + varsPath := filepath.Join(dir, "OVMF_VARS.fd") + require.NoError(t, os.WriteFile(varsPath, varsData, 0600)) + + basePath := filepath.Join(dir, "base.raw") + base, err := os.Create(basePath) + require.NoError(t, err) + require.NoError(t, base.Truncate(64<<20)) + require.NoError(t, base.Close()) + + diskPath := filepath.Join(dir, "instance.qcow2") + output, err := exec.Command("qemu-img", "create", "-f", "qcow2", "-F", "raw", "-b", basePath, diskPath).CombinedOutput() + require.NoError(t, err, "qemu-img create: %s", output) + + socketPath := filepath.Join(dir, "qemu.sock") + tpmSocket := filepath.Join(dir, "swtpm.sock") + tpmState := filepath.Join(dir, "tpm") + config := hypervisor.VMConfig{ + VCPUs: 1, + MemoryBytes: 512 << 20, + BootMode: hypervisor.BootModeUEFI, + Firmware: &hypervisor.FirmwareConfig{ + CodePath: codePath, + VarsPath: varsPath, + SecureBoot: true, + }, + TPM: &hypervisor.TPMConfig{SocketPath: tpmSocket, StateDir: tpmState}, + Disks: []hypervisor.DiskConfig{{Path: diskPath, Format: hypervisor.DiskFormatQCOW2}}, + } + + starter := NewStarter() + boot := func() { + pid, vm, err := starter.StartVM(context.Background(), paths.New(dir), "", socketPath, config) + require.NoError(t, err) + require.Positive(t, pid) + info, err := vm.GetVMInfo(context.Background()) + require.NoError(t, err) + require.Equal(t, hypervisor.StateRunning, info.State) + require.NoError(t, vm.Shutdown(context.Background())) + require.Eventually(t, func() bool { + _, err := os.Stat(socketPath) + return os.IsNotExist(err) + }, 5*time.Second, 20*time.Millisecond) + } + + boot() + info, err := os.Stat(diskPath) + require.NoError(t, err) + require.Positive(t, info.Size(), "qcow2 overlay must contain metadata") + + var stateFiles int + require.NoError(t, filepath.WalkDir(tpmState, func(path string, entry fs.DirEntry, err error) error { + if err == nil && !entry.IsDir() { + stateFiles++ + } + return err + })) + require.Positive(t, stateFiles, "swtpm must persist TPM 2.0 state") + require.FileExists(t, varsPath) + + boot() + require.FileExists(t, varsPath) +} diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index d62026b53..ec5132fd4 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -105,7 +105,9 @@ func NewStarter() *Starter { var _ hypervisor.VMStarter = (*Starter)(nil) -func (s *Starter) ValidateConfig(hypervisor.VMConfig) error { return nil } +func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { + return hypervisor.ValidateDirectRawConfig("vz", config) +} func (s *Starter) SocketName() string { return "vz.sock" @@ -130,6 +132,9 @@ func (s *Starter) ResolveVersion(p *paths.Paths, requested string) (string, erro // StartVM spawns a vz-shim subprocess to host the VM. func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { + if err := s.ValidateConfig(config); err != nil { + return 0, nil, fmt.Errorf("validate vz config: %w", err) + } shimConfig := buildShimConfigFromVMConfig(config, socketPath) return s.startShim(ctx, p, version, shimConfig, 30*time.Second) } From a5f3c40b617ea6975d0067640ea16e511568bfa8 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:31:30 +0000 Subject: [PATCH 02/29] Make UEFI profile test architecture-aware --- lib/hypervisor/qemu/config_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/hypervisor/qemu/config_test.go b/lib/hypervisor/qemu/config_test.go index 890a76993..ad3995fa2 100644 --- a/lib/hypervisor/qemu/config_test.go +++ b/lib/hypervisor/qemu/config_test.go @@ -222,7 +222,11 @@ func TestProfilesValidateFirmwareAndDiskFormats(t *testing.T) { Firmware: &hypervisor.FirmwareConfig{CodePath: "/code", VarsPath: "/vars"}, Disks: []hypervisor.DiskConfig{{Path: "/disk", Format: hypervisor.DiskFormatQCOW2}}, } - assert.NoError(t, StandardProfile{}.validateConfig(uefi)) + if standardMachineType() == MachineTypeQ35 { + assert.NoError(t, StandardProfile{}.validateConfig(uefi)) + } else { + assert.ErrorContains(t, StandardProfile{}.validateConfig(uefi), "supported only by qemu/q35 on amd64") + } assert.ErrorContains(t, MicroVMProfile{}.validateConfig(uefi), "does not support uefi boot") } From 54fab7a51b336f8b7ce96619fb220e4842dbf7e1 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:04:28 +0000 Subject: [PATCH 03/29] Support Debian Secure Boot firmware paths --- .github/workflows/test.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83cad318e..ac0b4f2a8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,11 +82,22 @@ jobs: ! qemu-system-x86_64 --version >/dev/null 2>&1 || \ ! command -v qemu-img &> /dev/null || \ ! command -v swtpm &> /dev/null || \ - ! test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd || \ - ! test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd; then + ! test -d /usr/share/OVMF; then apt_update_with_retry timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables ovmf qemu-system-x86 qemu-utils swtpm fi + if test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd && test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd; then + ovmf_code=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd + ovmf_vars=/usr/share/OVMF/OVMF_VARS_4M.ms.fd + elif test -f /usr/share/OVMF/OVMF_CODE.secboot.fd && test -f /usr/share/OVMF/OVMF_VARS.ms.fd; then + ovmf_code=/usr/share/OVMF/OVMF_CODE.secboot.fd + ovmf_vars=/usr/share/OVMF/OVMF_VARS.ms.fd + else + echo "Secure Boot OVMF firmware with Microsoft-enrolled variables is unavailable" >&2 + exit 1 + fi + echo "HYPEMAN_WINDOWS_OVMF_CODE=$ovmf_code" >> "$GITHUB_ENV" + echo "HYPEMAN_WINDOWS_OVMF_VARS=$ovmf_vars" >> "$GITHUB_ENV" go mod download - name: Verify Linux test toolchain @@ -100,8 +111,8 @@ jobs: fi sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin'" done - test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd - test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd + test -f "$HYPEMAN_WINDOWS_OVMF_CODE" + test -f "$HYPEMAN_WINDOWS_OVMF_VARS" # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. From 1ae68d6b4ae3f843f86b7b5fd6aa5aba6ab13f7a Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:12:56 +0000 Subject: [PATCH 04/29] Allow loaded hosts to start Windows devices --- lib/hypervisor/qemu/process.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index aa85455cd..daf8d5b1d 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -28,8 +28,8 @@ import ( // Timeout constants for QEMU operations const ( - // socketWaitTimeout is how long to wait for QMP socket to become available after process start - socketWaitTimeout = 10 * time.Second + // socketWaitTimeout is how long to wait for QEMU and swtpm sockets after process start. + socketWaitTimeout = 30 * time.Second // migrationTimeout is how long to wait for migration to complete migrationTimeout = 30 * time.Second From 9a4cafa6df5fa53359a76fff127225ff4e655e30 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:09:42 +0000 Subject: [PATCH 05/29] Isolate the Windows hypervisor CI gate --- .github/workflows/test.yml | 11 +++++++++++ .../qemu/windows_config_integration_linux_test.go | 3 +++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac0b4f2a8..f11f61de5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,6 +114,17 @@ jobs: test -f "$HYPEMAN_WINDOWS_OVMF_CODE" test -f "$HYPEMAN_WINDOWS_OVMF_VARS" + - name: Test Windows hypervisor primitives + run: | + TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" + sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_CONFIG_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run '^TestWindowsConfigIntegration$' -timeout 2m ./lib/hypervisor/qemu + # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. - name: Login to Docker Hub diff --git a/lib/hypervisor/qemu/windows_config_integration_linux_test.go b/lib/hypervisor/qemu/windows_config_integration_linux_test.go index 2eab0ef14..0556c3b03 100644 --- a/lib/hypervisor/qemu/windows_config_integration_linux_test.go +++ b/lib/hypervisor/qemu/windows_config_integration_linux_test.go @@ -31,6 +31,9 @@ func requireWindowsConfigDependency(t *testing.T, path, description string) stri } func TestWindowsConfigIntegration(t *testing.T) { + if os.Getenv("HYPEMAN_RUN_WINDOWS_CONFIG_INTEGRATION") != "1" { + t.Skip("run by the dedicated Windows config CI gate") + } requireWindowsConfigDependency(t, "/dev/kvm", "KVM") if _, err := exec.LookPath("qemu-system-x86_64"); err != nil { requireWindowsConfigDependency(t, "", "qemu-system-x86_64") From 53c6be86a2812815687c58573a62faf9608fbc93 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:16:31 +0000 Subject: [PATCH 06/29] Retry the isolated Windows config gate --- .github/workflows/test.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f11f61de5..caf47c339 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,13 +117,19 @@ jobs: - name: Test Windows hypervisor primitives run: | TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" - sudo env \ - "PATH=$TEST_PATH" \ - "CI=true" \ - "HYPEMAN_RUN_WINDOWS_CONFIG_INTEGRATION=1" \ - "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ - "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ - go test -count=1 -run '^TestWindowsConfigIntegration$' -timeout 2m ./lib/hypervisor/qemu + for attempt in 1 2 3; do + if sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_CONFIG_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run '^TestWindowsConfigIntegration$' -timeout 2m ./lib/hypervisor/qemu; then + exit 0 + fi + test "$attempt" = 3 || sleep 5 + done + exit 1 # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. From 32eb71aa0885180a148893ed6a87d494306acb2f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:10:06 +0000 Subject: [PATCH 07/29] Address Windows hypervisor review feedback --- README.md | 3 + cmd/api/api/instances.go | 2 + lib/hypervisor/README.md | 2 + lib/hypervisor/hypervisor.go | 6 ++ lib/hypervisor/qemu/README.md | 8 +++ lib/hypervisor/qemu/config_test.go | 9 ++- lib/hypervisor/qemu/process.go | 23 +++---- lib/hypervisor/qemu/profile.go | 26 +++++--- lib/hypervisor/qemu/swtpm.go | 4 +- lib/instances/logs.go | 4 ++ lib/instances/manager.go | 4 +- lib/oapi/oapi.go | 101 +++++++++++++++-------------- lib/paths/paths.go | 5 ++ openapi.yaml | 2 +- 14 files changed, 124 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index ef1f0fe25..70d3ee66f 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,9 @@ hypeman logs --source vmm my-app # View Hypeman operational logs hypeman logs --source hypeman my-app + +# View software TPM logs for a TPM-backed QEMU guest +hypeman logs --source swtpm my-app ``` For all available commands, run `hypeman --help`. diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index adcaf05fa..c5ba06e42 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -884,6 +884,8 @@ func (s *ApiService) GetInstanceLogs(ctx context.Context, request oapi.GetInstan source = instances.LogSourceVMM case oapi.Hypeman: source = instances.LogSourceHypeman + case oapi.Swtpm: + source = instances.LogSourceSWTPM } } diff --git a/lib/hypervisor/README.md b/lib/hypervisor/README.md index 719df4f5f..cb1cf9a35 100644 --- a/lib/hypervisor/README.md +++ b/lib/hypervisor/README.md @@ -37,6 +37,8 @@ if hv.Capabilities().SupportsSnapshot { } ``` +Capabilities also describe boot requirements such as UEFI firmware and TPM support. Guest compatibility is checked against these properties instead of hard-coding a hypervisor type. Resource requirements and image policy remain guest-level validation concerns. + ## Platform Differences ### Linux (Cloud Hypervisor, QEMU) diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 947eade20..da117d8b0 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -378,6 +378,12 @@ type Capabilities struct { // SupportsVsock indicates if vsock communication is available SupportsVsock bool + // SupportsUEFIBoot indicates if firmware boot is available. + SupportsUEFIBoot bool + + // SupportsTPM indicates if a software TPM can be attached at boot. + SupportsTPM bool + // SupportsGPUPassthrough indicates if PCI device passthrough is available SupportsGPUPassthrough bool diff --git a/lib/hypervisor/qemu/README.md b/lib/hypervisor/qemu/README.md index 7229345e6..d79c81113 100644 --- a/lib/hypervisor/qemu/README.md +++ b/lib/hypervisor/qemu/README.md @@ -4,6 +4,14 @@ The `qemu` backend uses `q35` on amd64 and `virt` on arm64. These architecture-n QEMU ships many other machine models, including versioned compatibility aliases and hardware-emulation boards that do not fit Hypeman's guest contract. Hypeman intentionally does not mirror that open-ended list in its API. If another model eventually provides a useful, supportable capability profile, expose it as another hypervisor backend with explicit lifecycle and device guarantees rather than as an unchecked machine-type string. +## Firmware boot and TPM ownership + +The standard amd64 `qemu` profile supports UEFI firmware boot, Secure Boot variable storage, qcow2 disks, and software TPM 2.0 devices. These requirements are represented as runtime capabilities so callers can validate a guest's requirements without branching on the QEMU type name. Direct-kernel profiles reject firmware and TPM configuration. + +Firmware code is immutable host input. Each instance receives its own writable NVRAM file and TPM state directory, which must be preserved together with the guest disk. Hypeman starts `swtpm` first because QEMU connects to its control socket as a client; launching both processes concurrently would race that required ordering. Once the socket exists, QEMU starts immediately. + +QEMU and `swtpm` run in detached process groups and keep running if Hypeman restarts. QEMU retains its TPM control connection, and `swtpm --terminate` exits when that connection closes. Startup cleanup owns both processes as one cleanup stack so a partial boot cannot leave either process behind. TPM output is available through the instance logs API with `source=swtpm`. + ## `qemu-microvm` The `qemu-microvm` backend uses QEMU's Linux amd64-only `microvm` board. Upstream registers this board only in the x86 system emulator; `qemu-system-aarch64` does not provide an equivalent `microvm` machine. Hypeman uses direct kernel boot, `ttyS0` serial logs, and virtio-mmio transport for disks, networking, vsock, and the optional balloon. diff --git a/lib/hypervisor/qemu/config_test.go b/lib/hypervisor/qemu/config_test.go index ad3995fa2..4d4ae0595 100644 --- a/lib/hypervisor/qemu/config_test.go +++ b/lib/hypervisor/qemu/config_test.go @@ -222,10 +222,15 @@ func TestProfilesValidateFirmwareAndDiskFormats(t *testing.T) { Firmware: &hypervisor.FirmwareConfig{CodePath: "/code", VarsPath: "/vars"}, Disks: []hypervisor.DiskConfig{{Path: "/disk", Format: hypervisor.DiskFormatQCOW2}}, } + standard := StandardProfile{} if standardMachineType() == MachineTypeQ35 { - assert.NoError(t, StandardProfile{}.validateConfig(uefi)) + assert.True(t, standard.capabilities().SupportsUEFIBoot) + assert.True(t, standard.capabilities().SupportsTPM) + assert.NoError(t, standard.validateConfig(uefi)) } else { - assert.ErrorContains(t, StandardProfile{}.validateConfig(uefi), "supported only by qemu/q35 on amd64") + assert.False(t, standard.capabilities().SupportsUEFIBoot) + assert.False(t, standard.capabilities().SupportsTPM) + assert.ErrorContains(t, standard.validateConfig(uefi), "does not support UEFI boot on this host") } assert.ErrorContains(t, MicroVMProfile{}.validateConfig(uefi), "does not support uefi boot") } diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index daf8d5b1d..c3f59e8e5 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -28,8 +28,9 @@ import ( // Timeout constants for QEMU operations const ( - // socketWaitTimeout is how long to wait for QEMU and swtpm sockets after process start. - socketWaitTimeout = 30 * time.Second + // qemuSocketWaitTimeout bounds QMP startup on a heavily loaded host. Process + // exit is checked on every poll, so deterministic startup failures return early. + qemuSocketWaitTimeout = 30 * time.Second // migrationTimeout is how long to wait for migration to complete migrationTimeout = 30 * time.Second @@ -365,7 +366,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version // Wait for socket to be ready socketWaitStart := time.Now() - if err := waitForSocketOrExit(socketPath, socketWaitTimeout, proc); err != nil { + if err := waitForSocketOrExit(socketPath, qemuSocketWaitTimeout, proc); err != nil { processSpan.RecordError(err) processSpan.SetStatus(codes.Error, err.Error()) cu.Clean() @@ -426,22 +427,22 @@ func (s *Starter) validateSnapshotMachineType(stored MachineType) (MachineType, } func (s *Starter) startConfiguredProcess(ctx context.Context, p *paths.Paths, version, socketPath string, config hypervisor.VMConfig, args []string) (int, *QEMU, *cleanup.Cleanup, error) { + cu := cleanup.Make(func() {}) tpmProcess, err := startSWTPM(config.TPM, filepath.Dir(socketPath)) if err != nil { return 0, nil, nil, err } + if tpmProcess != nil { + cu.Add(tpmProcess.cleanup) + } - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, qemuCleanup, err := s.startQEMUProcess(ctx, p, version, socketPath, args) if err != nil { - if tpmProcess != nil { - tpmProcess.cleanup() - } + cu.Clean() return 0, nil, nil, err } - if tpmProcess != nil { - cu.Add(tpmProcess.cleanup) - } - return pid, hv, cu, nil + cu.Add(qemuCleanup.Clean) + return pid, hv, &cu, nil } // StartVM launches QEMU with the VM configuration and returns a Hypervisor client. diff --git a/lib/hypervisor/qemu/profile.go b/lib/hypervisor/qemu/profile.go index 565ab3a70..07485b849 100644 --- a/lib/hypervisor/qemu/profile.go +++ b/lib/hypervisor/qemu/profile.go @@ -25,16 +25,14 @@ func (StandardProfile) machineType() (MachineType, error) { return standardMachineType(), nil } func (StandardProfile) capabilities() hypervisor.Capabilities { - return qemuCapabilities(true) + supportsFirmware := standardMachineType() == MachineTypeQ35 + return qemuCapabilities(true, supportsFirmware) } -func (StandardProfile) validateConfig(cfg hypervisor.VMConfig) error { +func (p StandardProfile) validateConfig(cfg hypervisor.VMConfig) error { if err := hypervisor.ValidateBootConfig(cfg); err != nil { return err } - if cfg.EffectiveBootMode() == hypervisor.BootModeUEFI && standardMachineType() != MachineTypeQ35 { - return fmt.Errorf("UEFI boot is currently supported only by qemu/q35 on amd64") - } - return nil + return validateProfileCapabilities(p.hypervisorType(), p.capabilities(), cfg) } func (StandardProfile) requiresStoredMachineType() bool { return false } func (StandardProfile) requiresStoredVersion() bool { return false } @@ -48,7 +46,7 @@ func (MicroVMProfile) machineType() (MachineType, error) { return microVMMachineType() } func (MicroVMProfile) capabilities() hypervisor.Capabilities { - return qemuCapabilities(false) + return qemuCapabilities(false, false) } func (MicroVMProfile) validateConfig(cfg hypervisor.VMConfig) error { if err := hypervisor.ValidateDirectRawConfig("qemu-microvm", cfg); err != nil { @@ -76,7 +74,17 @@ func (MicroVMProfile) validateConfig(cfg hypervisor.VMConfig) error { func (MicroVMProfile) requiresStoredMachineType() bool { return true } func (MicroVMProfile) requiresStoredVersion() bool { return true } -func qemuCapabilities(supportsPCI bool) hypervisor.Capabilities { +func validateProfileCapabilities(name hypervisor.Type, caps hypervisor.Capabilities, cfg hypervisor.VMConfig) error { + if cfg.EffectiveBootMode() == hypervisor.BootModeUEFI && !caps.SupportsUEFIBoot { + return fmt.Errorf("%s does not support UEFI boot on this host", name) + } + if cfg.TPM != nil && !caps.SupportsTPM { + return fmt.Errorf("%s does not support TPM devices", name) + } + return nil +} + +func qemuCapabilities(supportsPCI, supportsFirmware bool) hypervisor.Capabilities { return hypervisor.Capabilities{ SupportsSnapshot: true, // PrepareFork rewrites the saved QEMU VM config for forks (fork.go); @@ -86,6 +94,8 @@ func qemuCapabilities(supportsPCI bool) hypervisor.Capabilities { SupportsBalloonControl: true, SupportsPause: true, SupportsVsock: true, + SupportsUEFIBoot: supportsFirmware, + SupportsTPM: supportsFirmware, SupportsGPUPassthrough: supportsPCI, SupportsDiskIOLimit: true, SupportsGracefulVMMShutdown: true, diff --git a/lib/hypervisor/qemu/swtpm.go b/lib/hypervisor/qemu/swtpm.go index e85147c02..68c409539 100644 --- a/lib/hypervisor/qemu/swtpm.go +++ b/lib/hypervisor/qemu/swtpm.go @@ -11,6 +11,8 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" ) +const swtpmSocketWaitTimeout = 30 * time.Second + func startSWTPM(config *hypervisor.TPMConfig, instanceDir string) (*startedProcess, error) { if config == nil { return nil, nil @@ -56,7 +58,7 @@ func startSWTPM(config *hypervisor.TPMConfig, instanceDir string) (*startedProce if err != nil { return nil, fmt.Errorf("start swtpm: %w", err) } - if err := waitForSocketFileOrExit(config.SocketPath, socketWaitTimeout, proc); err != nil { + if err := waitForSocketFileOrExit(config.SocketPath, swtpmSocketWaitTimeout, proc); err != nil { proc.cleanup() if logData, readErr := os.ReadFile(logPath); readErr == nil && len(logData) > 0 { const maxLogBytes = 32 << 10 diff --git a/lib/instances/logs.go b/lib/instances/logs.go index f10e8537d..54b280f81 100644 --- a/lib/instances/logs.go +++ b/lib/instances/logs.go @@ -24,6 +24,8 @@ const ( LogSourceVMM LogSource = "vmm" // LogSourceHypeman is the hypeman operations log LogSourceHypeman LogSource = "hypeman" + // LogSourceSWTPM is the software TPM log for TPM-backed QEMU guests. + LogSourceSWTPM LogSource = "swtpm" ) // ErrTailNotFound is returned when the tail command is not available @@ -73,6 +75,8 @@ func (m *manager) streamInstanceLogs(ctx context.Context, id string, tail int, f logPath = m.paths.InstanceVMMLog(id) case LogSourceHypeman: logPath = m.paths.InstanceHypemanLog(id) + case LogSourceSWTPM: + logPath = m.paths.InstanceSWTPMLog(id) default: // Default to app log for backwards compatibility logPath = m.paths.InstanceAppLog(id) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 5d595d823..0568ff4ba 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -781,7 +781,7 @@ func (m *manager) StreamInstanceLogs(ctx context.Context, id string, tail int, f return m.streamInstanceLogs(ctx, id, tail, follow, source) } -// RotateLogs rotates all instance logs (app, vmm, hypeman) that exceed maxBytes +// RotateLogs rotates all instance logs that exceed maxBytes func (m *manager) RotateLogs(ctx context.Context, maxBytes int64, maxFiles int) error { instances, err := m.listInstances(ctx) if err != nil { @@ -790,11 +790,11 @@ func (m *manager) RotateLogs(ctx context.Context, maxBytes int64, maxFiles int) var lastErr error for _, inst := range instances { - // Rotate all three log types logPaths := []string{ m.paths.InstanceAppLog(inst.Id), m.paths.InstanceVMMLog(inst.Id), m.paths.InstanceHypemanLog(inst.Id), + m.paths.InstanceSWTPMLog(inst.Id), } for _, logPath := range logPaths { if err := rotateLogIfNeeded(logPath, maxBytes, maxFiles); err != nil { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 52ab3b2c0..938c49b4f 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -278,6 +278,7 @@ const ( const ( App GetInstanceLogsParamsSource = "app" Hypeman GetInstanceLogsParamsSource = "hypeman" + Swtpm GetInstanceLogsParamsSource = "swtpm" Vmm GetInstanceLogsParamsSource = "vmm" ) @@ -19143,56 +19144,56 @@ var swaggerSpec = []string{ "tKZlwhDXb5mml8frtVFfn53BR4j9h1VQL4+Jr4da3B/avlUF9LOzSKg25BcHU7hTKOOwo5eGWn2zmN+u", "g/orsanHIgT7J9i1a5DPyGUFAfCyBZ7K89uf5fyTCWP99ooCOBcjiVMdgTaZiHttMRY8CYP/7Y9GIaDr", "jkCEOIyPjEO4Npif5byoZlAjZZplXcnXDROoeJmmG2iY7FRQzbSJZW7+ok3MlIKPHXW3ETfZoZGrZEWv", - "LKE6DDt/sHeB/IKRRAgvHlwqy1R7/R4Tedo7/tX9a5mmvX7PjacCS34D4X4LoGOzwfWIF7szFdTGr2L5", - "TfAY68y+AsjYuDmcOt0ukb/AF/70nkVvs/uEZAjyQcOI+zmJoJXx1g0+QhbAkl8WepQzNTWktnZTjZ/l", - "wA4vzn25nS4hHBfu0wv/5Weg/W6L7PBjJn66dx7isT6CLzkTVq/NZiZVE3JoW+zHZ09IH25L1qbahUK+", - "0ubN7XydCNNK6usswn0QY/k1mhuZUsMjKP0TLaTUFbIv8IGxSJcz3xaUCcYN1DNdCP2lJdVLZwi+dIL8", - "sTNaEVp95PoYwucu8D78hX9UfvFDRS8vOH7fC98Ajw+1yRVnM5LRXDMrV+UpI9EqslwRaz0xGi1IRDOT", - "KwZl7BhJueBpnlaBn+2OLSmAVFzup5d9Ms0NSaiag16EDzGcXrFIpikTMQML2VgsGF1yq9QpklDDRLQa", - "aAblb5eMXEt1lUgag5LvonCwfJ5ilgIBRTtlhsbUUBA1Lu2Jn2AWz2VRERcVa8HelNQQj4XKxbcI6W+b", - "vfQDvSQMMKu5XhSVEyMaMxEFsZwvPm829uGtwRfMNCf6ieJybsVLP2WgTtXq6YfzecTwfGHByFK5bezC", - "5jcIvbpdiaynP3gy+vc80jhXP8dP5OIplnjTKf48fDsF0X02/p1P78CRisQ5dlc5lUDmf1avTMFQquFO", - "kFqJ23hb10xRIq5Y5hvxvL0//J+nt7CmfSacsN+q2LcVIyon/TmwXLeqt+K5n8iM6GxJFavYJ2TBPqbq", - "k4lPUlW43Jdi7nQMG49mwber3MkoCtqXFF/ZdpNtu5CD27Jtb5tdc6pXGDkXA4jSDHNwZ8ZtZdXOdPBv", - "mgvSmF2FZX5yFln6Du6MLZ4WjBBZY0ZXiaTxnyFMd4MHJ5JKIf4DIEp8SfijFathNUAfbHNllbO+z618", - "fXa228YllNnII5T5gjlEJSnGfpYGSv8/XzKleOxgOsnJ2RMXMMs1UbkYkucpN8RIcsVYVuaUAJDH0M7P", - "I2GsV1SvQV70e0wYtcokF2brKMpXP85g3t2qDvsd80kHaP3VId3ZIQ2W/S+PnQGXgawJnMBmzdRQs7XQ", - "JhczqVKUy+hU5rZ1y4PsMtn9xIKVM54wvdKGpRgXOMsTOG5QHMEVwHXf4S73ISrWnhxMWMuYSrnWXAo9", - "Fi5bI2PK9m0/t+1XQpyCDgFDC/56jkzy8wifs4PBiDFq2lYNMIugsGbvuLdHs2wvpoa2hGi54b3HkH6A", - "eDiiV+lUJjwiCRdXmuwk/ArVE7LUJLF/7G4MqJvAdx+6vO/tT5Zd6VMxk8HiaUizBTH/qfKqHFvzjskv", - "jq09Y9XD4vkPbHSYrW0vIKwYTQZQkNcj15Dc8IS/RVZnG+Ha8AiTfkrIgtdnJWrBWJwxo+w7FJLLkoRF", - "xhts9jIlo71xPhodRhkH+LNDBoMDhtf+OIUeT85fYSIoS6Va9cfC/gMafvn4HL27M+qsCZWBusrB5HTv", - "+ZYQ4wtYpn/jGD2c4EbsgOCGf3UJ3hwRpPUM6ZYjKrNNqpLM/vRBpE6C+2pX+DLtCgDJVMxmZ65oBEKx", - "XuQmltcibENYyiRP7T/wj9NtwF6GRovX8OpnI+3icLZ24yf4RRxKN6eYYXHHT+L0wAX7UmNW7cL5KYAQ", - "U4sGDN4Cj82fkbo/vPm+uo6fobvTragvnPrZnK27vvncGDzGRXU9vpRjjpTmZ2LkZuvTNeXt1qfvExld", - "aQeGUjUbWr0NAMbtjyUgtHMRgpgAuZnEgQgR9ibjCpDfGgZIxNzRhBLDVMoFTfZgztgIQFt7KxZdSg4p", - "0lHCIUmNx4BalAA63fWCCWJnA4Yq30DFo6tdaanqO1VnpJFkyiKZMg/3vRtS3f5GuflBqjp29+fCF19W", - "1t/Ox07VznMLXHl7j+8FX35G30CodJw7h7If0c4zWf6IpqA+gb0Z9w5Hetzrk3HvIB337A6cUDChUkPu", - "kZSL3DA9JE/QvgVJsPdHRLNIilh71HFvwTsc6baUWCTLlvzK+/DdXYo9jqpgKV+4TkLswb5H7PeQtEN2", - "qgfOncm4D4cuJjI3aO5358q9FTMD5pHdO/fVVs7IV92+Cyf/mzu+NR4Fu2zZZWXrkbNnuV6wdpPbz1jJ", - "JzdTQLP21TX1gvxdTnWfCHaN1nClzXCN79mvz7GDu0Dat13dBGXfzf0rxH4HiP1yrcJwiRhgaa9kTx2I", - "mcjeICIs9WnvjoZAkwDsBhnRhDw/OR2LyLIiBPdTLJXAnRwgON7Cj/92QZ6evOiTJ1DpkfyYT3eH5LlI", - "Vr7eNvpoxgIlMWReERVkilTL4tD1jGMH6vmYweK2g09UOhlPRsCz4vfKB4n3ewtGY5BI/uj9LLGzAEbw", - "i5/tAbL7574str23UfjovWBGrQaPZ4ap9WbPXJ6UKFAr3CXtQeCc4IbQk7ZD7bHPyj5RNkBwisODXgCr", - "4t3Xqgcfv0Lo3XjJME4E681Nc8D6FJBkQOPVlxXLpBekYI4hFli9rou6AW1Zwo6XbVQwoMu2yO/PyOS+", - "kXfVkOD/XU8XzPSLdTRltX2yRFzUG9nq6fXJwQsEJHaOqohmNOJm1Sc0Sdwd5W6CIiJlUIi/U8XoVSyv", - "xXAsXhSVTlxCLzk5f9X3jloSc32FLThf7JA8XzKl82kxOAIHDb3GsOYsHgsjSUSTKE+suMFmMxZBLi4U", - "MNEtvtxiKL2PeHbKToLVVipR7fkXV+QtTBOweyVZNCluD7d6T7EooTxth/92ghoEHEKowdQ2KgXhYpa4", - "kKpISa2Ja2rAEj7n08QFCOkheblgRNOUjUWWUCGYIrnGqHg79EGmmNY5JnjbBgAmFymqT0pov0xJ40IT", - "EimVxmgCS+Gvz4g2LNtAZi+w5TOY80eSbbFx19MnMlI3xtBuCnGvELshSCm44JaO8sQHMN5pKDoO6FNL", - "iV/KwX+p+HzOlD0VFJkshuPhsfbLiYe+lrHcWvDxonirW8HHotVKVmIlY28jNNukRLuOezeL+gt0fsVb", - "0fvco5tlEf9kP+rYdz1bNTwI9+g9Z/lnqaN/UUkS7GrAKin8SzMnVUZeO6q1RNvtsFqdM2s/ZqZrZ/ys", - "Twab9SWjZdFa+mybwvv5EcLoblEe7rok2pdNWzW0q5pu2pLyvx3P/rOgwI8DZP+JUU5uAWT/WeXdA9L4", - "p8M/CR7UT5VHX/M9+2qzf3os+o+VPo+A9ADH1pY+j1zPBa9uVJReu3e6qUmuxT+TBO/iHW8gv/tl/6r1", - "d1AZKou1zQVtCZ6lmVn5gDbnqyyDzjR/y4YtjuAibvXjuYJvEdL54cjD02lrQOefszj8J4kZdcX7uCan", - "TwJV178wjMHqmatdLHv21hlQFS34krUb3esn2C1Rptggkxk4V2JcMLce/i4zVA3nb4lr3mGuun9B9UcA", - "y2cxiblikUlWWInTcgTs4xtNlLSaADyXatUeJYJH5Acl08duNlvuQ3emnDGsjDNMV4OYGjpYem6zwYT2", - "HtGdPp7SMjzCBXn2Pdlhb4zCGhNkZjUfwmfFkmK5fQ00uVsd8P6oxbLJ37LJfNpllBuqhTx31VhIlGsj", - "U7/3p0/IDlQfmzNh98KK+jOQZDMllzxmcW2MvaVMcFX3Wxb0pnZXK1QUpeO8coGD+yQyTJcLaf6WZ3W2", - "UITETLmgMLitdTnqZwqT+G1/lAsfgOP2yI/i6xXmNL8dr+xYSoRKmG4RjZQI8bz79Zr7kq+5ajKUv9Nq", - "t50Pz9lsvO6WH9UxbeljFH4ocufu1mz9+vNJ6eH6i8zmcabzZaGQtpnNPy8SHN3d/XDX5vLXX3AK6DPm", - "le+KqRwasC2GCOZniOmO2ZIlMkuhIjm82+v3cpX0jnsLY7LjvT2I/V5IbY6PHj047L377d3/HwAA//9w", - "eSdugOoBAA==", + "LKE6DDt/sHeB/IKRRAgvHlwqy1R7/R4Tedo7/tX9a5mmvX7Pjcd+d22ytAJPfgMhfwuwY7PB9cgXu0MV", + "9Mav4vlNcBnrTL8CzNi4QZxa3S6Zv8AX/vQeRm+7+4RkCHJCw5j7OYmilfHWDT9CFgCTXxaKlDM5NaS3", + "dpONn+XADi/OfdmdLqEcF+7TC//lZ6AFb4vw8GMmfrp3HuqxPoIvOSNWr81mJlUTemhbDMhnT0gfbkvW", + "ptqFQr7S5s3tfZ0I00rs6yzCfRBjGTaaG5lSwyMoARQtpNQVsi9wgrFYlzPjFpQJRg7UN10o/aUl1Utn", + "EL50Av2xM14RWn3k+hjC5y4AP/yFf1R+8UNFPy84ft8L4QCTDzXKFWczktFcMytX5Skj0SqyXBFrPjEa", + "LUhEM5MrBuXsGEm54GmeVgGg7Y4tKYBVXO6nl30yzQ1JqJqDfoQPMaxesUimKRMxA0vZWCwYXXKr3CmS", + "UMNEtBpoBmVwl4xcS3WVSBqDsu+icbCMnmKWAgFNO2WGxtRQEDUu7YmfYDbPZVEZFxVswd6U1BCPhcrF", + "twjtb5u99AO9JAywq7leFBUUIxozEQUxnS8+bzb24a3CF8w0J/qJ4nNuxUs/ZcBO1frph/N5xPJ8YUHJ", + "Urlt7MLmNwi9ul2JrKdBeDL69zzSOFc/x0/k6imWeNMp/jx8PAXRfTZ+nk/vyJGKxDl2VzmVQOZ/Vu9M", + "wVCqYU+QYonbeFsXTVEqrljmG/G8vT/8n6e3sKZ9Jpyw36rYtxUlKif9ObBct6q34rmfyIzobEkVq9gn", + "ZME+tuqTiU9SVbjcl2LudAwbj2bBt6vcySgK2pcUX9l2k2270IPbsm1vm11zrlcYORcDiNYMc3Bnxm1l", + "1c508G+aE9KYXYVlfnIWWfoO7owtnhaMEFljRleJpPGfIVx3gwcnkkohDgQgS3xJOKQVq2E1UB9sc2W1", + "s77PsXx9drbbxiWU2cgjlPmCOUQlOcZ+lsbrBtznS6YUjx1cJzk5e+ICZ7kmKhdD8jzlhhhJrhjLytwS", + "APQY2vl5RIz1yuo16It+jwmjVpnkwmwdRfnqxxnMu1vVY79jPumArb86pDs7pMGy/+WxM+AykD2BE9is", + "mRpqthbc5GImVYpyGZ3K3LZueZBdJrufWLhyxhOmV9qwFOMDZ3kCxw2KJLhCuO473OU+RMfak4OJaxlT", + "KdeaS6HHwmVtZEzZvu3ntv1KqFPQIWBowV/PkUl+HmF0djAYOUZN26oBdhEU2Owd9/Zolu3F1NCWUC03", + "vPcY0g8QF0f0Kp3KhEck4eJKk52EX6F6QpaaJPaP3Y2BdRP47kOX+b39ybIrfSpmMlhEDWm2IOY/VX6V", + "Y2veMfnFsbVnrHpYPP+BjQ6zte2FhBWjyQAK83oEG5IbnvC3yOpsI1wbHmHyTwld8PqsRC8YizNmlH2H", + "QpJZkrDIeIPNXqZktDfOR6PDKOMAg3bIYHDA8Nofp9DjyfkrTAhlqVSr/ljYf0DDLx+fo3d3Rp01oTJQ", + "V0GYnO493xJqfAHL9G8co4cT3IghENzwry7BmyODtJ4h3XJEZbZJVZLZnz6I1ElwX+0KX6ZdAaCZitns", + "zBWNQCjWi9zE8lqEbQhLmeSp/Qf+cboN4MvQaPEaXv1spF0cztZu/AS/iEPp5hQzLPL4SZweuGBfasyq", + "XTg/BRBiatGAwVvgsfkzUveHN99X1/EzdHe6FfUFVD+bs3XXN58bg8e6qK7Hl3LMkdL8TIzcbH26przd", + "+vR9IqMr7UBRqmZDq7cB0Lj9sQSGdi5CEBMgR5M4MCHC3mRcAQJcwwCJ2DuaUGKYSrmgyR7MGRsBiGtv", + "xaJLySFVOko4JKnxGNCLEkCpu14wQexswFDlG6h4dLUrMVV9p+qMNJJMWSRT5mG/d0Oq298oNz9IVcfw", + "/lz44svK+tv52KnaeW6BLW/v8b1gzM/oGwiVjnPnUPYj2nkmyx/RFNQnsDfj3uFIj3t9Mu4dpOOe3YET", + "CiZUasg9knKRG6aH5AnatyAZ9v6IaBZJEWuPPu4teIcj3ZYai2TZkmd5H767S7HHURUs5QvXSYg92PeI", + "/R6SdshO9cC5Mxn34dDFROYGzf3uXLm3YmbAPLJ7577ayhn5qtt34eR/c8e3xqNgly27rGw9cvYs1wvW", + "bnL7GSv65GYKqNa+yqZekL/Lqe4Twa7RGq60Ga7xPfv1OXZwF4j7tquboO27uX+F2u8AtV+uVRg2EQMs", + "7ZXsqQOxE9kbRIalPv3d0RBoEoDhICOakOcnp2MRWVaEIH+KpRK4kwMGx1v48d8uyNOTF33yBCo+kh/z", + "6e6QPBfJytfdRh/NWKAkhswrooJMkWpZHLqecexAPR8zWNx28IlKKOPJCHhW/F75IPF+b8FoDBLJH72f", + "JXYWwAp+8bM9QHb/3JfFtvc2Ch+9F8yo1eDxzDC13uyZy5MSBXqFu6Q9GJwT3BCC0naoPQZa2SfKBghS", + "cXjQC2BWvPta/eDjVwq9Gy8Zxolg3blpDpifApIMaLz6smKZ9IIUzDHEAqvXdVE/oC1L2PGyjQoGdNkW", + "+f0Zmdw38q4aIvy/6+mCmX6xjqastk+WiIu6I1s9vT45eIHAxM5RFdGMRtys+oQmibuj3E1QRKQMCvF3", + "qhi9iuW1GI7Fi6LiiUvoJSfnr/reUUtirq+wBeeLHZLnS6Z0Pi0GR+CgodcY1pzFY2EkiWgS5YkVN9hs", + "xiLIxYVCJrrFl1sMpfcRz07ZSbDqSiWqPf/iir2FaQJ2rySLJsXt4VbvKRYllKftMOBOUIOAQwg1mNpG", + "pSBczBIXUhUpqTVxTQ1Ywud8mrgAIT0kLxeMaJqyscgSKgRTJNcYFW+HPsgU0zrHBG/bAMDlIkX1SQnx", + "lylpXGhCIqXSGE1gKfz1GdGGZRvI7AW2fAZz/kiyLTbuevpERurGGNpNIe4VYjcEKQUX3NJRnvgAxjsN", + "RccBfWop8Us5+C8Vn8+ZsqeCIpPFcDw81n458dDXMpZbCz9eFG91K/xYtFrJSqxk7G2EaJuUqNdx72ZR", + "f4HOr3grip97dLMs4p/sRx37rmerhgfhHr3nLP8s9fQvKkmCXQ1YJYV/aeakyshrR7WWaLsdVqtzZu3H", + "zHTtjJ/1yWCzvmS0LFpLn21TeD8/QhjdLcrDXZdG+7Jpq4Z2VdNNW1L+t+PafxYU+HEA7T8xysktAO0/", + "q7x7QBz/dPgnwYP6qfLoa75nX3X2T49J/7HS5xGYHuDY2tLnkeu54NWNitJr9043Ncm1+GeS4F284w3k", + "d7/sX7X+DipDZbG2uaAtwbM0Mysf0OZ8lWXQmeZv2bDFEVzErX48V/AtQjo/HHl4Om0N6PxzFon/JDGj", + "rogf1+T0SaD6+heGMVg9c7WLZc/eOgOqogVfsnaje/0EuyXKFBtkMgPnSowL5tbD32WGquH8LXHNO8xV", + "9y+oAglg+SwmMVcsMskKK3JajoB9fKOJklYTgOdSrdqjRPCI/KBk+tjNZst96M6UM4aVcYbpahBTQwdL", + "z202mNDeI7rTx1Nahke4IM++JzvsjVFYa4LMrOZD+KxYUiy7r4Emd6sD3h+1WDb5WzaZT7uMckPVkOeu", + "KguJcm1k6vf+9AnZgSpkcybsXlhRfwaSbKbkkscsro2xt5QJrup+y4Le1O5qhYqihJxXLnBwn0SG6XIh", + "zd/yrM4WipCYKRcUBre1Lkf9TGESv+2PcuEDcNwe+VF8vcKc5rfjlR1LiVAR0y2ikRIhnne/XnNf8jVX", + "TYbyd1rttvPhOZuN193yozqmLX2Mwg9F7tzdmq1ffz4pPVx/kdk8znS+LBTSNrP550WCo7u7H+7aXP76", + "C04Bfca88l0xlUMDtsUQwfwMMd0xW7JEZilUJod3e/1erpLecW9hTHa8twex3wupzfHRoweHvXe/vfv/", + "AwAA//+bMKlSiOoBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index add23ff52..7e20cba38 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -241,6 +241,11 @@ func (p *Paths) InstanceHypemanLog(id string) string { return filepath.Join(p.InstanceLogs(id), "hypeman.log") } +// InstanceSWTPMLog returns the path to the instance software TPM log. +func (p *Paths) InstanceSWTPMLog(id string) string { + return filepath.Join(p.InstanceLogs(id), "swtpm.log") +} + // InstanceSnapshots returns the path to instance snapshots directory. func (p *Paths) InstanceSnapshots(id string) string { return filepath.Join(p.InstanceDir(id), "snapshots") diff --git a/openapi.yaml b/openapi.yaml index b117d2035..cd92d81bf 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -3570,7 +3570,7 @@ paths: required: false schema: type: string - enum: [app, vmm, hypeman] + enum: [app, vmm, hypeman, swtpm] default: app description: | Log source to stream: From 6b63aca10cccf072b2694d39344ebc5207b25bba Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 23:41:34 +0000 Subject: [PATCH 08/29] Rewrite Windows state paths for QEMU forks --- lib/hypervisor/qemu/fork.go | 8 ++++++++ lib/hypervisor/qemu/fork_test.go | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go index 1141b7f4d..4599c033e 100644 --- a/lib/hypervisor/qemu/fork.go +++ b/lib/hypervisor/qemu/fork.go @@ -89,6 +89,14 @@ func rewriteQEMUConfigPaths(cfg hypervisor.VMConfig, sourceDir, targetDir string cfg.VsockSocket = replace(cfg.VsockSocket) cfg.KernelPath = replace(cfg.KernelPath) cfg.InitrdPath = replace(cfg.InitrdPath) + if cfg.Firmware != nil { + cfg.Firmware.CodePath = replace(cfg.Firmware.CodePath) + cfg.Firmware.VarsPath = replace(cfg.Firmware.VarsPath) + } + if cfg.TPM != nil { + cfg.TPM.SocketPath = replace(cfg.TPM.SocketPath) + cfg.TPM.StateDir = replace(cfg.TPM.StateDir) + } return cfg } diff --git a/lib/hypervisor/qemu/fork_test.go b/lib/hypervisor/qemu/fork_test.go index 833b1fdc8..81f074e81 100644 --- a/lib/hypervisor/qemu/fork_test.go +++ b/lib/hypervisor/qemu/fork_test.go @@ -34,7 +34,16 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { VsockSocket: sourceDir + "/vsock/vsock.sock", KernelPath: sourceDir + "/kernel/vmlinuz", InitrdPath: sourceDir + "/kernel/initrd", - KernelArgs: "console=ttyS0 root=" + sourceDir + "/rootfs note=keep-" + sourceDir + "-as-substring", + Firmware: &hypervisor.FirmwareConfig{ + CodePath: sourceDir + "/OVMF_CODE.fd", + VarsPath: sourceDir + "/OVMF_VARS.fd", + SecureBoot: true, + }, + TPM: &hypervisor.TPMConfig{ + SocketPath: sourceDir + "/swtpm.sock", + StateDir: sourceDir + "/tpm", + }, + KernelArgs: "console=ttyS0 root=" + sourceDir + "/rootfs note=keep-" + sourceDir + "-as-substring", Disks: []hypervisor.DiskConfig{ {Path: sourceDir + "/overlay.raw"}, {Path: "/volumes/volume-data.raw"}, @@ -77,6 +86,12 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { assert.Equal(t, targetDir+"/logs/fork-app.log", updated.SerialLogPath) assert.Equal(t, targetDir+"/kernel/vmlinuz", updated.KernelPath) assert.Equal(t, targetDir+"/kernel/initrd", updated.InitrdPath) + require.NotNil(t, updated.Firmware) + assert.Equal(t, targetDir+"/OVMF_CODE.fd", updated.Firmware.CodePath) + assert.Equal(t, targetDir+"/OVMF_VARS.fd", updated.Firmware.VarsPath) + require.NotNil(t, updated.TPM) + assert.Equal(t, targetDir+"/swtpm.sock", updated.TPM.SocketPath) + assert.Equal(t, targetDir+"/tpm", updated.TPM.StateDir) assert.Equal(t, initial.KernelArgs, updated.KernelArgs) assert.Equal(t, targetDir+"/overlay.raw", updated.Disks[0].Path) assert.Equal(t, "/volumes/volume-data.raw", updated.Disks[1].Path, "non-instance paths should remain unchanged") From 00831431a4fa50153338441b268bf21b7917150f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:26:42 +0000 Subject: [PATCH 09/29] Add OCI Windows machine images --- .github/workflows/test.yml | 4 + docs/windows-images.md | 21 ++ lib/images/machine.go | 286 ++++++++++++++++++ lib/images/machine_oci_integration_test.go | 113 +++++++ lib/images/machine_test.go | 188 ++++++++++++ lib/images/manager.go | 46 ++- lib/images/manager_test.go | 2 +- lib/images/platform.go | 26 +- lib/images/platform_test.go | 10 +- lib/images/storage.go | 8 + lib/images/types.go | 1 + lib/instances/create.go | 93 ++++-- lib/instances/create_image.go | 2 +- lib/instances/create_image_test.go | 23 ++ lib/instances/start.go | 28 +- lib/instances/windows.go | 157 ++++++++++ .../windows_images_integration_linux_test.go | 128 ++++++++ lib/instances/windows_test.go | 70 +++++ lib/paths/paths.go | 20 ++ 19 files changed, 1150 insertions(+), 76 deletions(-) create mode 100644 docs/windows-images.md create mode 100644 lib/images/machine.go create mode 100644 lib/images/machine_oci_integration_test.go create mode 100644 lib/images/machine_test.go create mode 100644 lib/instances/windows.go create mode 100644 lib/instances/windows_images_integration_linux_test.go create mode 100644 lib/instances/windows_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index caf47c339..4e5988a68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,6 +113,10 @@ jobs: done test -f "$HYPEMAN_WINDOWS_OVMF_CODE" test -f "$HYPEMAN_WINDOWS_OVMF_VARS" + test -r /ci/windows/base.raw + test -r /ci/windows/persona.qcow2 + qemu-img info --output=json /ci/windows/persona.qcow2 \ + | jq -e '.format == "qcow2" and .["backing-filename-format"] == "raw"' >/dev/null - name: Test Windows hypervisor primitives run: | diff --git a/docs/windows-images.md b/docs/windows-images.md new file mode 100644 index 000000000..98ba2b527 --- /dev/null +++ b/docs/windows-images.md @@ -0,0 +1,21 @@ +# Windows machine images + +Hypeman accepts Windows desktop disks as OCI images for `windows/amd64`. Ordinary Windows container images are not bootable and are rejected. + +A machine image uses these OCI config labels: + +| Label | Base | Persona | +|---|---|---| +| `io.hypeman.machine-image.version` | `1` | `1` | +| `io.hypeman.machine-image.kind` | `windows-base` | `windows-persona` | +| `io.hypeman.machine-image.disk-path` | relative path to the source disk | relative path to a qcow2 delta | +| `io.hypeman.machine-image.disk-format` | `raw`, `qcow2`, `vhd`, or `vhdx` | `qcow2` | +| `io.hypeman.machine-image.base` | omitted | digest-pinned base reference | +| `io.hypeman.machine-image.tpm` | `2.0` | `2.0` | +| `io.hypeman.machine-image.secure-boot` | `required` | `required` | + +Hypeman materializes the base as immutable sparse raw. It rewrites the persona's qcow2 backing header to the cache-owned base path, ignoring any artifact-supplied backing path. At instance creation, Hypeman reflink-clones the immutable persona into a writable `windows.qcow2`; the clone remains backed directly by the raw base. + +The base must be pulled before its personas. A base cannot be deleted while any cached persona references its digest. + +Windows installation media, activation material, credentials, and generated disks belong in private registries and must not be committed to this repository. diff --git a/lib/images/machine.go b/lib/images/machine.go new file mode 100644 index 000000000..c56a62c5e --- /dev/null +++ b/lib/images/machine.go @@ -0,0 +1,286 @@ +package images + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/paths" +) + +const ( + MachineImageVersionLabel = "io.hypeman.machine-image.version" + MachineImageKindLabel = "io.hypeman.machine-image.kind" + MachineImageDiskPathLabel = "io.hypeman.machine-image.disk-path" + MachineImageDiskFormatLabel = "io.hypeman.machine-image.disk-format" + MachineImageBaseLabel = "io.hypeman.machine-image.base" + MachineImageTPMLabel = "io.hypeman.machine-image.tpm" + MachineImageSecureBootLabel = "io.hypeman.machine-image.secure-boot" + + MachineImageVersion = "1" +) + +type MachineImageKind string + +const ( + MachineImageWindowsBase MachineImageKind = "windows-base" + MachineImageWindowsPersona MachineImageKind = "windows-persona" +) + +// MachineImage describes a bootable disk artifact. The OCI manifest remains +// the distribution envelope; this metadata controls materialization. +type MachineImage struct { + Kind MachineImageKind `json:"kind"` + DiskPath string `json:"disk_path"` + DiskFormat string `json:"disk_format"` + Base string `json:"base,omitempty"` + TPM string `json:"tpm"` + SecureBoot string `json:"secure_boot"` + VirtualSize int64 `json:"virtual_size"` +} + +func parseMachineImage(meta *containerMetadata) (*MachineImage, error) { + version := strings.TrimSpace(meta.Labels[MachineImageVersionLabel]) + if version == "" { + if strings.EqualFold(meta.OS, "windows") { + return nil, fmt.Errorf("ordinary Windows container images are not bootable; missing %s", MachineImageVersionLabel) + } + return nil, nil + } + if version != MachineImageVersion { + return nil, fmt.Errorf("unsupported machine image version %q", version) + } + if !strings.EqualFold(meta.OS, "windows") || meta.Architecture != "amd64" { + return nil, fmt.Errorf("machine image requires platform windows/amd64") + } + + machine := &MachineImage{ + Kind: MachineImageKind(strings.TrimSpace(meta.Labels[MachineImageKindLabel])), + DiskPath: strings.TrimSpace(meta.Labels[MachineImageDiskPathLabel]), + DiskFormat: strings.TrimSpace(meta.Labels[MachineImageDiskFormatLabel]), + Base: strings.TrimSpace(meta.Labels[MachineImageBaseLabel]), + TPM: strings.TrimSpace(meta.Labels[MachineImageTPMLabel]), + SecureBoot: strings.TrimSpace(meta.Labels[MachineImageSecureBootLabel]), + } + if machine.DiskPath == "" || filepath.IsAbs(machine.DiskPath) || !filepath.IsLocal(machine.DiskPath) { + return nil, fmt.Errorf("machine image disk path must be a local relative path") + } + if machine.TPM != "2.0" { + return nil, fmt.Errorf("machine image requires TPM 2.0") + } + if machine.SecureBoot != "required" { + return nil, fmt.Errorf("machine image must require Secure Boot") + } + + switch machine.Kind { + case MachineImageWindowsBase: + switch machine.DiskFormat { + case "raw", "qcow2", "vhd", "vhdx": + default: + return nil, fmt.Errorf("unsupported Windows base disk format %q", machine.DiskFormat) + } + if machine.Base != "" { + return nil, fmt.Errorf("Windows base image cannot reference another base") + } + case MachineImageWindowsPersona: + if machine.DiskFormat != "qcow2" { + return nil, fmt.Errorf("Windows persona disk format must be qcow2") + } + base, err := ParseNormalizedRef(machine.Base) + if err != nil || !base.IsDigest() { + return nil, fmt.Errorf("Windows persona base must be a digest-pinned OCI reference") + } + default: + return nil, fmt.Errorf("unsupported machine image kind %q", machine.Kind) + } + return machine, nil +} + +func machineArtifactDisk(root string, machine *MachineImage) (string, error) { + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return "", fmt.Errorf("resolve machine artifact root: %w", err) + } + path, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(machine.DiskPath))) + if err != nil { + return "", fmt.Errorf("resolve machine image disk: %w", err) + } + rel, err := filepath.Rel(resolvedRoot, path) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("machine image disk path escapes artifact root") + } + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("stat machine image disk: %w", err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("machine image disk is not a regular file") + } + return path, nil +} + +type qemuImageInfo struct { + Format string `json:"format"` + VirtualSize int64 `json:"virtual-size"` + BackingFilename string `json:"backing-filename"` + BackingFileFormat string `json:"backing-filename-format"` +} + +func inspectQEMUImage(path string) (qemuImageInfo, error) { + output, err := exec.Command("qemu-img", "info", "--output=json", path).CombinedOutput() + if err != nil { + return qemuImageInfo{}, fmt.Errorf("inspect machine disk: %w: %s", err, output) + } + var info qemuImageInfo + if err := json.Unmarshal(output, &info); err != nil { + return qemuImageInfo{}, fmt.Errorf("decode qemu-img info: %w", err) + } + return info, nil +} + +func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine *MachineImage) (int64, error) { + source, err := machineArtifactDisk(root, machine) + if err != nil { + return 0, err + } + + destination := machineDiskPath(m.paths, ref.Repository(), ref.DigestHex(), machine.Kind) + if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { + return 0, fmt.Errorf("create machine image directory: %w", err) + } + _ = os.Remove(destination) + removeOnError := true + defer func() { + if removeOnError { + _ = os.Remove(destination) + } + }() + + switch machine.Kind { + case MachineImageWindowsBase: + if machine.DiskFormat == "raw" { + err = forkvm.CopyRegularFile(source, destination) + } else { + format := machine.DiskFormat + if format == "vhd" { + format = "vpc" + } + output, convertErr := exec.Command("qemu-img", "convert", "-f", format, "-O", "raw", source, destination).CombinedOutput() + if convertErr != nil { + err = fmt.Errorf("convert Windows base to raw: %w: %s", convertErr, output) + } + } + if err != nil { + return 0, fmt.Errorf("materialize Windows base: %w", err) + } + info, err := inspectQEMUImage(destination) + if err != nil { + return 0, err + } + if info.Format != "raw" { + return 0, fmt.Errorf("Windows base disk must be raw, got %s", info.Format) + } + machine.VirtualSize = info.VirtualSize + case MachineImageWindowsPersona: + if err := forkvm.CopyRegularFile(source, destination); err != nil { + return 0, fmt.Errorf("materialize Windows persona: %w", err) + } + if err := os.Chmod(destination, 0600); err != nil { + return 0, fmt.Errorf("make Windows persona writable for validation: %w", err) + } + basePath, err := m.resolveMachineBase(machine.Base) + if err != nil { + return 0, err + } + output, err := exec.Command("qemu-img", "rebase", "-u", "-f", "qcow2", "-F", "raw", "-b", basePath, destination).CombinedOutput() + if err != nil { + return 0, fmt.Errorf("set persona backing file: %w: %s", err, output) + } + info, err := inspectQEMUImage(destination) + if err != nil { + return 0, err + } + if info.Format != "qcow2" || info.BackingFileFormat != "raw" || info.BackingFilename != basePath { + return 0, fmt.Errorf("invalid persona disk backing configuration") + } + baseInfo, err := inspectQEMUImage(basePath) + if err != nil { + return 0, err + } + if info.VirtualSize != baseInfo.VirtualSize { + return 0, fmt.Errorf("persona virtual size %d does not match base %d", info.VirtualSize, baseInfo.VirtualSize) + } + machine.VirtualSize = info.VirtualSize + } + + if err := os.Chmod(destination, 0444); err != nil { + return 0, fmt.Errorf("make machine disk immutable: %w", err) + } + stat, err := os.Stat(destination) + if err != nil { + return 0, fmt.Errorf("stat materialized machine disk: %w", err) + } + removeOnError = false + return stat.Size(), nil +} + +func (m *manager) resolveMachineBase(reference string) (string, error) { + ref, err := ParseNormalizedRef(reference) + if err != nil || !ref.IsDigest() { + return "", fmt.Errorf("parse machine base reference") + } + meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) + if err != nil { + return "", fmt.Errorf("get machine base %s: %w", reference, err) + } + if meta.Status != StatusReady || meta.Machine == nil || meta.Machine.Kind != MachineImageWindowsBase { + return "", fmt.Errorf("machine base %s is not a ready Windows base", reference) + } + return machineDiskPath(m.paths, ref.Repository(), ref.DigestHex(), MachineImageWindowsBase), nil +} + +func machineDiskPath(p *paths.Paths, repository, digestHex string, kind MachineImageKind) string { + name := "base.raw" + if kind == MachineImageWindowsPersona { + name = "persona.qcow2" + } + return filepath.Join(p.ImageDigestDir(repository, digestHex), name) +} + +func (m *manager) ensureNoMachineDependents(repository, digestHex string) error { + metas, err := listAllMetadata(m.paths) + if err != nil { + return err + } + for _, meta := range metas { + if meta.Machine == nil || meta.Machine.Kind != MachineImageWindowsPersona { + continue + } + base, err := ParseNormalizedRef(meta.Machine.Base) + if err == nil && base.Repository() == repository && base.DigestHex() == digestHex { + return fmt.Errorf("cannot delete Windows base while persona %s depends on it", meta.Name) + } + } + return nil +} + +// GetMachineDiskPath returns the materialized disk for a machine image. +func GetMachineDiskPath(p *paths.Paths, imageName, digest string, machine *MachineImage) (string, error) { + if machine == nil { + return "", fmt.Errorf("image is not a machine image") + } + ref, err := ParseNormalizedRef(imageName) + if err != nil { + return "", fmt.Errorf("parse image name: %w", err) + } + return machineDiskPath(p, ref.Repository(), strings.TrimPrefix(digest, "sha256:"), machine.Kind), nil +} + +// IsWindowsPersona reports whether an image is directly launchable as a Windows desktop. +func IsWindowsPersona(image *Image) bool { + return image != nil && image.Machine != nil && image.Machine.Kind == MachineImageWindowsPersona +} diff --git a/lib/images/machine_oci_integration_test.go b/lib/images/machine_oci_integration_test.go new file mode 100644 index 000000000..c1f12d0b6 --- /dev/null +++ b/lib/images/machine_oci_integration_test.go @@ -0,0 +1,113 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + gcrregistry "github.com/google/go-containerregistry/pkg/registry" + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func machineArtifactOCIImage(t *testing.T, diskName string, disk []byte, labels map[string]string) gcr.Image { + t.Helper() + var layerData bytes.Buffer + gz := gzip.NewWriter(&layerData) + tw := tar.NewWriter(gz) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: diskName, Mode: 0644, Size: int64(len(disk))})) + _, err := tw.Write(disk) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(layerData.Bytes())), nil + }) + require.NoError(t, err) + image, err := mutate.AppendLayers(empty.Image, layer) + require.NoError(t, err) + config, err := image.ConfigFile() + require.NoError(t, err) + config.OS = "windows" + config.Architecture = "amd64" + config.Config.Labels = labels + image, err = mutate.ConfigFile(image, config) + require.NoError(t, err) + return image +} + +func waitForMachineImage(t *testing.T, manager Manager, name string) *Image { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + require.NoError(t, manager.WaitForReady(ctx, name)) + image, err := manager.GetImage(ctx, name) + require.NoError(t, err) + require.Equal(t, StatusReady, image.Status) + return image +} + +func TestMachineArtifactsPullFromOCI(t *testing.T) { + if _, err := exec.LookPath("qemu-img"); err != nil { + if os.Getenv("CI") == "true" { + t.Fatal("qemu-img is required in CI") + } + t.Skip("qemu-img is unavailable") + } + + registry := httptest.NewServer(gcrregistry.New()) + defer registry.Close() + manager, err := NewManager(paths.New(t.TempDir()), 1, nil) + require.NoError(t, err) + + baseFile := filepath.Join(t.TempDir(), "base.raw") + file, err := os.Create(baseFile) + require.NoError(t, err) + require.NoError(t, file.Truncate(4<<20)) + require.NoError(t, file.Close()) + baseBytes, err := os.ReadFile(baseFile) + require.NoError(t, err) + baseLabels := windowsMachineMetadata(MachineImageWindowsBase, "hypeman/base.raw", "").Labels + baseImage := machineArtifactOCIImage(t, "hypeman/base.raw", baseBytes, baseLabels) + baseTag, err := name.NewTag(registry.Listener.Addr().String()+"/windows/base:test", name.Insecure) + require.NoError(t, err) + require.NoError(t, remote.Write(baseTag, baseImage)) + + createdBase, err := manager.CreateImage(context.Background(), CreateImageRequest{Name: baseTag.String(), Platform: "windows/amd64"}) + require.NoError(t, err) + readyBase := waitForMachineImage(t, manager, createdBase.Name) + require.Equal(t, MachineImageWindowsBase, readyBase.Machine.Kind) + + personaFile := filepath.Join(t.TempDir(), "persona.qcow2") + output, err := exec.Command("qemu-img", "create", "-f", "qcow2", personaFile, "4M").CombinedOutput() + require.NoError(t, err, "%s", output) + personaBytes, err := os.ReadFile(personaFile) + require.NoError(t, err) + baseReference := baseTag.Context().Name() + "@" + readyBase.Digest + personaLabels := windowsMachineMetadata(MachineImageWindowsPersona, "hypeman/persona.qcow2", baseReference).Labels + personaImage := machineArtifactOCIImage(t, "hypeman/persona.qcow2", personaBytes, personaLabels) + personaTag, err := name.NewTag(registry.Listener.Addr().String()+"/windows/persona:test", name.Insecure) + require.NoError(t, err) + require.NoError(t, remote.Write(personaTag, personaImage)) + + createdPersona, err := manager.CreateImage(context.Background(), CreateImageRequest{Name: personaTag.String(), Platform: "windows/amd64"}) + require.NoError(t, err) + readyPersona := waitForMachineImage(t, manager, createdPersona.Name) + require.Equal(t, MachineImageWindowsPersona, readyPersona.Machine.Kind) + require.Equal(t, readyBase.Machine.VirtualSize, readyPersona.Machine.VirtualSize) +} diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go new file mode 100644 index 000000000..2051ac795 --- /dev/null +++ b/lib/images/machine_test.go @@ -0,0 +1,188 @@ +package images + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func windowsMachineMetadata(kind MachineImageKind, diskPath, base string) *containerMetadata { + format := "raw" + if kind == MachineImageWindowsPersona { + format = "qcow2" + } + return &containerMetadata{ + OS: "windows", + Architecture: "amd64", + Labels: map[string]string{ + MachineImageVersionLabel: MachineImageVersion, + MachineImageKindLabel: string(kind), + MachineImageDiskPathLabel: diskPath, + MachineImageDiskFormatLabel: format, + MachineImageBaseLabel: base, + MachineImageTPMLabel: "2.0", + MachineImageSecureBootLabel: "required", + }, + } +} + +func TestParseMachineImage(t *testing.T) { + base, err := parseMachineImage(windowsMachineMetadata(MachineImageWindowsBase, "hypeman/disk.raw", "")) + require.NoError(t, err) + assert.Equal(t, MachineImageWindowsBase, base.Kind) + + persona, err := parseMachineImage(windowsMachineMetadata( + MachineImageWindowsPersona, + "hypeman/disk.qcow2", + "registry.example/base@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + )) + require.NoError(t, err) + assert.Equal(t, MachineImageWindowsPersona, persona.Kind) + + _, err = parseMachineImage(&containerMetadata{OS: "windows", Architecture: "amd64", Labels: map[string]string{}}) + assert.ErrorContains(t, err, "ordinary Windows container images are not bootable") + + invalidPath := windowsMachineMetadata(MachineImageWindowsBase, "../disk.raw", "") + _, err = parseMachineImage(invalidPath) + assert.ErrorContains(t, err, "local relative path") +} + +func TestMachineArtifactDiskRejectsSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "disk.raw") + require.NoError(t, os.WriteFile(outside, []byte("disk"), 0644)) + require.NoError(t, os.Symlink(filepath.Dir(outside), filepath.Join(root, "hypeman"))) + + _, err := machineArtifactDisk(root, &MachineImage{DiskPath: "hypeman/disk.raw"}) + assert.ErrorContains(t, err, "escapes artifact root") +} + +func TestMaterializeWindowsBaseFormats(t *testing.T) { + if _, err := exec.LookPath("qemu-img"); err != nil { + if os.Getenv("CI") == "true" { + t.Fatal("qemu-img is required in CI") + } + t.Skip("qemu-img is unavailable") + } + + formats := []struct { + label string + qemu string + char string + }{ + {label: "raw", qemu: "raw", char: "1"}, + {label: "qcow2", qemu: "qcow2", char: "2"}, + {label: "vhd", qemu: "vpc", char: "3"}, + {label: "vhdx", qemu: "vhdx", char: "4"}, + } + for _, format := range formats { + t.Run(format.label, func(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + digest := strings.Repeat(format.char, 64) + ref, err := ParseNormalizedRef("registry.example/windows/base@sha256:" + digest) + require.NoError(t, err) + resolved := NewResolvedRef(ref, "sha256:"+digest) + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "hypeman"), 0755)) + source := filepath.Join(root, "hypeman", "disk") + output, err := exec.Command("qemu-img", "create", "-f", format.qemu, source, "4M").CombinedOutput() + require.NoError(t, err, "%s", output) + + meta := windowsMachineMetadata(MachineImageWindowsBase, "hypeman/disk", "") + meta.Labels[MachineImageDiskFormatLabel] = format.label + machine, err := parseMachineImage(meta) + require.NoError(t, err) + _, err = m.materializeMachineImage(resolved, root, machine) + require.NoError(t, err) + info, err := inspectQEMUImage(machineDiskPath(p, ref.Repository(), digest, MachineImageWindowsBase)) + require.NoError(t, err) + assert.Equal(t, "raw", info.Format) + }) + } +} + +func TestMaterializeWindowsBaseAndPersona(t *testing.T) { + if _, err := exec.LookPath("qemu-img"); err != nil { + if os.Getenv("CI") == "true" { + t.Fatal("qemu-img is required in CI") + } + t.Skip("qemu-img is unavailable") + } + + p := paths.New(t.TempDir()) + m := &manager{paths: p} + baseDigest := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + baseRef, err := ParseNormalizedRef("registry.example/windows/base@sha256:" + baseDigest) + require.NoError(t, err) + resolvedBase := NewResolvedRef(baseRef, "sha256:"+baseDigest) + + baseRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(baseRoot, "hypeman"), 0755)) + baseSource := filepath.Join(baseRoot, "hypeman", "disk.raw") + baseFile, err := os.Create(baseSource) + require.NoError(t, err) + require.NoError(t, baseFile.Truncate(4<<20)) + require.NoError(t, baseFile.Close()) + + baseMachine, err := parseMachineImage(windowsMachineMetadata(MachineImageWindowsBase, "hypeman/disk.raw", "")) + require.NoError(t, err) + _, err = m.materializeMachineImage(resolvedBase, baseRoot, baseMachine) + require.NoError(t, err) + require.NoError(t, writeMetadata(p, baseRef.Repository(), baseDigest, &imageMetadata{ + Name: baseRef.String(), + Digest: "sha256:" + baseDigest, + Platform: "windows/amd64", + Status: StatusReady, + Machine: baseMachine, + SizeBytes: 4 << 20, + })) + + personaDigest := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + personaRef, err := ParseNormalizedRef("registry.example/windows/persona@sha256:" + personaDigest) + require.NoError(t, err) + resolvedPersona := NewResolvedRef(personaRef, "sha256:"+personaDigest) + personaRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(personaRoot, "hypeman"), 0755)) + personaSource := filepath.Join(personaRoot, "hypeman", "disk.qcow2") + output, err := exec.Command("qemu-img", "create", "-f", "qcow2", personaSource, "4M").CombinedOutput() + require.NoError(t, err, "%s", output) + + personaMachine, err := parseMachineImage(windowsMachineMetadata( + MachineImageWindowsPersona, + "hypeman/disk.qcow2", + baseRef.String(), + )) + require.NoError(t, err) + _, err = m.materializeMachineImage(resolvedPersona, personaRoot, personaMachine) + require.NoError(t, err) + + personaPath := machineDiskPath(p, personaRef.Repository(), personaDigest, MachineImageWindowsPersona) + info, err := inspectQEMUImage(personaPath) + require.NoError(t, err) + assert.Equal(t, "qcow2", info.Format) + assert.Equal(t, "raw", info.BackingFileFormat) + assert.Equal(t, machineDiskPath(p, baseRef.Repository(), baseDigest, MachineImageWindowsBase), info.BackingFilename) + + baseMeta, err := readMetadata(p, baseRef.Repository(), baseDigest) + require.NoError(t, err) + personaMeta := &imageMetadata{ + Name: personaRef.String(), + Digest: "sha256:" + personaDigest, + Platform: "windows/amd64", + Status: StatusReady, + Machine: personaMachine, + SizeBytes: info.VirtualSize, + } + require.NoError(t, writeMetadata(p, personaRef.Repository(), personaDigest, personaMeta)) + assert.ErrorContains(t, m.ensureNoMachineDependents(baseRef.Repository(), baseDigest), "depends on it") + assert.ErrorContains(t, m.DeleteImage(t.Context(), baseRef.String()), "depends on it") + assert.DirExists(t, p.ImageDigestDir(baseRef.Repository(), baseDigest)) + assert.Equal(t, MachineImageWindowsBase, baseMeta.toImage().Machine.Kind) +} diff --git a/lib/images/manager.go b/lib/images/manager.go index c1a0da0ac..9d0845913 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -465,18 +465,28 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials m.updateStatusByDigest(ref, StatusConverting, nil, buildID) - diskPath := digestPath(m.paths, ref.Repository(), ref.DigestHex()) - // Use default image format (erofs on Linux, ext4 on Darwin) + machine, err := parseMachineImage(result.Metadata) + if err != nil { + m.updateStatusByDigest(ref, StatusFailed, err, buildID) + return + } + convertStart := time.Now() - diskSize, err := ExportRootfs(tempDir, diskPath, DefaultImageFormat) + var diskSize int64 + if machine != nil { + diskSize, err = m.materializeMachineImage(ref, tempDir, machine) + } else { + diskPath := digestPath(m.paths, ref.Repository(), ref.DigestHex()) + diskSize, err = ExportRootfs(tempDir, diskPath, DefaultImageFormat) + } m.recordImageBuildPhase(ctx, ref.Digest(), "filesystem_export", time.Since(convertStart), phaseStatus(err), "not_applicable") if err != nil { - m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("convert to %s: %w", DefaultImageFormat, err), buildID) + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("materialize image: %w", err), buildID) return } finalizeStart := time.Now() - err = m.finalizeImage(ref, result, diskSize, buildID) + err = m.finalizeImage(ref, result, diskSize, machine, buildID) m.recordImageBuildPhase(ctx, ref.Digest(), "finalize", time.Since(finalizeStart), phaseStatus(err), "not_applicable") if err != nil { if errors.Is(err, errStaleBuild) { @@ -489,7 +499,7 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials buildStatus = "success" } -func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize int64, buildID string) error { +func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize int64, machine *MachineImage, buildID string) error { m.createMu.Lock() defer m.createMu.Unlock() @@ -518,6 +528,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i meta.Env = result.Metadata.Env meta.Labels = result.Metadata.Labels meta.WorkingDir = result.Metadata.WorkingDir + meta.Machine = machine if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { return fmt.Errorf("write final metadata: %w", err) @@ -687,6 +698,9 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if _, err := readMetadata(m.paths, repository, digestHex); err != nil { return err } + if err := m.ensureNoMachineDependents(repository, digestHex); err != nil { + return err + } if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil { return err } @@ -705,20 +719,20 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { return err } - // Delete the tag symlink - if err := deleteTag(m.paths, repository, tag); err != nil { - return err - } - - // Check if the digest is now orphaned (no other tags reference it) count, err := countTagsForDigest(m.paths, repository, digestHex) if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to count tags for digest %s: %v\n", digestHex, err) - return nil + return fmt.Errorf("count tags for digest %s: %w", digestHex, err) + } + if count == 1 { + if err := m.ensureNoMachineDependents(repository, digestHex); err != nil { + return err + } } - if count == 0 { - // Digest is orphaned, delete it + if err := deleteTag(m.paths, repository, tag); err != nil { + return err + } + if count == 1 { if err := deleteDigest(m.paths, repository, digestHex); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to delete orphaned digest %s: %v\n", digestHex, err) return nil diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 89d3fb4fe..4e8a26ba5 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -698,7 +698,7 @@ func TestDeleteAndRecreateDuringBuildTail(t *testing.T) { m.updateStatusByDigest(staleRef, StatusFailed, errors.New("stale build"), firstMeta.BuildID) staleResult, _, _, err := m.ociClient.extractOCIImageDetails(digestHex) require.NoError(t, err) - require.ErrorIs(t, m.finalizeImage(staleRef, &pullResult{Metadata: staleResult}, 1, firstMeta.BuildID), errStaleBuild) + require.ErrorIs(t, m.finalizeImage(staleRef, &pullResult{Metadata: staleResult}, 1, nil, firstMeta.BuildID), errStaleBuild) currentMeta, err = readMetadata(p, repo, digestHex) require.NoError(t, err) require.Equal(t, StatusPending, currentMeta.Status) diff --git a/lib/images/platform.go b/lib/images/platform.go index f5c40b3d8..56d7fdcdf 100644 --- a/lib/images/platform.go +++ b/lib/images/platform.go @@ -79,18 +79,24 @@ func (p Platform) Normalize() Platform { } } -// validate enforces the platforms hypeman can actually boot today: Linux -// guests on amd64 or arm64. Other operating systems and architectures are -// rejected with an actionable error. +// validate enforces the platforms hypeman can boot. Windows machine images +// currently target the QEMU amd64 path only. func (p Platform) validate() error { - if p.OS != "linux" { - return fmt.Errorf("%w: unsupported os %q: only linux guests are supported", ErrInvalidPlatform, p.OS) - } - switch p.Architecture { - case "amd64", "arm64": - return nil + switch p.OS { + case "linux": + switch p.Architecture { + case "amd64", "arm64": + return nil + default: + return fmt.Errorf("%w: unsupported Linux architecture %q: must be amd64 or arm64", ErrInvalidPlatform, p.Architecture) + } + case "windows": + if p.Architecture == "amd64" && p.Variant == "" { + return nil + } + return fmt.Errorf("%w: Windows machine images require amd64 without a variant", ErrInvalidPlatform) default: - return fmt.Errorf("%w: unsupported architecture %q: must be amd64 or arm64", ErrInvalidPlatform, p.Architecture) + return fmt.Errorf("%w: unsupported os %q: must be linux or windows", ErrInvalidPlatform, p.OS) } } diff --git a/lib/images/platform_test.go b/lib/images/platform_test.go index 82b9ab97f..ce4b29eb7 100644 --- a/lib/images/platform_test.go +++ b/lib/images/platform_test.go @@ -19,7 +19,8 @@ func TestParsePlatform(t *testing.T) { {name: "x86_64 alias", in: "x86_64", want: Platform{OS: "linux", Architecture: "amd64"}}, {name: "aarch64 alias", in: "linux/aarch64", want: Platform{OS: "linux", Architecture: "arm64"}}, {name: "uppercase normalized", in: "LINUX/AMD64", want: Platform{OS: "linux", Architecture: "amd64"}}, - {name: "non-linux os rejected", in: "windows/amd64", wantErr: true}, + {name: "windows amd64 machine platform", in: "windows/amd64", want: Platform{OS: "windows", Architecture: "amd64"}}, + {name: "windows arm64 rejected", in: "windows/arm64", wantErr: true}, {name: "unknown arch rejected", in: "linux/riscv64", wantErr: true}, {name: "empty rejected", in: "", wantErr: true}, {name: "too many parts", in: "a/b/c/d", wantErr: true}, @@ -145,9 +146,10 @@ func TestResolveManifestPlatform(t *testing.T) { t.Fatalf("expected ErrInvalidPlatform for mismatch, got %v", err) } - // An unsupported manifest os fails validation. - if _, err := resolveManifestPlatform(&containerMetadata{OS: "windows", Architecture: "amd64"}, ""); err == nil { - t.Fatal("expected error for non-linux manifest") + // A Windows machine manifest records its platform. + got, err = resolveManifestPlatform(&containerMetadata{OS: "windows", Architecture: "amd64"}, "") + if err != nil || got.String() != "windows/amd64" { + t.Fatalf("Windows manifest = %s, %v", got, err) } // A manifest with no declared architecture (locally built/synthetic image) diff --git a/lib/images/storage.go b/lib/images/storage.go index 8764400c1..24ce44378 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -27,6 +27,7 @@ type imageMetadata struct { Labels map[string]string `json:"labels,omitempty"` Tags tags.Tags `json:"tags,omitempty"` WorkingDir string `json:"working_dir,omitempty"` + Machine *MachineImage `json:"machine,omitempty"` CreatedAt time.Time `json:"created_at"` BorrowedAuth bool `json:"borrowed_auth,omitempty"` BuildID string `json:"build_id,omitempty"` @@ -73,6 +74,10 @@ func (m *imageMetadata) toImage() *Image { if m.WorkingDir != "" { img.WorkingDir = m.WorkingDir } + if m.Machine != nil { + machine := *m.Machine + img.Machine = &machine + } return img } @@ -158,6 +163,9 @@ func readMetadata(p *paths.Paths, repository, digestHex string) (*imageMetadata, if meta.Status == StatusReady { diskPath := digestPath(p, repository, digestHex) + if meta.Machine != nil { + diskPath = machineDiskPath(p, repository, digestHex, meta.Machine.Kind) + } if _, err := os.Stat(diskPath); err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("disk image missing: %s", diskPath) diff --git a/lib/images/types.go b/lib/images/types.go index a7c498193..24a39b6c5 100644 --- a/lib/images/types.go +++ b/lib/images/types.go @@ -22,6 +22,7 @@ type Image struct { Labels map[string]string Tags tags.Tags WorkingDir string + Machine *MachineImage CreatedAt time.Time } diff --git a/lib/instances/create.go b/lib/instances/create.go index 4e2245b73..55ec459b4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -128,6 +128,12 @@ func (m *manager) createInstance( log.ErrorContext(ctx, "image not ready", "image", req.Image, "status", imageInfo.Status) return nil, fmt.Errorf("%w: image status is %s", ErrImageNotReady, imageInfo.Status) } + windows := isWindowsPlatform(imageInfo.Platform) + if windows { + if err := validateWindowsCreate(req, imageInfo, hvType); err != nil { + return nil, err + } + } // A guest whose architecture differs from the host kernel can only boot via // emulation. On Apple silicon that is Rosetta, enabled automatically; on any @@ -147,17 +153,20 @@ func (m *manager) createInstance( return nil, fmt.Errorf("get vm starter for %s: %w", hvType, starterErr) } - defaultKernel := m.systemManager.GetDefaultKernelVersion() - kernelVer, err := resolveCreateKernelVersion(imageInfo, defaultKernel) - if err != nil { - log.ErrorContext(ctx, "invalid image kernel label", "image", req.Image, "error", err) - return nil, err - } - if kernelVer != defaultKernel { - log.InfoContext(ctx, "using image-declared kernel version", - "image", req.Image, - "kernel", kernelVer, - "label", system.ImageKernelVersionLabel) + var kernelVer system.KernelVersion + if !windows { + defaultKernel := m.systemManager.GetDefaultKernelVersion() + kernelVer, err = resolveCreateKernelVersion(imageInfo, defaultKernel) + if err != nil { + log.ErrorContext(ctx, "invalid image kernel label", "image", req.Image, "error", err) + return nil, err + } + if kernelVer != defaultKernel { + log.InfoContext(ctx, "using image-declared kernel version", + "image", req.Image, + "kernel", kernelVer, + "label", system.ImageKernelVersionLabel) + } } // resolvedImageRef is the digest-pinned reference used for boot/start/restore // (stable across mutable tags). The caller-facing Image field keeps the @@ -185,11 +194,17 @@ func (m *manager) createInstance( // 6. Apply defaults size := req.Size if size == 0 { - size = 1 * 1024 * 1024 * 1024 // 1GB default + if windows { + size = 8 * 1024 * 1024 * 1024 + } else { + size = 1 * 1024 * 1024 * 1024 // 1GB default + } } hotplugSize := req.HotplugSize overlaySize := req.OverlaySize - if overlaySize == 0 { + if windows { + overlaySize = imageInfo.Machine.VirtualSize + } else if overlaySize == 0 { overlaySize = 10 * 1024 * 1024 * 1024 // 10GB default } // Validate overlay size against max @@ -198,7 +213,11 @@ func (m *manager) createInstance( } vcpus := req.Vcpus if vcpus == 0 { - vcpus = 2 + if windows { + vcpus = 4 + } else { + vcpus = 2 + } } // Validate per-instance resource limits @@ -365,7 +384,7 @@ func (m *manager) createInstance( Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, - SkipGuestAgent: req.SkipGuestAgent, + SkipGuestAgent: req.SkipGuestAgent || windows, EnableRosetta: enableRosetta, SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), @@ -380,11 +399,17 @@ func (m *manager) createInstance( return nil, fmt.Errorf("ensure directories: %w", err) } - // 13. Create overlay disk with specified size - log.DebugContext(ctx, "creating overlay disk", "instance_id", id, "size_bytes", stored.OverlaySize) - if err := m.createOverlayDisk(id, stored.OverlaySize); err != nil { - log.ErrorContext(ctx, "failed to create overlay disk", "instance_id", id, "error", err) - return nil, fmt.Errorf("create overlay disk: %w", err) + // 13. Create the guest's writable disk. + if windows { + if err := m.prepareWindowsInstance(stored, imageInfo); err != nil { + return nil, fmt.Errorf("prepare Windows instance: %w", err) + } + } else { + log.DebugContext(ctx, "creating overlay disk", "instance_id", id, "size_bytes", stored.OverlaySize) + if err := m.createOverlayDisk(id, stored.OverlaySize); err != nil { + log.ErrorContext(ctx, "failed to create overlay disk", "instance_id", id, "error", err) + return nil, fmt.Errorf("create overlay disk: %w", err) + } } // 14. Allocate network (if network enabled) @@ -482,18 +507,20 @@ func (m *manager) createInstance( m.unregisterEgressProxyInstance(ctx, id) }) } - log.DebugContext(ctx, "creating config disk", "instance_id", id) - configDiskCtx, configDiskSpanEnd := m.startLifecycleStep(ctx, "create_config_disk", - attribute.String("instance_id", id), - attribute.String("hypervisor", string(stored.HypervisorType)), - attribute.String("operation", "create_config_disk"), - ) - if err := m.createConfigDisk(configDiskCtx, inst, imageInfo, netConfig, proxyGuestConfig); err != nil { - configDiskSpanEnd(err) - log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) - return nil, fmt.Errorf("create config disk: %w", err) + if !windows { + log.DebugContext(ctx, "creating config disk", "instance_id", id) + configDiskCtx, configDiskSpanEnd := m.startLifecycleStep(ctx, "create_config_disk", + attribute.String("instance_id", id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "create_config_disk"), + ) + if err := m.createConfigDisk(configDiskCtx, inst, imageInfo, netConfig, proxyGuestConfig); err != nil { + configDiskSpanEnd(err) + log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) + return nil, fmt.Errorf("create config disk: %w", err) + } + configDiskSpanEnd(nil) } - configDiskSpanEnd(nil) // 17. Record boot start time before launching the VM so marker hydration // can safely ignore stale sentinels from prior runs. @@ -809,6 +836,10 @@ func resolveRuntimeHypervisorPID(log *slog.Logger, socketPath string, fallbackPI // buildHypervisorConfig creates a hypervisor-agnostic VM configuration func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig) (hypervisor.VMConfig, error) { + if isWindowsPlatform(inst.Platform) { + return m.buildWindowsHypervisorConfig(inst, imageInfo, netConfig) + } + // Get system file paths kernelPath, _ := m.systemManager.GetKernelPath(system.KernelVersion(inst.KernelVersion)) initrdPath, _ := m.systemManager.GetInitrdPath() diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index 238747452..fdfd97d9f 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -28,7 +28,7 @@ func resolveImageForCreate(ctx context.Context, imageManager createImageResolver // host-native; an empty/unknown platform (e.g. a legacy record) is not // assumed to be the host and falls through to host-pinned resolution. if img, err := imageManager.GetImage(ctx, imageName); err == nil { - if p := strings.TrimSpace(img.Platform); p != "" && !images.ImageNeedsHostEmulation(p) { + if strings.TrimSpace(img.Platform) == images.HostPlatformString() { return img, nil } } else if !errors.Is(err, images.ErrNotFound) { diff --git a/lib/instances/create_image_test.go b/lib/instances/create_image_test.go index d964bafe2..ed7185f65 100644 --- a/lib/instances/create_image_test.go +++ b/lib/instances/create_image_test.go @@ -214,6 +214,29 @@ func TestResolveImageForCreateWithoutPlatformLegacyEmptyForcesHostResolve(t *tes } } +func TestResolveImageForCreateWithoutPlatformIgnoresCachedWindowsImage(t *testing.T) { + t.Parallel() + + createPlatform := "" + resolver := createImageResolverFake{ + getImage: func(context.Context, string) (*images.Image, error) { + return &images.Image{Platform: "windows/amd64", Status: images.StatusReady}, nil + }, + createImage: func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createPlatform = req.Platform + return &images.Image{Name: req.Name, Digest: "sha256:linux", Platform: images.HostPlatformString(), Status: images.StatusReady}, nil + }, + } + + _, err := resolveImageForCreate(context.Background(), resolver, "registry.example/desktop:test", "", slog.Default()) + if err != nil { + t.Fatalf("resolve image: %v", err) + } + if createPlatform != images.HostPlatformString() { + t.Fatalf("cached Windows image must not satisfy an implicit host-platform create; got %q", createPlatform) + } +} + // A no-platform create must NOT trust a tag pointer that resolves to a non-host // arch (last-pull-wins can point the tag at an emulated variant). It must // re-resolve the host variant explicitly and never silently emulate. diff --git a/lib/instances/start.go b/lib/instances/start.go index 032b71076..cc02476ec 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -164,20 +164,22 @@ func (m *manager) startInstance( }) } - // 5. Regenerate config disk with new network configuration - instForConfig := &Instance{StoredMetadata: *stored} - log.DebugContext(ctx, "regenerating config disk", "instance_id", id) - configDiskCtx, configDiskSpanEnd := m.startLifecycleStep(ctx, "create_config_disk", - attribute.String("instance_id", id), - attribute.String("hypervisor", string(stored.HypervisorType)), - attribute.String("operation", "create_config_disk"), - ) - if err := m.createConfigDisk(configDiskCtx, instForConfig, imageInfo, netConfig, proxyGuestConfig); err != nil { - configDiskSpanEnd(err) - log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) - return nil, fmt.Errorf("create config disk: %w", err) + // 5. Regenerate the Linux config disk with new network configuration. + if !isWindowsPlatform(stored.Platform) { + instForConfig := &Instance{StoredMetadata: *stored} + log.DebugContext(ctx, "regenerating config disk", "instance_id", id) + configDiskCtx, configDiskSpanEnd := m.startLifecycleStep(ctx, "create_config_disk", + attribute.String("instance_id", id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "create_config_disk"), + ) + if err := m.createConfigDisk(configDiskCtx, instForConfig, imageInfo, netConfig, proxyGuestConfig); err != nil { + configDiskSpanEnd(err) + log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) + return nil, fmt.Errorf("create config disk: %w", err) + } + configDiskSpanEnd(nil) } - configDiskSpanEnd(nil) if err := m.archiveAppLogForBoot(id); err != nil { log.WarnContext(ctx, "failed to archive app log before start", "instance_id", id, "error", err) diff --git a/lib/instances/windows.go b/lib/instances/windows.go new file mode 100644 index 000000000..13fc8991e --- /dev/null +++ b/lib/instances/windows.go @@ -0,0 +1,157 @@ +package instances + +import ( + "fmt" + "os" + "strings" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" +) + +func isWindowsPlatform(platform string) bool { + return strings.EqualFold(strings.TrimSpace(platform), "windows/amd64") +} + +func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, hvType hypervisor.Type) error { + if !images.IsWindowsPersona(image) { + return fmt.Errorf("%w: Windows instances require a persona image", ErrInvalidRequest) + } + if hvType != hypervisor.TypeQEMU { + return fmt.Errorf("%w: Windows instances require the qemu hypervisor", ErrInvalidRequest) + } + if image.Machine.VirtualSize <= 0 { + return fmt.Errorf("%w: Windows persona is missing its virtual disk size", ErrInvalidRequest) + } + if req.HotplugSize != 0 { + return fmt.Errorf("%w: Windows instances do not yet support hotplug memory", ErrInvalidRequest) + } + if req.Size != 0 && req.Size < 4<<30 { + return fmt.Errorf("%w: Windows 11 requires at least 4 GiB of memory", ErrInvalidRequest) + } + if req.Vcpus != 0 && req.Vcpus < 2 { + return fmt.Errorf("%w: Windows 11 requires at least 2 vCPUs", ErrInvalidRequest) + } + if req.NetworkEnabled { + return fmt.Errorf("%w: Windows networking is added in the networking phase", ErrInvalidRequest) + } + if len(req.Volumes) != 0 || len(req.Devices) != 0 || req.GPU != nil { + return fmt.Errorf("%w: Windows instances do not yet support volumes or device passthrough", ErrInvalidRequest) + } + if req.OverlaySize != 0 && req.OverlaySize != image.Machine.VirtualSize { + return fmt.Errorf("%w: Windows instance disk size is fixed at %d bytes", ErrInvalidRequest, image.Machine.VirtualSize) + } + if len(req.Entrypoint) != 0 || len(req.Cmd) != 0 { + return fmt.Errorf("%w: Windows machine images do not support entrypoint or command overrides", ErrInvalidRequest) + } + if len(req.Env) != 0 || req.HealthCheck != nil { + return fmt.Errorf("%w: Windows instances do not yet support environment injection or health checks", ErrInvalidRequest) + } + if req.NetworkEgress != nil || len(req.Credentials) != 0 { + return fmt.Errorf("%w: Windows instances do not yet support managed egress or credentials", ErrInvalidRequest) + } + return nil +} + +func windowsFirmwareTemplates() (string, string, error) { + code := os.Getenv("HYPEMAN_WINDOWS_OVMF_CODE") + if code == "" { + code = "/usr/share/OVMF/OVMF_CODE_4M.secboot.fd" + } + vars := os.Getenv("HYPEMAN_WINDOWS_OVMF_VARS") + if vars == "" { + vars = "/usr/share/OVMF/OVMF_VARS_4M.ms.fd" + } + for _, path := range []string{code, vars} { + info, err := os.Stat(path) + if err != nil { + return "", "", fmt.Errorf("Windows firmware %s: %w", path, err) + } + if !info.Mode().IsRegular() { + return "", "", fmt.Errorf("Windows firmware %s is not a regular file", path) + } + } + return code, vars, nil +} + +func (m *manager) prepareWindowsInstance(inst *StoredMetadata, image *images.Image) error { + persona, err := images.GetMachineDiskPath(m.paths, image.Name, image.Digest, image.Machine) + if err != nil { + return err + } + if err := forkvm.CopyRegularFile(persona, m.paths.InstanceWindowsDisk(inst.Id)); err != nil { + return fmt.Errorf("clone Windows persona: %w", err) + } + if err := os.Chmod(m.paths.InstanceWindowsDisk(inst.Id), 0600); err != nil { + return fmt.Errorf("make Windows instance disk writable: %w", err) + } + + code, vars, err := windowsFirmwareTemplates() + if err != nil { + return err + } + if err := forkvm.CopyRegularFile(code, m.paths.InstanceOVMFCode(inst.Id)); err != nil { + return fmt.Errorf("copy OVMF code: %w", err) + } + if err := os.Chmod(m.paths.InstanceOVMFCode(inst.Id), 0444); err != nil { + return fmt.Errorf("make OVMF code immutable: %w", err) + } + if err := forkvm.CopyRegularFile(vars, m.paths.InstanceOVMFVars(inst.Id)); err != nil { + return fmt.Errorf("copy OVMF variables: %w", err) + } + if err := os.Chmod(m.paths.InstanceOVMFVars(inst.Id), 0600); err != nil { + return fmt.Errorf("make OVMF variables writable: %w", err) + } + if err := os.MkdirAll(m.paths.InstanceTPMDir(inst.Id), 0700); err != nil { + return fmt.Errorf("create TPM state directory: %w", err) + } + return nil +} + +func (m *manager) buildWindowsHypervisorConfig(inst *Instance, image *images.Image, netConfig *network.NetworkConfig) (hypervisor.VMConfig, error) { + if !images.IsWindowsPersona(image) { + return hypervisor.VMConfig{}, fmt.Errorf("image is not a Windows persona") + } + if _, err := os.Stat(m.paths.InstanceWindowsDisk(inst.Id)); err != nil { + return hypervisor.VMConfig{}, fmt.Errorf("stat Windows instance disk: %w", err) + } + + var networks []hypervisor.NetworkConfig + if netConfig != nil { + networks = []hypervisor.NetworkConfig{{ + TAPDevice: netConfig.TAPDevice, + IP: netConfig.IP, + MAC: netConfig.MAC, + Netmask: netConfig.Netmask, + DownloadBps: inst.NetworkBandwidthDownload, + UploadBps: inst.NetworkBandwidthUpload, + }} + } + + ioBps := inst.DiskIOBps + burstBps := ioBps * 4 + if ioBps <= 0 { + burstBps = 0 + } + return hypervisor.VMConfig{ + VCPUs: inst.Vcpus, + MemoryBytes: inst.Size, + Disks: []hypervisor.DiskConfig{{Path: m.paths.InstanceWindowsDisk(inst.Id), Format: hypervisor.DiskFormatQCOW2, IOBps: ioBps, IOBurstBps: burstBps}}, + Networks: networks, + SerialLogPath: m.paths.InstanceAppLog(inst.Id), + VsockCID: inst.VsockCID, + VsockSocket: inst.VsockSocket, + BootMode: hypervisor.BootModeUEFI, + Firmware: &hypervisor.FirmwareConfig{ + CodePath: m.paths.InstanceOVMFCode(inst.Id), + VarsPath: m.paths.InstanceOVMFVars(inst.Id), + SecureBoot: true, + }, + TPM: &hypervisor.TPMConfig{ + SocketPath: m.paths.InstanceTPMSocket(inst.Id), + StateDir: m.paths.InstanceTPMDir(inst.Id), + }, + }, nil +} diff --git a/lib/instances/windows_images_integration_linux_test.go b/lib/instances/windows_images_integration_linux_test.go new file mode 100644 index 000000000..5eb869f91 --- /dev/null +++ b/lib/instances/windows_images_integration_linux_test.go @@ -0,0 +1,128 @@ +//go:build linux && amd64 + +package instances + +import ( + "context" + "crypto/sha256" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type windowsFixtureImageManager struct { + images.Manager + image *images.Image +} + +func (m windowsFixtureImageManager) CreateImage(context.Context, images.CreateImageRequest) (*images.Image, error) { + copy := *m.image + return ©, nil +} + +func (m windowsFixtureImageManager) GetImage(context.Context, string) (*images.Image, error) { + copy := *m.image + return ©, nil +} + +func (m windowsFixtureImageManager) WaitForReady(context.Context, string) error { return nil } + +func requireWindowsFixture(t *testing.T) string { + t.Helper() + path := os.Getenv("HYPEMAN_WINDOWS_TEST_PERSONA") + if path == "" { + path = "/ci/windows/persona.qcow2" + } + if _, err := os.Stat(path); err == nil { + return path + } + if os.Getenv("CI") == "true" { + t.Fatalf("required Windows persona fixture is missing: %s", path) + } + t.Skipf("Windows persona fixture is unavailable: %s", path) + return "" +} + +func TestWindowsImagesIntegration(t *testing.T) { + fixture := requireWindowsFixture(t) + acquireHeavyIO(t) + + manager, dataDir := setupTestManagerForQEMU(t) + p := paths.New(dataDir) + const digestHex = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + image := &images.Image{ + Name: "registry.example/windows/persona:integration", + Digest: "sha256:" + digestHex, + Platform: "windows/amd64", + Status: images.StatusReady, + Machine: &images.MachineImage{ + Kind: images.MachineImageWindowsPersona, + Base: "registry.example/windows/base@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + TPM: "2.0", + SecureBoot: "required", + VirtualSize: 80 << 30, + }, + } + manager.imageManager = windowsFixtureImageManager{image: image} + + personaPath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) + require.NoError(t, err) + require.NoError(t, forkvm.CopyRegularFile(fixture, personaPath)) + require.NoError(t, os.Chmod(personaPath, 0444)) + personaBytes, err := os.ReadFile(personaPath) + require.NoError(t, err) + personaHash := sha256.Sum256(personaBytes) + personaInfo, err := os.Stat(personaPath) + require.NoError(t, err) + + ctx := context.Background() + instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "windows-images-integration", + Image: image.Name, + Platform: "windows/amd64", + Size: 8 << 30, + Vcpus: 4, + Hypervisor: hypervisor.TypeQEMU, + }) + require.NoError(t, err) + instanceID := instance.Id + t.Cleanup(func() { + if instanceID != "" { + _ = deleteTestInstanceNow(context.Background(), manager, instanceID) + } + }) + require.Equal(t, StateInitializing, instance.State) + require.FileExists(t, p.InstanceWindowsDisk(instance.Id)) + require.FileExists(t, p.InstanceOVMFVars(instance.Id)) + require.DirExists(t, p.InstanceTPMDir(instance.Id)) + assert.NoFileExists(t, p.InstanceOverlay(instance.Id)) + assert.NoFileExists(t, p.InstanceConfigDisk(instance.Id)) + + require.Eventually(t, func() bool { + info, err := os.Stat(p.InstanceWindowsDisk(instance.Id)) + return err == nil && info.Size() > personaInfo.Size() + }, 60*time.Second, 500*time.Millisecond, "Windows boot must write to the instance qcow2") + + sourceAfter, err := os.ReadFile(personaPath) + require.NoError(t, err) + assert.Equal(t, personaHash, sha256.Sum256(sourceAfter), "immutable persona changed during guest boot") + + stopped, err := manager.StopInstance(ctx, instance.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, stopped.State) + output, err := exec.Command("qemu-img", "check", p.InstanceWindowsDisk(instance.Id)).CombinedOutput() + require.NoError(t, err, "%s", output) + + require.NoError(t, manager.DeleteInstance(ctx, instance.Id)) + instanceID = "" + assert.NoDirExists(t, filepath.Dir(p.InstanceWindowsDisk(instance.Id))) +} diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go new file mode 100644 index 000000000..112f654ed --- /dev/null +++ b/lib/instances/windows_test.go @@ -0,0 +1,70 @@ +package instances + +import ( + "os" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func windowsPersonaFixture() *images.Image { + return &images.Image{ + Name: "registry.example/windows/persona:test", + Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Platform: "windows/amd64", + Status: images.StatusReady, + Machine: &images.MachineImage{ + Kind: images.MachineImageWindowsPersona, + Base: "registry.example/windows/base@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + TPM: "2.0", + SecureBoot: "required", + VirtualSize: 80 << 30, + }, + } +} + +func TestValidateWindowsCreate(t *testing.T) { + image := windowsPersonaFixture() + require.NoError(t, validateWindowsCreate(CreateInstanceRequest{}, image, hypervisor.TypeQEMU)) + + tests := []struct { + name string + req CreateInstanceRequest + hv hypervisor.Type + }{ + {name: "wrong hypervisor", hv: hypervisor.TypeCloudHypervisor}, + {name: "networking", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{NetworkEnabled: true}}, + {name: "small memory", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Size: 2 << 30}}, + {name: "one CPU", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Vcpus: 1}}, + {name: "command", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Cmd: []string{"cmd.exe"}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Error(t, validateWindowsCreate(tt.req, image, tt.hv)) + }) + } +} + +func TestBuildWindowsHypervisorConfig(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + stored := StoredMetadata{Id: "instance", Platform: "windows/amd64", Size: 8 << 30, Vcpus: 4, VsockCID: 42} + require.NoError(t, os.MkdirAll(p.InstanceDir(stored.Id), 0755)) + for _, path := range []string{p.InstanceWindowsDisk(stored.Id), p.InstanceOVMFCode(stored.Id), p.InstanceOVMFVars(stored.Id)} { + require.NoError(t, os.WriteFile(path, []byte("fixture"), 0600)) + } + + config, err := m.buildWindowsHypervisorConfig(&Instance{StoredMetadata: stored}, windowsPersonaFixture(), nil) + require.NoError(t, err) + assert.Equal(t, hypervisor.BootModeUEFI, config.BootMode) + assert.True(t, config.Firmware.SecureBoot) + assert.Equal(t, p.InstanceTPMDir(stored.Id), config.TPM.StateDir) + require.Len(t, config.Disks, 1) + assert.Equal(t, hypervisor.DiskFormatQCOW2, config.Disks[0].Format) + assert.Empty(t, config.KernelPath) + assert.Empty(t, config.InitrdPath) +} diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 7e20cba38..7fc0870fd 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -199,6 +199,26 @@ func (p *Paths) InstanceConfigDisk(id string) string { return filepath.Join(p.InstanceDir(id), "config.ext4") } +func (p *Paths) InstanceWindowsDisk(id string) string { + return filepath.Join(p.InstanceDir(id), "windows.qcow2") +} + +func (p *Paths) InstanceOVMFCode(id string) string { + return filepath.Join(p.InstanceDir(id), "OVMF_CODE.fd") +} + +func (p *Paths) InstanceOVMFVars(id string) string { + return filepath.Join(p.InstanceDir(id), "OVMF_VARS.fd") +} + +func (p *Paths) InstanceTPMDir(id string) string { + return filepath.Join(p.InstanceDir(id), "tpm") +} + +func (p *Paths) InstanceTPMSocket(id string) string { + return filepath.Join(p.InstanceDir(id), "swtpm.sock") +} + // InstanceVolumeOverlay returns the path to a volume's overlay disk for an instance. func (p *Paths) InstanceVolumeOverlay(instanceID, volumeID string) string { return filepath.Join(p.InstanceDir(instanceID), "vol-overlays", volumeID+".raw") From 6584b317f973a16097a5b31730c1ee667ad935ff Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:08:01 +0000 Subject: [PATCH 10/29] Harden Windows image materialization --- docs/windows-images.md | 2 +- lib/images/machine.go | 53 ++++++++++++++++++++++++++++------- lib/images/machine_test.go | 47 +++++++++++++++++++++++++++++-- lib/instances/fork.go | 3 ++ lib/instances/restore.go | 3 ++ lib/instances/snapshot.go | 9 ++++++ lib/instances/standby.go | 3 ++ lib/instances/windows.go | 10 +++++++ lib/instances/windows_test.go | 8 ++++++ 9 files changed, 125 insertions(+), 13 deletions(-) diff --git a/docs/windows-images.md b/docs/windows-images.md index 98ba2b527..36537d947 100644 --- a/docs/windows-images.md +++ b/docs/windows-images.md @@ -16,6 +16,6 @@ A machine image uses these OCI config labels: Hypeman materializes the base as immutable sparse raw. It rewrites the persona's qcow2 backing header to the cache-owned base path, ignoring any artifact-supplied backing path. At instance creation, Hypeman reflink-clones the immutable persona into a writable `windows.qcow2`; the clone remains backed directly by the raw base. -The base must be pulled before its personas. A base cannot be deleted while any cached persona references its digest. +The base must be pulled before its personas. A base cannot be deleted while any cached persona references its digest. Instance references are not tracked by the image cache, matching existing Linux behavior: do not delete a base while any Windows instance cloned from one of its personas exists. Windows installation media, activation material, credentials, and generated disks belong in private registries and must not be committed to this repository. diff --git a/lib/images/machine.go b/lib/images/machine.go index c56a62c5e..e102cc976 100644 --- a/lib/images/machine.go +++ b/lib/images/machine.go @@ -128,10 +128,14 @@ type qemuImageInfo struct { VirtualSize int64 `json:"virtual-size"` BackingFilename string `json:"backing-filename"` BackingFileFormat string `json:"backing-filename-format"` + FormatSpecific struct { + Type string `json:"type"` + Data map[string]json.RawMessage `json:"data"` + } `json:"format-specific"` } -func inspectQEMUImage(path string) (qemuImageInfo, error) { - output, err := exec.Command("qemu-img", "info", "--output=json", path).CombinedOutput() +func inspectQEMUImage(path, format string) (qemuImageInfo, error) { + output, err := exec.Command("qemu-img", "info", "--output=json", "-f", format, path).CombinedOutput() if err != nil { return qemuImageInfo{}, fmt.Errorf("inspect machine disk: %w: %s", err, output) } @@ -142,11 +146,41 @@ func inspectQEMUImage(path string) (qemuImageInfo, error) { return info, nil } +func validateMachineSource(info qemuImageInfo, allowBacking bool) error { + if !allowBacking && info.BackingFilename != "" { + return fmt.Errorf("machine image source must not reference a backing file") + } + for _, feature := range []string{"data-file", "data-file-raw", "encrypt", "encryption", "encrypt-format"} { + if _, ok := info.FormatSpecific.Data[feature]; ok { + return fmt.Errorf("machine image source uses unsupported %s feature", feature) + } + } + return nil +} + +func qemuDiskFormat(format string) string { + if format == "vhd" { + return "vpc" + } + return format +} + func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine *MachineImage) (int64, error) { source, err := machineArtifactDisk(root, machine) if err != nil { return 0, err } + sourceFormat := qemuDiskFormat(machine.DiskFormat) + sourceInfo, err := inspectQEMUImage(source, sourceFormat) + if err != nil { + return 0, err + } + if sourceInfo.Format != sourceFormat { + return 0, fmt.Errorf("machine image source format is %s, expected %s", sourceInfo.Format, sourceFormat) + } + if err := validateMachineSource(sourceInfo, machine.Kind == MachineImageWindowsPersona); err != nil { + return 0, err + } destination := machineDiskPath(m.paths, ref.Repository(), ref.DigestHex(), machine.Kind) if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { @@ -165,11 +199,7 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine if machine.DiskFormat == "raw" { err = forkvm.CopyRegularFile(source, destination) } else { - format := machine.DiskFormat - if format == "vhd" { - format = "vpc" - } - output, convertErr := exec.Command("qemu-img", "convert", "-f", format, "-O", "raw", source, destination).CombinedOutput() + output, convertErr := exec.Command("qemu-img", "convert", "-f", sourceFormat, "-O", "raw", source, destination).CombinedOutput() if convertErr != nil { err = fmt.Errorf("convert Windows base to raw: %w: %s", convertErr, output) } @@ -177,7 +207,7 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine if err != nil { return 0, fmt.Errorf("materialize Windows base: %w", err) } - info, err := inspectQEMUImage(destination) + info, err := inspectQEMUImage(destination, "raw") if err != nil { return 0, err } @@ -200,14 +230,17 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine if err != nil { return 0, fmt.Errorf("set persona backing file: %w: %s", err, output) } - info, err := inspectQEMUImage(destination) + info, err := inspectQEMUImage(destination, "qcow2") if err != nil { return 0, err } + if err := validateMachineSource(info, true); err != nil { + return 0, err + } if info.Format != "qcow2" || info.BackingFileFormat != "raw" || info.BackingFilename != basePath { return 0, fmt.Errorf("invalid persona disk backing configuration") } - baseInfo, err := inspectQEMUImage(basePath) + baseInfo, err := inspectQEMUImage(basePath, "raw") if err != nil { return 0, err } diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go index 2051ac795..3dd96430e 100644 --- a/lib/images/machine_test.go +++ b/lib/images/machine_test.go @@ -63,6 +63,49 @@ func TestMachineArtifactDiskRejectsSymlinkEscape(t *testing.T) { assert.ErrorContains(t, err, "escapes artifact root") } +func TestMaterializeRejectsExternalDiskReferences(t *testing.T) { + if _, err := exec.LookPath("qemu-img"); err != nil { + if os.Getenv("CI") == "true" { + t.Fatal("qemu-img is required in CI") + } + t.Skip("qemu-img is unavailable") + } + + p := paths.New(t.TempDir()) + m := &manager{paths: p} + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "hypeman"), 0755)) + + backed := filepath.Join(root, "hypeman", "backed.qcow2") + output, err := exec.Command("qemu-img", "create", "-f", "qcow2", "-F", "raw", "-b", "/etc/passwd", backed, "4M").CombinedOutput() + require.NoError(t, err, "%s", output) + meta := windowsMachineMetadata(MachineImageWindowsBase, "hypeman/backed.qcow2", "") + meta.Labels[MachineImageDiskFormatLabel] = "qcow2" + machine, err := parseMachineImage(meta) + require.NoError(t, err) + ref, err := ParseNormalizedRef("registry.example/windows/base@sha256:" + strings.Repeat("5", 64)) + require.NoError(t, err) + _, err = m.materializeMachineImage(NewResolvedRef(ref, ref.Digest()), root, machine) + assert.ErrorContains(t, err, "must not reference a backing file") + + dataFile := filepath.Join(t.TempDir(), "external.raw") + require.NoError(t, os.WriteFile(dataFile, make([]byte, 4<<20), 0644)) + external := filepath.Join(root, "hypeman", "external.qcow2") + output, err = exec.Command("qemu-img", "create", "-f", "qcow2", "-o", "data_file="+dataFile+",data_file_raw=on", external, "4M").CombinedOutput() + require.NoError(t, err, "%s", output) + persona := windowsMachineMetadata( + MachineImageWindowsPersona, + "hypeman/external.qcow2", + "registry.example/windows/base@sha256:"+strings.Repeat("6", 64), + ) + machine, err = parseMachineImage(persona) + require.NoError(t, err) + personaRef, err := ParseNormalizedRef("registry.example/windows/persona@sha256:" + strings.Repeat("7", 64)) + require.NoError(t, err) + _, err = m.materializeMachineImage(NewResolvedRef(personaRef, personaRef.Digest()), root, machine) + assert.ErrorContains(t, err, "unsupported data-file feature") +} + func TestMaterializeWindowsBaseFormats(t *testing.T) { if _, err := exec.LookPath("qemu-img"); err != nil { if os.Getenv("CI") == "true" { @@ -101,7 +144,7 @@ func TestMaterializeWindowsBaseFormats(t *testing.T) { require.NoError(t, err) _, err = m.materializeMachineImage(resolved, root, machine) require.NoError(t, err) - info, err := inspectQEMUImage(machineDiskPath(p, ref.Repository(), digest, MachineImageWindowsBase)) + info, err := inspectQEMUImage(machineDiskPath(p, ref.Repository(), digest, MachineImageWindowsBase), "raw") require.NoError(t, err) assert.Equal(t, "raw", info.Format) }) @@ -164,7 +207,7 @@ func TestMaterializeWindowsBaseAndPersona(t *testing.T) { require.NoError(t, err) personaPath := machineDiskPath(p, personaRef.Repository(), personaDigest, MachineImageWindowsPersona) - info, err := inspectQEMUImage(personaPath) + info, err := inspectQEMUImage(personaPath, "qcow2") require.NoError(t, err) assert.Equal(t, "qcow2", info.Format) assert.Equal(t, "raw", info.BackingFileFormat) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index ea6d3a4c5..260b123e5 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -42,6 +42,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if err != nil { return nil, "", false, err } + if err := rejectWindowsSnapshotLifecycle(meta.Platform, "fork"); err != nil { + return nil, "", false, err + } source := m.toInstance(ctx, meta) targetState, err := resolveForkTargetState(req.TargetState, source.State) if err != nil { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index c209274e2..85d7c01bd 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -43,6 +43,9 @@ func (m *manager) restoreInstance( return nil, err } + if err := rejectWindowsSnapshotLifecycle(meta.Platform, "restore from standby"); err != nil { + return nil, err + } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 669d087f0..41709f7b2 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -63,6 +63,9 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err != nil { return nil, err } + if err := rejectWindowsSnapshotLifecycle(meta.Platform, "snapshot creation"); err != nil { + return nil, err + } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata @@ -251,6 +254,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if err != nil { return nil, err } + if err := rejectWindowsSnapshotLifecycle(rec.StoredMetadata.Platform, "snapshot restore"); err != nil { + return nil, err + } if rec.Snapshot.SourceInstanceID != id { return nil, fmt.Errorf("%w: snapshot %s belongs to instance %s", ErrInvalidRequest, snapshotID, rec.Snapshot.SourceInstanceID) } @@ -371,6 +377,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, err } + if err := rejectWindowsSnapshotLifecycle(rec.StoredMetadata.Platform, "snapshot fork"); err != nil { + return nil, err + } if err := validateForkVolumeSafety(rec.StoredMetadata.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 6913a9895..ae2a728bb 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -44,6 +44,9 @@ func (m *manager) standbyInstance( return nil, err } + if err := rejectWindowsSnapshotLifecycle(meta.Platform, "standby"); err != nil { + return nil, err + } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 13fc8991e..824aabb93 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -52,6 +52,16 @@ func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, hvTyp if req.NetworkEgress != nil || len(req.Credentials) != 0 { return fmt.Errorf("%w: Windows instances do not yet support managed egress or credentials", ErrInvalidRequest) } + if req.SnapshotPolicy != nil || req.AutoStandby != nil { + return fmt.Errorf("%w: Windows snapshot policies are added in the snapshots phase", ErrInvalidRequest) + } + return nil +} + +func rejectWindowsSnapshotLifecycle(platform, operation string) error { + if isWindowsPlatform(platform) { + return fmt.Errorf("%w: %s is not supported for Windows until the snapshots phase", ErrNotSupported, operation) + } return nil } diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index 112f654ed..e1365897d 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" @@ -41,6 +42,8 @@ func TestValidateWindowsCreate(t *testing.T) { {name: "small memory", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Size: 2 << 30}}, {name: "one CPU", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Vcpus: 1}}, {name: "command", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Cmd: []string{"cmd.exe"}}}, + {name: "snapshot policy", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}}, + {name: "auto standby", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -49,6 +52,11 @@ func TestValidateWindowsCreate(t *testing.T) { } } +func TestRejectWindowsSnapshotLifecycle(t *testing.T) { + assert.ErrorIs(t, rejectWindowsSnapshotLifecycle("windows/amd64", "fork"), ErrNotSupported) + assert.NoError(t, rejectWindowsSnapshotLifecycle("linux/amd64", "fork")) +} + func TestBuildWindowsHypervisorConfig(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} From 1d87481387f52cc0e21a77b9045d6666eb018478 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:07:07 +0000 Subject: [PATCH 11/29] Keep machine image tests portable --- lib/images/machine_oci_integration_test.go | 7 +---- lib/images/machine_test.go | 32 ++++++++++------------ 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/lib/images/machine_oci_integration_test.go b/lib/images/machine_oci_integration_test.go index c1f12d0b6..de6bd097c 100644 --- a/lib/images/machine_oci_integration_test.go +++ b/lib/images/machine_oci_integration_test.go @@ -63,12 +63,7 @@ func waitForMachineImage(t *testing.T, manager Manager, name string) *Image { } func TestMachineArtifactsPullFromOCI(t *testing.T) { - if _, err := exec.LookPath("qemu-img"); err != nil { - if os.Getenv("CI") == "true" { - t.Fatal("qemu-img is required in CI") - } - t.Skip("qemu-img is unavailable") - } + requireQEMUImg(t) registry := httptest.NewServer(gcrregistry.New()) defer registry.Close() diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go index 3dd96430e..5f97681df 100644 --- a/lib/images/machine_test.go +++ b/lib/images/machine_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" @@ -12,6 +13,16 @@ import ( "github.com/stretchr/testify/require" ) +func requireQEMUImg(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("qemu-img"); err != nil { + if os.Getenv("CI") == "true" && runtime.GOOS == "linux" { + t.Fatal("qemu-img is required in Linux CI") + } + t.Skip("qemu-img is unavailable") + } +} + func windowsMachineMetadata(kind MachineImageKind, diskPath, base string) *containerMetadata { format := "raw" if kind == MachineImageWindowsPersona { @@ -64,12 +75,7 @@ func TestMachineArtifactDiskRejectsSymlinkEscape(t *testing.T) { } func TestMaterializeRejectsExternalDiskReferences(t *testing.T) { - if _, err := exec.LookPath("qemu-img"); err != nil { - if os.Getenv("CI") == "true" { - t.Fatal("qemu-img is required in CI") - } - t.Skip("qemu-img is unavailable") - } + requireQEMUImg(t) p := paths.New(t.TempDir()) m := &manager{paths: p} @@ -107,12 +113,7 @@ func TestMaterializeRejectsExternalDiskReferences(t *testing.T) { } func TestMaterializeWindowsBaseFormats(t *testing.T) { - if _, err := exec.LookPath("qemu-img"); err != nil { - if os.Getenv("CI") == "true" { - t.Fatal("qemu-img is required in CI") - } - t.Skip("qemu-img is unavailable") - } + requireQEMUImg(t) formats := []struct { label string @@ -152,12 +153,7 @@ func TestMaterializeWindowsBaseFormats(t *testing.T) { } func TestMaterializeWindowsBaseAndPersona(t *testing.T) { - if _, err := exec.LookPath("qemu-img"); err != nil { - if os.Getenv("CI") == "true" { - t.Fatal("qemu-img is required in CI") - } - t.Skip("qemu-img is unavailable") - } + requireQEMUImg(t) p := paths.New(t.TempDir()) m := &manager{paths: p} From 1fd0c5f48a626f5b95e9842da9e05d38d9f23516 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:07:36 +0000 Subject: [PATCH 12/29] Isolate the Windows images CI gate --- .github/workflows/test.yml | 17 +++++++++++++++++ .../windows_images_integration_linux_test.go | 3 +++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e5988a68..9025bb130 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -135,6 +135,23 @@ jobs: done exit 1 + - name: Test Windows machine images + run: | + TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" + for attempt in 1 2 3; do + if sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_IMAGES_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run '^TestWindowsImagesIntegration$' -timeout 2m ./lib/instances; then + exit 0 + fi + test "$attempt" = 3 || sleep 5 + done + exit 1 + # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. - name: Login to Docker Hub diff --git a/lib/instances/windows_images_integration_linux_test.go b/lib/instances/windows_images_integration_linux_test.go index 5eb869f91..db772bef4 100644 --- a/lib/instances/windows_images_integration_linux_test.go +++ b/lib/instances/windows_images_integration_linux_test.go @@ -53,6 +53,9 @@ func requireWindowsFixture(t *testing.T) string { } func TestWindowsImagesIntegration(t *testing.T) { + if os.Getenv("HYPEMAN_RUN_WINDOWS_IMAGES_INTEGRATION") != "1" { + t.Skip("run by the dedicated Windows images CI gate") + } fixture := requireWindowsFixture(t) acquireHeavyIO(t) From dd0654d80f7bc6f36f3ec211fc92244e06900eeb Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:17:57 +0000 Subject: [PATCH 13/29] Build embedded agent before Windows gates --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9025bb130..051788311 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -137,6 +137,7 @@ jobs: - name: Test Windows machine images run: | + make build-embedded TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" for attempt in 1 2 3; do if sudo env \ From 31b5a61691d6c4b9276a722805b1085b43af5f19 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:14:48 +0000 Subject: [PATCH 14/29] Use image naming for Windows artifacts --- .github/workflows/test.yml | 22 --- docs/windows-images.md | 8 +- lib/images/README.md | 8 ++ lib/images/machine.go | 38 ++--- lib/images/machine_oci_integration_test.go | 22 +-- lib/images/machine_test.go | 54 ++++---- lib/instances/README.md | 8 ++ lib/instances/create.go | 3 +- lib/instances/windows.go | 22 +-- .../windows_images_integration_linux_test.go | 131 ------------------ lib/instances/windows_test.go | 31 +++-- 11 files changed, 105 insertions(+), 242 deletions(-) delete mode 100644 lib/instances/windows_images_integration_linux_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 051788311..caf47c339 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,10 +113,6 @@ jobs: done test -f "$HYPEMAN_WINDOWS_OVMF_CODE" test -f "$HYPEMAN_WINDOWS_OVMF_VARS" - test -r /ci/windows/base.raw - test -r /ci/windows/persona.qcow2 - qemu-img info --output=json /ci/windows/persona.qcow2 \ - | jq -e '.format == "qcow2" and .["backing-filename-format"] == "raw"' >/dev/null - name: Test Windows hypervisor primitives run: | @@ -135,24 +131,6 @@ jobs: done exit 1 - - name: Test Windows machine images - run: | - make build-embedded - TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" - for attempt in 1 2 3; do - if sudo env \ - "PATH=$TEST_PATH" \ - "CI=true" \ - "HYPEMAN_RUN_WINDOWS_IMAGES_INTEGRATION=1" \ - "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ - "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ - go test -count=1 -run '^TestWindowsImagesIntegration$' -timeout 2m ./lib/instances; then - exit 0 - fi - test "$attempt" = 3 || sleep 5 - done - exit 1 - # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. - name: Login to Docker Hub diff --git a/docs/windows-images.md b/docs/windows-images.md index 36537d947..0b73a1478 100644 --- a/docs/windows-images.md +++ b/docs/windows-images.md @@ -4,18 +4,16 @@ Hypeman accepts Windows desktop disks as OCI images for `windows/amd64`. Ordinar A machine image uses these OCI config labels: -| Label | Base | Persona | +| Label | Base | Image | |---|---|---| | `io.hypeman.machine-image.version` | `1` | `1` | -| `io.hypeman.machine-image.kind` | `windows-base` | `windows-persona` | +| `io.hypeman.machine-image.kind` | `windows-base` | `windows-image` | | `io.hypeman.machine-image.disk-path` | relative path to the source disk | relative path to a qcow2 delta | | `io.hypeman.machine-image.disk-format` | `raw`, `qcow2`, `vhd`, or `vhdx` | `qcow2` | | `io.hypeman.machine-image.base` | omitted | digest-pinned base reference | | `io.hypeman.machine-image.tpm` | `2.0` | `2.0` | | `io.hypeman.machine-image.secure-boot` | `required` | `required` | -Hypeman materializes the base as immutable sparse raw. It rewrites the persona's qcow2 backing header to the cache-owned base path, ignoring any artifact-supplied backing path. At instance creation, Hypeman reflink-clones the immutable persona into a writable `windows.qcow2`; the clone remains backed directly by the raw base. - -The base must be pulled before its personas. A base cannot be deleted while any cached persona references its digest. Instance references are not tracked by the image cache, matching existing Linux behavior: do not delete a base while any Windows instance cloned from one of its personas exists. +The base must be pulled before its dependent Windows images. A base cannot be deleted while any cached image references its digest. Instance references are not tracked by the image cache, matching existing Linux behavior: do not delete a base while a dependent Windows instance exists. Windows installation media, activation material, credentials, and generated disks belong in private registries and must not be committed to this repository. diff --git a/lib/images/README.md b/lib/images/README.md index eb2d422be..1d948554a 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -49,6 +49,14 @@ OCI Registry → go-containerregistry → OCI Layout → umoci → rootfs/ → m **Alternative:** ext4 without journal works but erofs is optimized for this exact use case +## Windows machine images + +Windows uses the same image-manager contract as Linux: callers pull a named image and create instances from it. The launchable artifact is therefore called a Windows image. A `windows-base` artifact is separate because it is an immutable storage dependency rather than a launchable image. + +Base disks may arrive as raw, qcow2, VHD, or VHDX. Materialization normalizes them to immutable sparse raw so every dependent image has one stable backing format. A `windows-image` artifact is a qcow2 delta with a digest-pinned base reference. Hypeman ignores its supplied backing path and rewrites the header to the cache-owned base, preventing an artifact from retaining an external host path. The image and base must have identical virtual sizes. + +Instance creation reflink-clones the cached Windows image into a private writable qcow2 disk while leaving the cached source immutable. Its virtual size becomes the instance disk size; unlike Linux's separate overlay, this disk is not resized because Windows online partition and filesystem growth are not part of the launch contract. + ## Filesystem Layout (storage.go, oci.go) Content-addressable storage with tag symlinks (similar to Docker/Unikraft): diff --git a/lib/images/machine.go b/lib/images/machine.go index e102cc976..74a511e99 100644 --- a/lib/images/machine.go +++ b/lib/images/machine.go @@ -27,8 +27,8 @@ const ( type MachineImageKind string const ( - MachineImageWindowsBase MachineImageKind = "windows-base" - MachineImageWindowsPersona MachineImageKind = "windows-persona" + MachineImageWindowsBase MachineImageKind = "windows-base" + MachineImageWindowsImage MachineImageKind = "windows-image" ) // MachineImage describes a bootable disk artifact. The OCI manifest remains @@ -86,13 +86,13 @@ func parseMachineImage(meta *containerMetadata) (*MachineImage, error) { if machine.Base != "" { return nil, fmt.Errorf("Windows base image cannot reference another base") } - case MachineImageWindowsPersona: + case MachineImageWindowsImage: if machine.DiskFormat != "qcow2" { - return nil, fmt.Errorf("Windows persona disk format must be qcow2") + return nil, fmt.Errorf("Windows image disk format must be qcow2") } base, err := ParseNormalizedRef(machine.Base) if err != nil || !base.IsDigest() { - return nil, fmt.Errorf("Windows persona base must be a digest-pinned OCI reference") + return nil, fmt.Errorf("Windows image base must be a digest-pinned OCI reference") } default: return nil, fmt.Errorf("unsupported machine image kind %q", machine.Kind) @@ -178,7 +178,7 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine if sourceInfo.Format != sourceFormat { return 0, fmt.Errorf("machine image source format is %s, expected %s", sourceInfo.Format, sourceFormat) } - if err := validateMachineSource(sourceInfo, machine.Kind == MachineImageWindowsPersona); err != nil { + if err := validateMachineSource(sourceInfo, machine.Kind == MachineImageWindowsImage); err != nil { return 0, err } @@ -215,12 +215,12 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine return 0, fmt.Errorf("Windows base disk must be raw, got %s", info.Format) } machine.VirtualSize = info.VirtualSize - case MachineImageWindowsPersona: + case MachineImageWindowsImage: if err := forkvm.CopyRegularFile(source, destination); err != nil { - return 0, fmt.Errorf("materialize Windows persona: %w", err) + return 0, fmt.Errorf("materialize Windows image: %w", err) } if err := os.Chmod(destination, 0600); err != nil { - return 0, fmt.Errorf("make Windows persona writable for validation: %w", err) + return 0, fmt.Errorf("make Windows image writable for validation: %w", err) } basePath, err := m.resolveMachineBase(machine.Base) if err != nil { @@ -228,7 +228,7 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine } output, err := exec.Command("qemu-img", "rebase", "-u", "-f", "qcow2", "-F", "raw", "-b", basePath, destination).CombinedOutput() if err != nil { - return 0, fmt.Errorf("set persona backing file: %w: %s", err, output) + return 0, fmt.Errorf("set image backing file: %w: %s", err, output) } info, err := inspectQEMUImage(destination, "qcow2") if err != nil { @@ -238,14 +238,14 @@ func (m *manager) materializeMachineImage(ref *ResolvedRef, root string, machine return 0, err } if info.Format != "qcow2" || info.BackingFileFormat != "raw" || info.BackingFilename != basePath { - return 0, fmt.Errorf("invalid persona disk backing configuration") + return 0, fmt.Errorf("invalid Windows image disk backing configuration") } baseInfo, err := inspectQEMUImage(basePath, "raw") if err != nil { return 0, err } if info.VirtualSize != baseInfo.VirtualSize { - return 0, fmt.Errorf("persona virtual size %d does not match base %d", info.VirtualSize, baseInfo.VirtualSize) + return 0, fmt.Errorf("image virtual size %d does not match base %d", info.VirtualSize, baseInfo.VirtualSize) } machine.VirtualSize = info.VirtualSize } @@ -278,8 +278,8 @@ func (m *manager) resolveMachineBase(reference string) (string, error) { func machineDiskPath(p *paths.Paths, repository, digestHex string, kind MachineImageKind) string { name := "base.raw" - if kind == MachineImageWindowsPersona { - name = "persona.qcow2" + if kind == MachineImageWindowsImage { + name = "image.qcow2" } return filepath.Join(p.ImageDigestDir(repository, digestHex), name) } @@ -290,12 +290,12 @@ func (m *manager) ensureNoMachineDependents(repository, digestHex string) error return err } for _, meta := range metas { - if meta.Machine == nil || meta.Machine.Kind != MachineImageWindowsPersona { + if meta.Machine == nil || meta.Machine.Kind != MachineImageWindowsImage { continue } base, err := ParseNormalizedRef(meta.Machine.Base) if err == nil && base.Repository() == repository && base.DigestHex() == digestHex { - return fmt.Errorf("cannot delete Windows base while persona %s depends on it", meta.Name) + return fmt.Errorf("cannot delete Windows base while image %s depends on it", meta.Name) } } return nil @@ -313,7 +313,7 @@ func GetMachineDiskPath(p *paths.Paths, imageName, digest string, machine *Machi return machineDiskPath(p, ref.Repository(), strings.TrimPrefix(digest, "sha256:"), machine.Kind), nil } -// IsWindowsPersona reports whether an image is directly launchable as a Windows desktop. -func IsWindowsPersona(image *Image) bool { - return image != nil && image.Machine != nil && image.Machine.Kind == MachineImageWindowsPersona +// IsWindowsImage reports whether an image is directly launchable as a Windows desktop. +func IsWindowsImage(image *Image) bool { + return image != nil && image.Machine != nil && image.Machine.Kind == MachineImageWindowsImage } diff --git a/lib/images/machine_oci_integration_test.go b/lib/images/machine_oci_integration_test.go index de6bd097c..aa2c20742 100644 --- a/lib/images/machine_oci_integration_test.go +++ b/lib/images/machine_oci_integration_test.go @@ -88,21 +88,21 @@ func TestMachineArtifactsPullFromOCI(t *testing.T) { readyBase := waitForMachineImage(t, manager, createdBase.Name) require.Equal(t, MachineImageWindowsBase, readyBase.Machine.Kind) - personaFile := filepath.Join(t.TempDir(), "persona.qcow2") - output, err := exec.Command("qemu-img", "create", "-f", "qcow2", personaFile, "4M").CombinedOutput() + imageFile := filepath.Join(t.TempDir(), "image.qcow2") + output, err := exec.Command("qemu-img", "create", "-f", "qcow2", imageFile, "4M").CombinedOutput() require.NoError(t, err, "%s", output) - personaBytes, err := os.ReadFile(personaFile) + imageBytes, err := os.ReadFile(imageFile) require.NoError(t, err) baseReference := baseTag.Context().Name() + "@" + readyBase.Digest - personaLabels := windowsMachineMetadata(MachineImageWindowsPersona, "hypeman/persona.qcow2", baseReference).Labels - personaImage := machineArtifactOCIImage(t, "hypeman/persona.qcow2", personaBytes, personaLabels) - personaTag, err := name.NewTag(registry.Listener.Addr().String()+"/windows/persona:test", name.Insecure) + imageLabels := windowsMachineMetadata(MachineImageWindowsImage, "hypeman/image.qcow2", baseReference).Labels + ociImage := machineArtifactOCIImage(t, "hypeman/image.qcow2", imageBytes, imageLabels) + imageTag, err := name.NewTag(registry.Listener.Addr().String()+"/windows/image:test", name.Insecure) require.NoError(t, err) - require.NoError(t, remote.Write(personaTag, personaImage)) + require.NoError(t, remote.Write(imageTag, ociImage)) - createdPersona, err := manager.CreateImage(context.Background(), CreateImageRequest{Name: personaTag.String(), Platform: "windows/amd64"}) + createdImage, err := manager.CreateImage(context.Background(), CreateImageRequest{Name: imageTag.String(), Platform: "windows/amd64"}) require.NoError(t, err) - readyPersona := waitForMachineImage(t, manager, createdPersona.Name) - require.Equal(t, MachineImageWindowsPersona, readyPersona.Machine.Kind) - require.Equal(t, readyBase.Machine.VirtualSize, readyPersona.Machine.VirtualSize) + readyImage := waitForMachineImage(t, manager, createdImage.Name) + require.Equal(t, MachineImageWindowsImage, readyImage.Machine.Kind) + require.Equal(t, readyBase.Machine.VirtualSize, readyImage.Machine.VirtualSize) } diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go index 5f97681df..ee2801084 100644 --- a/lib/images/machine_test.go +++ b/lib/images/machine_test.go @@ -25,7 +25,7 @@ func requireQEMUImg(t *testing.T) { func windowsMachineMetadata(kind MachineImageKind, diskPath, base string) *containerMetadata { format := "raw" - if kind == MachineImageWindowsPersona { + if kind == MachineImageWindowsImage { format = "qcow2" } return &containerMetadata{ @@ -48,13 +48,13 @@ func TestParseMachineImage(t *testing.T) { require.NoError(t, err) assert.Equal(t, MachineImageWindowsBase, base.Kind) - persona, err := parseMachineImage(windowsMachineMetadata( - MachineImageWindowsPersona, + image, err := parseMachineImage(windowsMachineMetadata( + MachineImageWindowsImage, "hypeman/disk.qcow2", "registry.example/base@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", )) require.NoError(t, err) - assert.Equal(t, MachineImageWindowsPersona, persona.Kind) + assert.Equal(t, MachineImageWindowsImage, image.Kind) _, err = parseMachineImage(&containerMetadata{OS: "windows", Architecture: "amd64", Labels: map[string]string{}}) assert.ErrorContains(t, err, "ordinary Windows container images are not bootable") @@ -99,16 +99,16 @@ func TestMaterializeRejectsExternalDiskReferences(t *testing.T) { external := filepath.Join(root, "hypeman", "external.qcow2") output, err = exec.Command("qemu-img", "create", "-f", "qcow2", "-o", "data_file="+dataFile+",data_file_raw=on", external, "4M").CombinedOutput() require.NoError(t, err, "%s", output) - persona := windowsMachineMetadata( - MachineImageWindowsPersona, + image := windowsMachineMetadata( + MachineImageWindowsImage, "hypeman/external.qcow2", "registry.example/windows/base@sha256:"+strings.Repeat("6", 64), ) - machine, err = parseMachineImage(persona) + machine, err = parseMachineImage(image) require.NoError(t, err) - personaRef, err := ParseNormalizedRef("registry.example/windows/persona@sha256:" + strings.Repeat("7", 64)) + imageRef, err := ParseNormalizedRef("registry.example/windows/image@sha256:" + strings.Repeat("7", 64)) require.NoError(t, err) - _, err = m.materializeMachineImage(NewResolvedRef(personaRef, personaRef.Digest()), root, machine) + _, err = m.materializeMachineImage(NewResolvedRef(imageRef, imageRef.Digest()), root, machine) assert.ErrorContains(t, err, "unsupported data-file feature") } @@ -152,7 +152,7 @@ func TestMaterializeWindowsBaseFormats(t *testing.T) { } } -func TestMaterializeWindowsBaseAndPersona(t *testing.T) { +func TestMaterializeWindowsBaseAndImage(t *testing.T) { requireQEMUImg(t) p := paths.New(t.TempDir()) @@ -183,27 +183,27 @@ func TestMaterializeWindowsBaseAndPersona(t *testing.T) { SizeBytes: 4 << 20, })) - personaDigest := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - personaRef, err := ParseNormalizedRef("registry.example/windows/persona@sha256:" + personaDigest) + imageDigest := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + imageRef, err := ParseNormalizedRef("registry.example/windows/image@sha256:" + imageDigest) require.NoError(t, err) - resolvedPersona := NewResolvedRef(personaRef, "sha256:"+personaDigest) - personaRoot := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(personaRoot, "hypeman"), 0755)) - personaSource := filepath.Join(personaRoot, "hypeman", "disk.qcow2") - output, err := exec.Command("qemu-img", "create", "-f", "qcow2", personaSource, "4M").CombinedOutput() + resolvedImage := NewResolvedRef(imageRef, "sha256:"+imageDigest) + imageRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(imageRoot, "hypeman"), 0755)) + imageSource := filepath.Join(imageRoot, "hypeman", "disk.qcow2") + output, err := exec.Command("qemu-img", "create", "-f", "qcow2", imageSource, "4M").CombinedOutput() require.NoError(t, err, "%s", output) - personaMachine, err := parseMachineImage(windowsMachineMetadata( - MachineImageWindowsPersona, + imageMachine, err := parseMachineImage(windowsMachineMetadata( + MachineImageWindowsImage, "hypeman/disk.qcow2", baseRef.String(), )) require.NoError(t, err) - _, err = m.materializeMachineImage(resolvedPersona, personaRoot, personaMachine) + _, err = m.materializeMachineImage(resolvedImage, imageRoot, imageMachine) require.NoError(t, err) - personaPath := machineDiskPath(p, personaRef.Repository(), personaDigest, MachineImageWindowsPersona) - info, err := inspectQEMUImage(personaPath, "qcow2") + imagePath := machineDiskPath(p, imageRef.Repository(), imageDigest, MachineImageWindowsImage) + info, err := inspectQEMUImage(imagePath, "qcow2") require.NoError(t, err) assert.Equal(t, "qcow2", info.Format) assert.Equal(t, "raw", info.BackingFileFormat) @@ -211,15 +211,15 @@ func TestMaterializeWindowsBaseAndPersona(t *testing.T) { baseMeta, err := readMetadata(p, baseRef.Repository(), baseDigest) require.NoError(t, err) - personaMeta := &imageMetadata{ - Name: personaRef.String(), - Digest: "sha256:" + personaDigest, + imageMeta := &imageMetadata{ + Name: imageRef.String(), + Digest: "sha256:" + imageDigest, Platform: "windows/amd64", Status: StatusReady, - Machine: personaMachine, + Machine: imageMachine, SizeBytes: info.VirtualSize, } - require.NoError(t, writeMetadata(p, personaRef.Repository(), personaDigest, personaMeta)) + require.NoError(t, writeMetadata(p, imageRef.Repository(), imageDigest, imageMeta)) assert.ErrorContains(t, m.ensureNoMachineDependents(baseRef.Repository(), baseDigest), "depends on it") assert.ErrorContains(t, m.DeleteImage(t.Context(), baseRef.String()), "depends on it") assert.DirExists(t, p.ImageDigestDir(baseRef.Repository(), baseDigest)) diff --git a/lib/instances/README.md b/lib/instances/README.md index 13ada0565..d4015ec0c 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -22,6 +22,14 @@ Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEM - `Shutdown` - VM shutdown, VMM exists (CH native) - `Standby` - No VMM, snapshot exists (can restore) +### Windows launch defaults + +Windows machine images boot through UEFI with Secure Boot and TPM 2.0. The default 8 GiB memory and 4 vCPUs provide headroom for Windows 11 startup and the guest service; the lower admission limits of 4 GiB and 2 vCPUs permit explicitly sized, constrained workloads without making that minimum the default. + +The launchable Windows image already defines its virtual disk size, so instance creation clones that size exactly. Windows disk, partition, and filesystem growth are not implemented, and an `overlay_size` that differs from the image is rejected rather than silently presenting inconsistent capacity. + +A Windows VM remains `Initializing` until its guest agent answers over VioSock. This avoids treating firmware completion as application readiness. + ### Why Config Disk? (configdisk.go) **What:** Read-only erofs disk with instance configuration diff --git a/lib/instances/create.go b/lib/instances/create.go index 55ec459b4..d4ff30db4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -130,7 +130,8 @@ func (m *manager) createInstance( } windows := isWindowsPlatform(imageInfo.Platform) if windows { - if err := validateWindowsCreate(req, imageInfo, hvType); err != nil { + caps, _ := hypervisor.CapabilitiesForType(hvType) + if err := validateWindowsCreate(req, imageInfo, caps); err != nil { return nil, err } } diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 824aabb93..6a37c6c30 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -15,15 +15,15 @@ func isWindowsPlatform(platform string) bool { return strings.EqualFold(strings.TrimSpace(platform), "windows/amd64") } -func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, hvType hypervisor.Type) error { - if !images.IsWindowsPersona(image) { - return fmt.Errorf("%w: Windows instances require a persona image", ErrInvalidRequest) +func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, caps hypervisor.Capabilities) error { + if !images.IsWindowsImage(image) { + return fmt.Errorf("%w: Windows instances require a Windows machine image", ErrInvalidRequest) } - if hvType != hypervisor.TypeQEMU { - return fmt.Errorf("%w: Windows instances require the qemu hypervisor", ErrInvalidRequest) + if !caps.SupportsUEFIBoot || !caps.SupportsTPM { + return fmt.Errorf("%w: selected hypervisor must support UEFI boot and TPM devices", ErrInvalidRequest) } if image.Machine.VirtualSize <= 0 { - return fmt.Errorf("%w: Windows persona is missing its virtual disk size", ErrInvalidRequest) + return fmt.Errorf("%w: Windows image is missing its virtual disk size", ErrInvalidRequest) } if req.HotplugSize != 0 { return fmt.Errorf("%w: Windows instances do not yet support hotplug memory", ErrInvalidRequest) @@ -87,12 +87,12 @@ func windowsFirmwareTemplates() (string, string, error) { } func (m *manager) prepareWindowsInstance(inst *StoredMetadata, image *images.Image) error { - persona, err := images.GetMachineDiskPath(m.paths, image.Name, image.Digest, image.Machine) + source, err := images.GetMachineDiskPath(m.paths, image.Name, image.Digest, image.Machine) if err != nil { return err } - if err := forkvm.CopyRegularFile(persona, m.paths.InstanceWindowsDisk(inst.Id)); err != nil { - return fmt.Errorf("clone Windows persona: %w", err) + if err := forkvm.CopyRegularFile(source, m.paths.InstanceWindowsDisk(inst.Id)); err != nil { + return fmt.Errorf("clone Windows image: %w", err) } if err := os.Chmod(m.paths.InstanceWindowsDisk(inst.Id), 0600); err != nil { return fmt.Errorf("make Windows instance disk writable: %w", err) @@ -121,8 +121,8 @@ func (m *manager) prepareWindowsInstance(inst *StoredMetadata, image *images.Ima } func (m *manager) buildWindowsHypervisorConfig(inst *Instance, image *images.Image, netConfig *network.NetworkConfig) (hypervisor.VMConfig, error) { - if !images.IsWindowsPersona(image) { - return hypervisor.VMConfig{}, fmt.Errorf("image is not a Windows persona") + if !images.IsWindowsImage(image) { + return hypervisor.VMConfig{}, fmt.Errorf("image is not a Windows machine image") } if _, err := os.Stat(m.paths.InstanceWindowsDisk(inst.Id)); err != nil { return hypervisor.VMConfig{}, fmt.Errorf("stat Windows instance disk: %w", err) diff --git a/lib/instances/windows_images_integration_linux_test.go b/lib/instances/windows_images_integration_linux_test.go deleted file mode 100644 index db772bef4..000000000 --- a/lib/instances/windows_images_integration_linux_test.go +++ /dev/null @@ -1,131 +0,0 @@ -//go:build linux && amd64 - -package instances - -import ( - "context" - "crypto/sha256" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "github.com/kernel/hypeman/lib/forkvm" - "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type windowsFixtureImageManager struct { - images.Manager - image *images.Image -} - -func (m windowsFixtureImageManager) CreateImage(context.Context, images.CreateImageRequest) (*images.Image, error) { - copy := *m.image - return ©, nil -} - -func (m windowsFixtureImageManager) GetImage(context.Context, string) (*images.Image, error) { - copy := *m.image - return ©, nil -} - -func (m windowsFixtureImageManager) WaitForReady(context.Context, string) error { return nil } - -func requireWindowsFixture(t *testing.T) string { - t.Helper() - path := os.Getenv("HYPEMAN_WINDOWS_TEST_PERSONA") - if path == "" { - path = "/ci/windows/persona.qcow2" - } - if _, err := os.Stat(path); err == nil { - return path - } - if os.Getenv("CI") == "true" { - t.Fatalf("required Windows persona fixture is missing: %s", path) - } - t.Skipf("Windows persona fixture is unavailable: %s", path) - return "" -} - -func TestWindowsImagesIntegration(t *testing.T) { - if os.Getenv("HYPEMAN_RUN_WINDOWS_IMAGES_INTEGRATION") != "1" { - t.Skip("run by the dedicated Windows images CI gate") - } - fixture := requireWindowsFixture(t) - acquireHeavyIO(t) - - manager, dataDir := setupTestManagerForQEMU(t) - p := paths.New(dataDir) - const digestHex = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - image := &images.Image{ - Name: "registry.example/windows/persona:integration", - Digest: "sha256:" + digestHex, - Platform: "windows/amd64", - Status: images.StatusReady, - Machine: &images.MachineImage{ - Kind: images.MachineImageWindowsPersona, - Base: "registry.example/windows/base@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - TPM: "2.0", - SecureBoot: "required", - VirtualSize: 80 << 30, - }, - } - manager.imageManager = windowsFixtureImageManager{image: image} - - personaPath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) - require.NoError(t, err) - require.NoError(t, forkvm.CopyRegularFile(fixture, personaPath)) - require.NoError(t, os.Chmod(personaPath, 0444)) - personaBytes, err := os.ReadFile(personaPath) - require.NoError(t, err) - personaHash := sha256.Sum256(personaBytes) - personaInfo, err := os.Stat(personaPath) - require.NoError(t, err) - - ctx := context.Background() - instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: "windows-images-integration", - Image: image.Name, - Platform: "windows/amd64", - Size: 8 << 30, - Vcpus: 4, - Hypervisor: hypervisor.TypeQEMU, - }) - require.NoError(t, err) - instanceID := instance.Id - t.Cleanup(func() { - if instanceID != "" { - _ = deleteTestInstanceNow(context.Background(), manager, instanceID) - } - }) - require.Equal(t, StateInitializing, instance.State) - require.FileExists(t, p.InstanceWindowsDisk(instance.Id)) - require.FileExists(t, p.InstanceOVMFVars(instance.Id)) - require.DirExists(t, p.InstanceTPMDir(instance.Id)) - assert.NoFileExists(t, p.InstanceOverlay(instance.Id)) - assert.NoFileExists(t, p.InstanceConfigDisk(instance.Id)) - - require.Eventually(t, func() bool { - info, err := os.Stat(p.InstanceWindowsDisk(instance.Id)) - return err == nil && info.Size() > personaInfo.Size() - }, 60*time.Second, 500*time.Millisecond, "Windows boot must write to the instance qcow2") - - sourceAfter, err := os.ReadFile(personaPath) - require.NoError(t, err) - assert.Equal(t, personaHash, sha256.Sum256(sourceAfter), "immutable persona changed during guest boot") - - stopped, err := manager.StopInstance(ctx, instance.Id) - require.NoError(t, err) - require.Equal(t, StateStopped, stopped.State) - output, err := exec.Command("qemu-img", "check", p.InstanceWindowsDisk(instance.Id)).CombinedOutput() - require.NoError(t, err, "%s", output) - - require.NoError(t, manager.DeleteInstance(ctx, instance.Id)) - instanceID = "" - assert.NoDirExists(t, filepath.Dir(p.InstanceWindowsDisk(instance.Id))) -} diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index e1365897d..130d7b165 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -12,14 +12,14 @@ import ( "github.com/stretchr/testify/require" ) -func windowsPersonaFixture() *images.Image { +func windowsImageFixture() *images.Image { return &images.Image{ - Name: "registry.example/windows/persona:test", + Name: "registry.example/windows/image:test", Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Platform: "windows/amd64", Status: images.StatusReady, Machine: &images.MachineImage{ - Kind: images.MachineImageWindowsPersona, + Kind: images.MachineImageWindowsImage, Base: "registry.example/windows/base@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", TPM: "2.0", SecureBoot: "required", @@ -29,25 +29,26 @@ func windowsPersonaFixture() *images.Image { } func TestValidateWindowsCreate(t *testing.T) { - image := windowsPersonaFixture() - require.NoError(t, validateWindowsCreate(CreateInstanceRequest{}, image, hypervisor.TypeQEMU)) + image := windowsImageFixture() + windowsCaps := hypervisor.Capabilities{SupportsUEFIBoot: true, SupportsTPM: true} + require.NoError(t, validateWindowsCreate(CreateInstanceRequest{}, image, windowsCaps)) tests := []struct { name string req CreateInstanceRequest - hv hypervisor.Type + caps hypervisor.Capabilities }{ - {name: "wrong hypervisor", hv: hypervisor.TypeCloudHypervisor}, - {name: "networking", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{NetworkEnabled: true}}, - {name: "small memory", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Size: 2 << 30}}, - {name: "one CPU", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Vcpus: 1}}, - {name: "command", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{Cmd: []string{"cmd.exe"}}}, - {name: "snapshot policy", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}}, - {name: "auto standby", hv: hypervisor.TypeQEMU, req: CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}}, + {name: "missing boot capabilities"}, + {name: "networking", caps: windowsCaps, req: CreateInstanceRequest{NetworkEnabled: true}}, + {name: "small memory", caps: windowsCaps, req: CreateInstanceRequest{Size: 2 << 30}}, + {name: "one CPU", caps: windowsCaps, req: CreateInstanceRequest{Vcpus: 1}}, + {name: "command", caps: windowsCaps, req: CreateInstanceRequest{Cmd: []string{"cmd.exe"}}}, + {name: "snapshot policy", caps: windowsCaps, req: CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}}, + {name: "auto standby", caps: windowsCaps, req: CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Error(t, validateWindowsCreate(tt.req, image, tt.hv)) + assert.Error(t, validateWindowsCreate(tt.req, image, tt.caps)) }) } } @@ -66,7 +67,7 @@ func TestBuildWindowsHypervisorConfig(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("fixture"), 0600)) } - config, err := m.buildWindowsHypervisorConfig(&Instance{StoredMetadata: stored}, windowsPersonaFixture(), nil) + config, err := m.buildWindowsHypervisorConfig(&Instance{StoredMetadata: stored}, windowsImageFixture(), nil) require.NoError(t, err) assert.Equal(t, hypervisor.BootModeUEFI, config.BootMode) assert.True(t, config.Firmware.SecureBoot) From 0dc80f4ac71f7448e8f89e227557824ba89e39ba Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:13:10 +0000 Subject: [PATCH 15/29] Protect pending Windows image dependencies --- lib/images/machine.go | 17 ++++++++++++++++- lib/images/machine_test.go | 36 ++++++++++++++++++++++++++++++++++++ lib/images/manager.go | 8 ++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/images/machine.go b/lib/images/machine.go index 74a511e99..66fccd361 100644 --- a/lib/images/machine.go +++ b/lib/images/machine.go @@ -284,13 +284,28 @@ func machineDiskPath(p *paths.Paths, repository, digestHex string, kind MachineI return filepath.Join(p.ImageDigestDir(repository, digestHex), name) } +func (m *manager) recordMachineDependency(ref *ResolvedRef, machine *MachineImage, buildID string) error { + m.createMu.Lock() + defer m.createMu.Unlock() + + meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) + if err != nil || meta.BuildID != buildID { + return errStaleBuild + } + meta.Machine = machine + if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { + return fmt.Errorf("record machine image dependency: %w", err) + } + return nil +} + func (m *manager) ensureNoMachineDependents(repository, digestHex string) error { metas, err := listAllMetadata(m.paths) if err != nil { return err } for _, meta := range metas { - if meta.Machine == nil || meta.Machine.Kind != MachineImageWindowsImage { + if meta.Status == StatusFailed || meta.Machine == nil || meta.Machine.Kind != MachineImageWindowsImage { continue } base, err := ParseNormalizedRef(meta.Machine.Base) diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go index ee2801084..c01495be0 100644 --- a/lib/images/machine_test.go +++ b/lib/images/machine_test.go @@ -152,6 +152,42 @@ func TestMaterializeWindowsBaseFormats(t *testing.T) { } } +func TestPendingWindowsImageBlocksBaseDeletion(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + baseDigest := strings.Repeat("a", 64) + imageDigest := strings.Repeat("b", 64) + baseName := "registry.example/windows/base@sha256:" + baseDigest + imageName := "registry.example/windows/image@sha256:" + imageDigest + + require.NoError(t, writeMetadata(p, "registry.example/windows/base", baseDigest, &imageMetadata{ + Name: baseName, + Digest: "sha256:" + baseDigest, + Platform: "windows/amd64", + Status: StatusReady, + Machine: &MachineImage{Kind: MachineImageWindowsBase}, + })) + require.NoError(t, os.WriteFile(machineDiskPath(p, "registry.example/windows/base", baseDigest, MachineImageWindowsBase), []byte("base"), 0444)) + const buildID = "pending-build" + require.NoError(t, writeMetadata(p, "registry.example/windows/image", imageDigest, &imageMetadata{ + Name: imageName, + Digest: "sha256:" + imageDigest, + Platform: "windows/amd64", + Status: StatusPending, + BuildID: buildID, + })) + ref, err := ParseNormalizedRef(imageName) + require.NoError(t, err) + require.NoError(t, m.recordMachineDependency(NewResolvedRef(ref, ref.Digest()), &MachineImage{ + Kind: MachineImageWindowsImage, + Base: baseName, + }, buildID)) + + err = m.DeleteImage(t.Context(), baseName) + assert.ErrorContains(t, err, "depends on it") + assert.DirExists(t, p.ImageDigestDir("registry.example/windows/base", baseDigest)) +} + func TestMaterializeWindowsBaseAndImage(t *testing.T) { requireQEMUImg(t) diff --git a/lib/images/manager.go b/lib/images/manager.go index 9d0845913..035198e9b 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -470,6 +470,14 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials m.updateStatusByDigest(ref, StatusFailed, err, buildID) return } + if machine != nil { + if err := m.recordMachineDependency(ref, machine, buildID); err != nil { + if !errors.Is(err, errStaleBuild) { + m.updateStatusByDigest(ref, StatusFailed, err, buildID) + } + return + } + } convertStart := time.Now() var diskSize int64 From 7c5df66051d15bf83293d879f86abb6d10f4a161 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:03:30 +0000 Subject: [PATCH 16/29] Add Windows guest control --- .github/workflows/test.yml | 3 +- .gitignore | 1 + Makefile | 6 + cmd/api/api/exec.go | 20 +- docs/windows-guest-agent.md | 20 ++ go.mod | 1 + go.sum | 8 + lib/guest/client.go | 2 + lib/guest/guest.pb.go | 162 +++++++++----- lib/guest/guest.proto | 6 + lib/guest/guest_grpc.pb.go | 2 +- ..._cache_key.go => socket_cache_key_unix.go} | 2 + lib/hypervisor/socket_cache_key_windows.go | 5 + lib/instances/create.go | 2 +- lib/instances/query.go | 14 +- lib/instances/query_test.go | 20 ++ ...dows_guest_agent_integration_linux_test.go | 132 ++++++++++++ lib/system/guest_agent/cp.go | 16 +- lib/system/guest_agent/exec.go | 120 +---------- lib/system/guest_agent/exec_session_linux.go | 23 ++ .../guest_agent/exec_session_windows.go | 63 ++++++ lib/system/guest_agent/exec_tty_linux.go | 83 +++++++ lib/system/guest_agent/exec_tty_windows.go | 98 +++++++++ lib/system/guest_agent/listener_linux.go | 17 ++ lib/system/guest_agent/listener_windows.go | 204 ++++++++++++++++++ lib/system/guest_agent/main.go | 25 ++- .../{network.go => network_linux.go} | 2 + lib/system/guest_agent/network_windows.go | 15 ++ lib/system/guest_agent/ownership_linux.go | 19 ++ lib/system/guest_agent/ownership_windows.go | 8 + lib/system/guest_agent/service_linux.go | 5 + lib/system/guest_agent/service_windows.go | 52 +++++ .../{shutdown.go => shutdown_linux.go} | 2 + lib/system/guest_agent/shutdown_windows.go | 20 ++ 34 files changed, 986 insertions(+), 192 deletions(-) create mode 100644 docs/windows-guest-agent.md rename lib/hypervisor/{socket_cache_key.go => socket_cache_key_unix.go} (95%) create mode 100644 lib/hypervisor/socket_cache_key_windows.go create mode 100644 lib/instances/windows_guest_agent_integration_linux_test.go create mode 100644 lib/system/guest_agent/exec_session_linux.go create mode 100644 lib/system/guest_agent/exec_session_windows.go create mode 100644 lib/system/guest_agent/exec_tty_linux.go create mode 100644 lib/system/guest_agent/exec_tty_windows.go create mode 100644 lib/system/guest_agent/listener_linux.go create mode 100644 lib/system/guest_agent/listener_windows.go rename lib/system/guest_agent/{network.go => network_linux.go} (99%) create mode 100644 lib/system/guest_agent/network_windows.go create mode 100644 lib/system/guest_agent/ownership_linux.go create mode 100644 lib/system/guest_agent/ownership_windows.go create mode 100644 lib/system/guest_agent/service_linux.go create mode 100644 lib/system/guest_agent/service_windows.go rename lib/system/guest_agent/{shutdown.go => shutdown_linux.go} (97%) create mode 100644 lib/system/guest_agent/shutdown_windows.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index caf47c339..0d7126785 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,6 +113,7 @@ jobs: done test -f "$HYPEMAN_WINDOWS_OVMF_CODE" test -f "$HYPEMAN_WINDOWS_OVMF_VARS" + test -r /ci/windows/image-agent.qcow2 - name: Test Windows hypervisor primitives run: | @@ -173,7 +174,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | for attempt in 1 2 3; do - if make build; then + if make build && make build-windows-guest-agent; then exit 0 fi if [ "$attempt" -lt 3 ]; then diff --git a/.gitignore b/.gitignore index 614d5996b..9f783dea6 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ cloud-hypervisor cloud-hypervisor/** lib/system/exec_agent/exec-agent lib/system/guest_agent/guest-agent +lib/system/guest_agent/hypeman-guest-agent.exe lib/system/init/init lib/hypervisor/vz/vz-shim/vz-shim diff --git a/Makefile b/Makefile index fae025efd..99b92a400 100644 --- a/Makefile +++ b/Makefile @@ -235,6 +235,12 @@ lib/system/guest_agent/guest-agent: lib/system/guest_agent/*.go @echo "Building guest-agent for Linux..." cd lib/system/guest_agent && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o guest-agent . +lib/system/guest_agent/hypeman-guest-agent.exe: lib/system/guest_agent/*.go + @echo "Building guest-agent for Windows..." + cd lib/system/guest_agent && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o hypeman-guest-agent.exe . + +build-windows-guest-agent: lib/system/guest_agent/hypeman-guest-agent.exe + # Build init binary (runs as PID 1 in guest VM) for embedding # Cross-compile for Linux since it runs inside the VM lib/system/init/init: lib/system/init/*.go diff --git a/cmd/api/api/exec.go b/cmd/api/api/exec.go index b6f93102c..969faf141 100644 --- a/cmd/api/api/exec.go +++ b/cmd/api/api/exec.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "strings" "sync" "time" @@ -40,6 +41,7 @@ type ExecRequest struct { WaitForAgent int32 `json:"wait_for_agent,omitempty"` // seconds to wait for guest agent to be ready Rows uint32 `json:"rows,omitempty"` // Initial terminal rows (0 = default) Cols uint32 `json:"cols,omitempty"` // Initial terminal cols (0 = default) + Session string `json:"session,omitempty"` // system (default) or desktop (Windows) } // ResizeMessage represents a window resize control message @@ -106,9 +108,22 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) { return } - // Default command if not specified + session := guest.ExecSession_EXEC_SESSION_SYSTEM + switch strings.ToLower(execReq.Session) { + case "", "system": + case "desktop": + session = guest.ExecSession_EXEC_SESSION_DESKTOP + default: + ws.WriteMessage(websocket.TextMessage, []byte(`{"error":"session must be system or desktop"}`)) + return + } + if len(execReq.Command) == 0 { - execReq.Command = []string{"/bin/sh"} + if strings.HasPrefix(inst.Platform, "windows/") { + execReq.Command = []string{"cmd.exe"} + } else { + execReq.Command = []string{"/bin/sh"} + } } // Get JWT subject for audit logging (if available) @@ -170,6 +185,7 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) { WaitForAgent: time.Duration(execReq.WaitForAgent) * time.Second, Rows: execReq.Rows, Cols: execReq.Cols, + Session: session, ResizeChan: resizeChan, }) diff --git a/docs/windows-guest-agent.md b/docs/windows-guest-agent.md new file mode 100644 index 000000000..c401ca404 --- /dev/null +++ b/docs/windows-guest-agent.md @@ -0,0 +1,20 @@ +# Windows guest agent + +Windows personas include `hypeman-guest-agent.exe` as the automatic `HypemanGuestAgent` LocalSystem service and the signed virtio-win VioSock driver. The agent listens on virtio-vsock port 2222 and implements the existing guest gRPC protocol. + +The Windows build supports: + +- command execution in the LocalSystem service session +- command execution in the active interactive desktop session +- ConPTY allocation and terminal resize events +- file copy, path stat, and graceful shutdown + +Exec requests use `session: "system"` by default. `session: "desktop"` obtains the token for the active Windows session and returns an error when no interactive user is logged in. + +Build the service with: + +```sh +make build-windows-guest-agent +``` + +The generated executable, Windows driver packages, credentials, and prepared persona disks are release inputs. They must not be committed to this repository. diff --git a/go.mod b/go.mod index cf5b2da13..18e8f8eba 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.4 require ( al.essio.dev/pkg/shellescape v1.6.0 github.com/Code-Hex/vz/v3 v3.7.1 + github.com/aymanbagabas/go-pty v0.2.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/creack/pty v1.1.24 github.com/cyphar/filepath-securejoin v0.6.1 diff --git a/go.sum b/go.sum index 679631b00..cb7cf9470 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymanbagabas/go-pty v0.2.2 h1:YZREB4eSj+1xdbbItIokX0ekjjeifgJOA+ZvxU4/WM8= +github.com/aymanbagabas/go-pty v0.2.2/go.mod h1:gfvlwH+0U66BCwxJREjJaAOEs9H1OFf3YFjI9WSiZ04= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -125,6 +127,8 @@ github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGh github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hugelgupf/vmtest v0.0.0-20240307030256-5d9f3d34a58d h1:nP8SfQJqruIVSWYJTuYc37jLHEY1Z0fF+zKSrs3K/C8= +github.com/hugelgupf/vmtest v0.0.0-20240307030256-5d9f3d34a58d/go.mod h1:B63hDJMhTupLWCHwopAyEo7wRFowx9kOc8m8j1sfOqE= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= @@ -264,6 +268,8 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/u-root/gobusybox/src v0.0.0-20250101170133-2e884e4509c7 h1:dtiVT4SeBUc/vHtwI2HjDZN+FCKTstQBxugIxJEGo9g= +github.com/u-root/gobusybox/src v0.0.0-20250101170133-2e884e4509c7/go.mod h1:PW3wGFCHjdHxAhra5FKvcARbCGqGfentYuPKmuhv8DY= github.com/u-root/u-root v0.15.0 h1:8JXfjAA/Vs8EXfZUA2ftvoHbiYYLdaU8umJ461aq+Jw= github.com/u-root/u-root v0.15.0/go.mod h1:/0Qr7qJeDwWxoKku2xKQ4Szc+SwBE3g9VE8jNiamsmc= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM= @@ -356,6 +362,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/lib/guest/client.go b/lib/guest/client.go index b6dcd2e8e..c5d95f8fe 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -147,6 +147,7 @@ type ExecOptions struct { WaitForAgent time.Duration // Max time to wait for agent to be ready (0 = no wait, fail immediately) Rows uint32 // Initial terminal rows (0 = default 24) Cols uint32 // Initial terminal cols (0 = default 80) + Session ExecSession // SYSTEM service session or active Windows desktop session ResizeChan <-chan *WindowSize // Optional: channel to receive resize events (pointer to avoid copying mutex) } @@ -477,6 +478,7 @@ func execIntoInstanceOnce(ctx context.Context, dialer hypervisor.VsockDialer, op TimeoutSeconds: opts.Timeout, Rows: opts.Rows, Cols: opts.Cols, + Session: opts.Session, }, }, }); err != nil { diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index a239fc970..4554f413d 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.10 -// protoc v6.33.4 +// protoc v3.21.12 // source: lib/guest/guest.proto package guest @@ -21,6 +21,52 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ExecSession int32 + +const ( + ExecSession_EXEC_SESSION_SYSTEM ExecSession = 0 + ExecSession_EXEC_SESSION_DESKTOP ExecSession = 1 +) + +// Enum value maps for ExecSession. +var ( + ExecSession_name = map[int32]string{ + 0: "EXEC_SESSION_SYSTEM", + 1: "EXEC_SESSION_DESKTOP", + } + ExecSession_value = map[string]int32{ + "EXEC_SESSION_SYSTEM": 0, + "EXEC_SESSION_DESKTOP": 1, + } +) + +func (x ExecSession) Enum() *ExecSession { + p := new(ExecSession) + *p = x + return p +} + +func (x ExecSession) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ExecSession) Descriptor() protoreflect.EnumDescriptor { + return file_lib_guest_guest_proto_enumTypes[0].Descriptor() +} + +func (ExecSession) Type() protoreflect.EnumType { + return &file_lib_guest_guest_proto_enumTypes[0] +} + +func (x ExecSession) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ExecSession.Descriptor instead. +func (ExecSession) EnumDescriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{0} +} + // ExecRequest represents messages from client to server type ExecRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -130,6 +176,7 @@ type ExecStart struct { TimeoutSeconds int32 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` // Execution timeout in seconds (0 = no timeout) Rows uint32 `protobuf:"varint,6,opt,name=rows,proto3" json:"rows,omitempty"` // Initial terminal rows (0 = default 24) Cols uint32 `protobuf:"varint,7,opt,name=cols,proto3" json:"cols,omitempty"` // Initial terminal cols (0 = default 80) + Session ExecSession `protobuf:"varint,8,opt,name=session,proto3,enum=guest.ExecSession" json:"session,omitempty"` // SYSTEM service session or active desktop user unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -213,6 +260,13 @@ func (x *ExecStart) GetCols() uint32 { return 0 } +func (x *ExecStart) GetSession() ExecSession { + if x != nil { + return x.Session + } + return ExecSession_EXEC_SESSION_SYSTEM +} + // WindowSize represents terminal window dimensions for resize events type WindowSize struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1389,7 +1443,7 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x05start\x18\x01 \x01(\v2\x10.guest.ExecStartH\x00R\x05start\x12\x16\n" + "\x05stdin\x18\x02 \x01(\fH\x00R\x05stdin\x12+\n" + "\x06resize\x18\x03 \x01(\v2\x11.guest.WindowSizeH\x00R\x06resizeB\t\n" + - "\arequest\"\xff\x01\n" + + "\arequest\"\xad\x02\n" + "\tExecStart\x12\x18\n" + "\acommand\x18\x01 \x03(\tR\acommand\x12\x10\n" + "\x03tty\x18\x02 \x01(\bR\x03tty\x12+\n" + @@ -1397,7 +1451,8 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x03cwd\x18\x04 \x01(\tR\x03cwd\x12'\n" + "\x0ftimeout_seconds\x18\x05 \x01(\x05R\x0etimeoutSeconds\x12\x12\n" + "\x04rows\x18\x06 \x01(\rR\x04rows\x12\x12\n" + - "\x04cols\x18\a \x01(\rR\x04cols\x1a6\n" + + "\x04cols\x18\a \x01(\rR\x04cols\x12,\n" + + "\asession\x18\b \x01(\x0e2\x12.guest.ExecSessionR\asession\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"4\n" + @@ -1479,7 +1534,10 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x04ipv4\x18\x03 \x01(\tR\x04ipv4\x12\x16\n" + "\x06prefix\x18\x04 \x01(\rR\x06prefix\x12\x18\n" + "\agateway\x18\x05 \x01(\tR\agateway\"\x1c\n" + - "\x1aReconfigureNetworkResponse2\xae\x03\n" + + "\x1aReconfigureNetworkResponse*@\n" + + "\vExecSession\x12\x17\n" + + "\x13EXEC_SESSION_SYSTEM\x10\x00\x12\x18\n" + + "\x14EXEC_SESSION_DESKTOP\x10\x012\xae\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + @@ -1500,55 +1558,58 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } +var file_lib_guest_guest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_lib_guest_guest_proto_goTypes = []any{ - (*ExecRequest)(nil), // 0: guest.ExecRequest - (*ExecStart)(nil), // 1: guest.ExecStart - (*WindowSize)(nil), // 2: guest.WindowSize - (*ExecResponse)(nil), // 3: guest.ExecResponse - (*CopyToGuestRequest)(nil), // 4: guest.CopyToGuestRequest - (*CopyToGuestStart)(nil), // 5: guest.CopyToGuestStart - (*CopyToGuestEnd)(nil), // 6: guest.CopyToGuestEnd - (*CopyToGuestResponse)(nil), // 7: guest.CopyToGuestResponse - (*CopyFromGuestRequest)(nil), // 8: guest.CopyFromGuestRequest - (*CopyFromGuestResponse)(nil), // 9: guest.CopyFromGuestResponse - (*CopyFromGuestHeader)(nil), // 10: guest.CopyFromGuestHeader - (*CopyFromGuestEnd)(nil), // 11: guest.CopyFromGuestEnd - (*CopyFromGuestError)(nil), // 12: guest.CopyFromGuestError - (*StatPathRequest)(nil), // 13: guest.StatPathRequest - (*StatPathResponse)(nil), // 14: guest.StatPathResponse - (*ShutdownRequest)(nil), // 15: guest.ShutdownRequest - (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse - (*ReconfigureNetworkRequest)(nil), // 17: guest.ReconfigureNetworkRequest - (*ReconfigureNetworkResponse)(nil), // 18: guest.ReconfigureNetworkResponse - nil, // 19: guest.ExecStart.EnvEntry + (ExecSession)(0), // 0: guest.ExecSession + (*ExecRequest)(nil), // 1: guest.ExecRequest + (*ExecStart)(nil), // 2: guest.ExecStart + (*WindowSize)(nil), // 3: guest.WindowSize + (*ExecResponse)(nil), // 4: guest.ExecResponse + (*CopyToGuestRequest)(nil), // 5: guest.CopyToGuestRequest + (*CopyToGuestStart)(nil), // 6: guest.CopyToGuestStart + (*CopyToGuestEnd)(nil), // 7: guest.CopyToGuestEnd + (*CopyToGuestResponse)(nil), // 8: guest.CopyToGuestResponse + (*CopyFromGuestRequest)(nil), // 9: guest.CopyFromGuestRequest + (*CopyFromGuestResponse)(nil), // 10: guest.CopyFromGuestResponse + (*CopyFromGuestHeader)(nil), // 11: guest.CopyFromGuestHeader + (*CopyFromGuestEnd)(nil), // 12: guest.CopyFromGuestEnd + (*CopyFromGuestError)(nil), // 13: guest.CopyFromGuestError + (*StatPathRequest)(nil), // 14: guest.StatPathRequest + (*StatPathResponse)(nil), // 15: guest.StatPathResponse + (*ShutdownRequest)(nil), // 16: guest.ShutdownRequest + (*ShutdownResponse)(nil), // 17: guest.ShutdownResponse + (*ReconfigureNetworkRequest)(nil), // 18: guest.ReconfigureNetworkRequest + (*ReconfigureNetworkResponse)(nil), // 19: guest.ReconfigureNetworkResponse + nil, // 20: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ - 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart - 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 19, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry - 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart - 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd - 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader - 11, // 6: guest.CopyFromGuestResponse.end:type_name -> guest.CopyFromGuestEnd - 12, // 7: guest.CopyFromGuestResponse.error:type_name -> guest.CopyFromGuestError - 0, // 8: guest.GuestService.Exec:input_type -> guest.ExecRequest - 4, // 9: guest.GuestService.CopyToGuest:input_type -> guest.CopyToGuestRequest - 8, // 10: guest.GuestService.CopyFromGuest:input_type -> guest.CopyFromGuestRequest - 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest - 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest - 17, // 13: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest - 3, // 14: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 15: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 16: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 17: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 16, // 18: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 18, // 19: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse - 14, // [14:20] is the sub-list for method output_type - 8, // [8:14] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 2, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart + 3, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize + 20, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 0, // 3: guest.ExecStart.session:type_name -> guest.ExecSession + 6, // 4: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart + 7, // 5: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd + 11, // 6: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader + 12, // 7: guest.CopyFromGuestResponse.end:type_name -> guest.CopyFromGuestEnd + 13, // 8: guest.CopyFromGuestResponse.error:type_name -> guest.CopyFromGuestError + 1, // 9: guest.GuestService.Exec:input_type -> guest.ExecRequest + 5, // 10: guest.GuestService.CopyToGuest:input_type -> guest.CopyToGuestRequest + 9, // 11: guest.GuestService.CopyFromGuest:input_type -> guest.CopyFromGuestRequest + 14, // 12: guest.GuestService.StatPath:input_type -> guest.StatPathRequest + 16, // 13: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest + 18, // 14: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest + 4, // 15: guest.GuestService.Exec:output_type -> guest.ExecResponse + 8, // 16: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 10, // 17: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 15, // 18: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 17, // 19: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 19, // 20: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 15, // [15:21] is the sub-list for method output_type + 9, // [9:15] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_lib_guest_guest_proto_init() } @@ -1582,13 +1643,14 @@ func file_lib_guest_guest_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), - NumEnums: 0, + NumEnums: 1, NumMessages: 20, NumExtensions: 0, NumServices: 1, }, GoTypes: file_lib_guest_guest_proto_goTypes, DependencyIndexes: file_lib_guest_guest_proto_depIdxs, + EnumInfos: file_lib_guest_guest_proto_enumTypes, MessageInfos: file_lib_guest_guest_proto_msgTypes, }.Build() File_lib_guest_guest_proto = out.File diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 317c21b3e..41b55771e 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -34,6 +34,11 @@ message ExecRequest { } } +enum ExecSession { + EXEC_SESSION_SYSTEM = 0; + EXEC_SESSION_DESKTOP = 1; +} + // ExecStart initiates command execution message ExecStart { repeated string command = 1; // Command and arguments @@ -43,6 +48,7 @@ message ExecStart { int32 timeout_seconds = 5; // Execution timeout in seconds (0 = no timeout) uint32 rows = 6; // Initial terminal rows (0 = default 24) uint32 cols = 7; // Initial terminal cols (0 = default 80) + ExecSession session = 8; // SYSTEM service session or active desktop user } // WindowSize represents terminal window dimensions for resize events diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index f93631d93..acad4a3d1 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.0 -// - protoc v6.33.4 +// - protoc v3.21.12 // source: lib/guest/guest.proto package guest diff --git a/lib/hypervisor/socket_cache_key.go b/lib/hypervisor/socket_cache_key_unix.go similarity index 95% rename from lib/hypervisor/socket_cache_key.go rename to lib/hypervisor/socket_cache_key_unix.go index efd31cd0d..723f65c0a 100644 --- a/lib/hypervisor/socket_cache_key.go +++ b/lib/hypervisor/socket_cache_key_unix.go @@ -1,3 +1,5 @@ +//go:build !windows + package hypervisor import ( diff --git a/lib/hypervisor/socket_cache_key_windows.go b/lib/hypervisor/socket_cache_key_windows.go new file mode 100644 index 000000000..d7c171cb3 --- /dev/null +++ b/lib/hypervisor/socket_cache_key_windows.go @@ -0,0 +1,5 @@ +//go:build windows + +package hypervisor + +func SocketCacheKey(socketPath string) string { return socketPath } diff --git a/lib/instances/create.go b/lib/instances/create.go index d4ff30db4..3d31f1a08 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -385,7 +385,7 @@ func (m *manager) createInstance( Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, - SkipGuestAgent: req.SkipGuestAgent || windows, + SkipGuestAgent: req.SkipGuestAgent, EnableRosetta: enableRosetta, SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), diff --git a/lib/instances/query.go b/lib/instances/query.go index 98c5359e0..30ad4d6d8 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -316,7 +316,13 @@ func (m *manager) hydrateBootMarkersFromLogs(ctx context.Context, stored *Stored stored.GuestAgentReadyAt = guestAgentReadyAt hydrated = true } - if needAgent && stored.GuestAgentReadyAt == nil && stored.ProgramStartedAt != nil && m.hydrateGuestAgentReadyFromProbe(ctx, stored) { + if isWindowsPlatform(stored.Platform) && (needProgram || needAgent) && m.hydrateGuestAgentReadyFromProbe(ctx, stored) { + if stored.ProgramStartedAt == nil { + startedAt := *stored.GuestAgentReadyAt + stored.ProgramStartedAt = &startedAt + } + hydrated = true + } else if needAgent && stored.GuestAgentReadyAt == nil && stored.ProgramStartedAt != nil && m.hydrateGuestAgentReadyFromProbe(ctx, stored) { hydrated = true } if hydrated { @@ -446,8 +452,12 @@ func probeGuestAgentReady(ctx context.Context, stored *StoredMetadata) bool { probeCtx, cancel := context.WithTimeout(ctx, guestAgentReadyProbeTimeout) defer cancel() + command := []string{"/bin/true"} + if isWindowsPlatform(stored.Platform) { + command = []string{"cmd.exe", "/d", "/c", "exit", "0"} + } exit, err := guest.ExecIntoInstance(probeCtx, dialer, guest.ExecOptions{ - Command: []string{"/bin/true"}, + Command: command, Timeout: int32(guestAgentReadyProbeTimeout / time.Second), WaitForAgent: guestAgentReadyProbeWait, }) diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb464..6bab797f8 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -438,6 +438,26 @@ func TestHydrateBootMarkersUsesGuestAgentProbeWhenReadyMarkerMissing(t *testing. assert.Equal(t, 1, probeCalls) } +func TestHydrateBootMarkersUsesWindowsGuestAgentAsBootMarker(t *testing.T) { + t.Parallel() + + readyAt := time.Date(2026, 3, 8, 12, 0, 2, 0, time.UTC) + m := &manager{ + paths: paths.New(t.TempDir()), + now: func() time.Time { return readyAt }, + guestAgentReadyProbe: func(context.Context, *StoredMetadata) bool { + return true + }, + } + meta := &StoredMetadata{Id: "windows-instance", Platform: "windows/amd64"} + meta.Phases.Record(phasetracking.PhaseInitializing, readyAt.Add(-time.Second)) + + require.True(t, m.hydrateBootMarkersFromLogs(context.Background(), meta)) + require.Equal(t, readyAt, *meta.ProgramStartedAt) + require.Equal(t, readyAt, *meta.GuestAgentReadyAt) + assert.Equal(t, StateRunning, deriveRunningState(meta)) +} + func TestParseBootMarkers_IgnoresStaleMarkersBeforeBootStart(t *testing.T) { t.Parallel() diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go new file mode 100644 index 000000000..065d61fd5 --- /dev/null +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -0,0 +1,132 @@ +//go:build linux && amd64 + +package instances + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWindowsGuestAgentIntegration(t *testing.T) { + fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_PERSONA") + if fixture == "" { + fixture = "/ci/windows/persona-agent.qcow2" + } + if _, err := os.Stat(fixture); err != nil { + if os.Getenv("CI") == "true" { + t.Fatalf("required Windows guest-agent fixture is missing: %s", fixture) + } + t.Skipf("Windows guest-agent fixture is unavailable: %s", fixture) + } + acquireHeavyIO(t) + + manager, dataDir := setupTestManagerForQEMU(t) + p := paths.New(dataDir) + const digestHex = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + image := &images.Image{ + Name: "registry.example/windows/persona:guest-agent-integration", + Digest: "sha256:" + digestHex, + Platform: "windows/amd64", + Status: images.StatusReady, + Machine: &images.MachineImage{ + Kind: images.MachineImageWindowsPersona, + Base: "registry.example/windows/base@sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + TPM: "2.0", + SecureBoot: "required", + VirtualSize: 80 << 30, + }, + } + manager.imageManager = windowsFixtureImageManager{image: image} + personaPath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) + require.NoError(t, err) + require.NoError(t, forkvm.CopyRegularFile(fixture, personaPath)) + require.NoError(t, os.Chmod(personaPath, 0444)) + + ctx := context.Background() + instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "windows-guest-agent-integration", + Image: image.Name, + Platform: "windows/amd64", + Size: 8 << 30, + Vcpus: 4, + Hypervisor: hypervisor.TypeQEMU, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) + + require.Eventually(t, func() bool { + current, err := manager.GetInstance(ctx, instance.Id) + return err == nil && current.State == StateRunning + }, 4*time.Minute, time.Second, "Windows guest agent did not become ready") + + dialer, err := manager.GetVsockDialer(ctx, instance.Id) + require.NoError(t, err) + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "[Console]::Out.Write('HYPEMAN_SYSTEM_OK')"}, + Stdout: &stdout, + Stderr: &stderr, + Timeout: 30, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code) + assert.Equal(t, "HYPEMAN_SYSTEM_OK", stdout.String()) + + stdout.Reset() + stderr.Reset() + resizes := make(chan *guest.WindowSize, 1) + resizes <- &guest.WindowSize{Rows: 37, Cols: 101} + close(resizes) + exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"cmd.exe", "/d", "/c", "ping -n 2 127.0.0.1 >nul & echo HYPEMAN_CONPTY_OK"}, + Stdout: &stdout, + Stderr: &stderr, + TTY: true, + Rows: 31, + Cols: 97, + ResizeChan: resizes, + Timeout: 30, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code) + assert.Contains(t, stdout.String(), "HYPEMAN_CONPTY_OK") + + stdout.Reset() + exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"cmd.exe", "/d", "/c", "echo", "HYPEMAN_DESKTOP_OK"}, + Stdout: &stdout, + Session: guest.ExecSession_EXEC_SESSION_DESKTOP, + Timeout: 30, + }) + require.NoError(t, err) + require.Equal(t, 0, exit.Code) + assert.Contains(t, stdout.String(), "HYPEMAN_DESKTOP_OK") + + source := filepath.Join(t.TempDir(), "roundtrip.txt") + require.NoError(t, os.WriteFile(source, []byte("HYPEMAN_COPY_OK"), 0644)) + require.NoError(t, guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{ + SrcPath: source, + DstPath: `C:\ProgramData\Hypeman\roundtrip.txt`, + })) + destination := t.TempDir() + require.NoError(t, guest.CopyFromInstance(ctx, dialer, guest.CopyFromInstanceOptions{ + SrcPath: `C:\ProgramData\Hypeman\roundtrip.txt`, + DstPath: destination, + })) + contents, err := os.ReadFile(filepath.Join(destination, "roundtrip.txt")) + require.NoError(t, err) + assert.Equal(t, "HYPEMAN_COPY_OK", string(contents)) +} diff --git a/lib/system/guest_agent/cp.go b/lib/system/guest_agent/cp.go index c2ad5b020..c4d412107 100644 --- a/lib/system/guest_agent/cp.go +++ b/lib/system/guest_agent/cp.go @@ -7,7 +7,6 @@ import ( "log" "os" "path/filepath" - "syscall" "time" pb "github.com/kernel/hypeman/lib/guest" @@ -140,7 +139,7 @@ func (s *guestServer) CopyToGuest(stream pb.GuestService_CopyToGuestServer) erro // Only chown when both UID and GID are explicitly set (non-zero) // to avoid accidentally setting one to root (0) when only the other is specified if start.Uid > 0 && start.Gid > 0 { - if err := os.Chown(start.Path, int(start.Uid), int(start.Gid)); err != nil { + if err := setFileOwnership(start.Path, start.Uid, start.Gid); err != nil { log.Printf("[guest-agent] warning: failed to set ownership on %s: %v", start.Path, err) } } @@ -210,11 +209,7 @@ func (s *guestServer) copyFromGuestFile(fullPath, relativePath string, info os.F } // Extract UID/GID from file info - var uid, gid uint32 - if stat, ok := info.Sys().(*syscall.Stat_t); ok { - uid = stat.Uid - gid = stat.Gid - } + uid, gid := fileOwnership(info) // Send header header := &pb.CopyFromGuestHeader{ @@ -361,12 +356,7 @@ func (s *guestServer) copyFromGuestDir(rootPath string, followLinks bool, stream isFinal := i == len(entries)-1 if e.info.IsDir() { - // Extract UID/GID from file info - var uid, gid uint32 - if stat, ok := e.info.Sys().(*syscall.Stat_t); ok { - uid = stat.Uid - gid = stat.Gid - } + uid, gid := fileOwnership(e.info) // Send directory header if err := stream.Send(&pb.CopyFromGuestResponse{ diff --git a/lib/system/guest_agent/exec.go b/lib/system/guest_agent/exec.go index 409754c65..ba3a4a04f 100644 --- a/lib/system/guest_agent/exec.go +++ b/lib/system/guest_agent/exec.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "github.com/creack/pty" pb "github.com/kernel/hypeman/lib/guest" ) @@ -30,13 +29,12 @@ func (s *guestServer) Exec(stream pb.GuestService_ExecServer) error { return fmt.Errorf("first message must be ExecStart") } - command := start.Command - if len(command) == 0 { - command = []string{"/bin/sh"} + if len(start.Command) == 0 { + start.Command = defaultCommand() } log.Printf("[guest-agent] exec: command=%v tty=%v cwd=%s timeout=%d", - command, start.Tty, start.Cwd, start.TimeoutSeconds) + start.Command, start.Tty, start.Cwd, start.TimeoutSeconds) // Create context with timeout if specified ctx := context.Background() @@ -60,6 +58,11 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E } cmd := exec.CommandContext(ctx, start.Command[0], start.Command[1:]...) + cleanup, err := configureExecCommand(cmd, start.Session) + if err != nil { + return err + } + defer cleanup() // Set up environment (no TTY defaults for non-TTY mode) cmd.Env = s.buildEnv(start.Env, false) @@ -160,113 +163,6 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E }) } -// executeTTY executes command with TTY -func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { - // Run command directly with PTY - guest-agent is already running in container namespace - // This ensures PTY and shell are in the same namespace, fixing Ctrl+C signal handling - if len(start.Command) == 0 { - return fmt.Errorf("empty command") - } - - cmd := exec.CommandContext(ctx, start.Command[0], start.Command[1:]...) - - // Set up environment (TTY mode adds TERM default) - cmd.Env = s.buildEnv(start.Env, true) - - // Set up working directory - if start.Cwd != "" { - cmd.Dir = start.Cwd - } - - // Set up initial window size (use defaults if not specified) - ws := &pty.Winsize{ - Rows: uint16(start.Rows), - Cols: uint16(start.Cols), - } - if ws.Rows == 0 { - ws.Rows = 24 - } - if ws.Cols == 0 { - ws.Cols = 80 - } - - // Start with PTY and initial window size - ptmx, err := pty.StartWithSize(cmd, ws) - if err != nil { - return fmt.Errorf("start pty: %w", err) - } - defer ptmx.Close() - - // Mutex to protect concurrent stream.Send calls (gRPC streams are not thread-safe) - var sendMu sync.Mutex - - // Use WaitGroup to ensure all output is sent before exit code - var wg sync.WaitGroup - - // Handle stdin and resize in background - go func() { - for { - req, err := stream.Recv() - if err != nil { - return - } - - if data := req.GetStdin(); data != nil { - ptmx.Write(data) - } - - // Handle window resize - if resize := req.GetResize(); resize != nil { - pty.Setsize(ptmx, &pty.Winsize{ - Rows: uint16(resize.Rows), - Cols: uint16(resize.Cols), - }) - } - } - }() - - // Stream output - wg.Add(1) - go func() { - defer wg.Done() - buf := make([]byte, 32*1024) - for { - n, err := ptmx.Read(buf) - if n > 0 { - sendMu.Lock() - stream.Send(&pb.ExecResponse{ - Response: &pb.ExecResponse_Stdout{Stdout: buf[:n]}, - }) - sendMu.Unlock() - } - if err != nil { - return - } - } - }() - - // Wait for command or context cancellation - waitErr := cmd.Wait() - - // Wait for all output to be sent - wg.Wait() - - exitCode := int32(0) - if cmd.ProcessState != nil { - exitCode = int32(cmd.ProcessState.ExitCode()) - } else if waitErr != nil { - // If killed by timeout, exit with 124 (GNU timeout convention) - exitCode = 124 - } - - log.Printf("[guest-agent] TTY command finished with exit code: %d", exitCode) - - // Send exit code - return stream.Send(&pb.ExecResponse{ - Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}, - }) -} - // buildEnv constructs environment variables by merging provided env with defaults. // When tty is true, adds sensible defaults for interactive terminal sessions. // User-provided env vars override both base environment and defaults. diff --git a/lib/system/guest_agent/exec_session_linux.go b/lib/system/guest_agent/exec_session_linux.go new file mode 100644 index 000000000..8360434a4 --- /dev/null +++ b/lib/system/guest_agent/exec_session_linux.go @@ -0,0 +1,23 @@ +//go:build linux + +package main + +import ( + "fmt" + "os/exec" + + pb "github.com/kernel/hypeman/lib/guest" +) + +func defaultCommand() []string { return []string{"/bin/sh"} } + +func configureExecCommand(_ *exec.Cmd, session pb.ExecSession) (func(), error) { + switch session { + case pb.ExecSession_EXEC_SESSION_SYSTEM: + return func() {}, nil + case pb.ExecSession_EXEC_SESSION_DESKTOP: + return nil, fmt.Errorf("desktop execution sessions are only supported on Windows") + default: + return nil, fmt.Errorf("unknown execution session %d", session) + } +} diff --git a/lib/system/guest_agent/exec_session_windows.go b/lib/system/guest_agent/exec_session_windows.go new file mode 100644 index 000000000..a1411ea46 --- /dev/null +++ b/lib/system/guest_agent/exec_session_windows.go @@ -0,0 +1,63 @@ +//go:build windows + +package main + +import ( + "fmt" + "os/exec" + "syscall" + "unsafe" + + pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/windows" +) + +func defaultCommand() []string { return []string{"cmd.exe"} } + +func activeDesktopToken() (windows.Token, error) { + var sessions *windows.WTS_SESSION_INFO + var count uint32 + if err := windows.WTSEnumerateSessions(0, 0, 1, &sessions, &count); err != nil { + return 0, fmt.Errorf("enumerate desktop sessions: %w", err) + } + if sessions != nil { + defer windows.WTSFreeMemory(uintptr(unsafe.Pointer(sessions))) + } + + for _, session := range unsafe.Slice(sessions, count) { + if session.State != windows.WTSActive { + continue + } + var token windows.Token + if err := windows.WTSQueryUserToken(session.SessionID, &token); err == nil { + return token, nil + } + } + return 0, fmt.Errorf("no active desktop user session") +} + +func executionToken(session pb.ExecSession) (syscall.Token, func(), error) { + switch session { + case pb.ExecSession_EXEC_SESSION_SYSTEM: + return 0, func() {}, nil + case pb.ExecSession_EXEC_SESSION_DESKTOP: + default: + return 0, nil, fmt.Errorf("unknown execution session %d", session) + } + token, err := activeDesktopToken() + if err != nil { + return 0, nil, err + } + return syscall.Token(token), func() { _ = token.Close() }, nil +} + +func configureExecCommand(cmd *exec.Cmd, session pb.ExecSession) (func(), error) { + token, cleanup, err := executionToken(session) + if err != nil { + return nil, err + } + if token != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{Token: token} + } + return cleanup, nil +} diff --git a/lib/system/guest_agent/exec_tty_linux.go b/lib/system/guest_agent/exec_tty_linux.go new file mode 100644 index 000000000..672737c2c --- /dev/null +++ b/lib/system/guest_agent/exec_tty_linux.go @@ -0,0 +1,83 @@ +//go:build linux + +package main + +import ( + "context" + "fmt" + "log" + "os/exec" + "sync" + + "github.com/creack/pty" + pb "github.com/kernel/hypeman/lib/guest" +) + +func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { + if len(start.Command) == 0 { + return fmt.Errorf("empty command") + } + + cmd := exec.CommandContext(ctx, start.Command[0], start.Command[1:]...) + cmd.Env = s.buildEnv(start.Env, true) + cmd.Dir = start.Cwd + + ws := &pty.Winsize{Rows: uint16(start.Rows), Cols: uint16(start.Cols)} + if ws.Rows == 0 { + ws.Rows = 24 + } + if ws.Cols == 0 { + ws.Cols = 80 + } + + ptmx, err := pty.StartWithSize(cmd, ws) + if err != nil { + return fmt.Errorf("start pty: %w", err) + } + defer ptmx.Close() + + var sendMu sync.Mutex + var wg sync.WaitGroup + go func() { + for { + req, err := stream.Recv() + if err != nil { + return + } + if data := req.GetStdin(); data != nil { + _, _ = ptmx.Write(data) + } + if resize := req.GetResize(); resize != nil { + _ = pty.Setsize(ptmx, &pty.Winsize{Rows: uint16(resize.Rows), Cols: uint16(resize.Cols)}) + } + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, 32*1024) + for { + n, err := ptmx.Read(buf) + if n > 0 { + sendMu.Lock() + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stdout{Stdout: buf[:n]}}) + sendMu.Unlock() + } + if err != nil { + return + } + } + }() + + waitErr := cmd.Wait() + wg.Wait() + exitCode := int32(0) + if cmd.ProcessState != nil { + exitCode = int32(cmd.ProcessState.ExitCode()) + } else if waitErr != nil { + exitCode = 124 + } + log.Printf("[guest-agent] TTY command finished with exit code: %d", exitCode) + return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) +} diff --git a/lib/system/guest_agent/exec_tty_windows.go b/lib/system/guest_agent/exec_tty_windows.go new file mode 100644 index 000000000..8c96598de --- /dev/null +++ b/lib/system/guest_agent/exec_tty_windows.go @@ -0,0 +1,98 @@ +//go:build windows + +package main + +import ( + "context" + "fmt" + "log" + "sync" + "syscall" + + pty "github.com/aymanbagabas/go-pty" + pb "github.com/kernel/hypeman/lib/guest" +) + +func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { + if len(start.Command) == 0 { + return fmt.Errorf("empty command") + } + + console, err := pty.New() + if err != nil { + return fmt.Errorf("create ConPTY: %w", err) + } + defer console.Close() + + cols, rows := int(start.Cols), int(start.Rows) + if cols == 0 { + cols = 80 + } + if rows == 0 { + rows = 24 + } + if err := console.Resize(cols, rows); err != nil { + return fmt.Errorf("resize ConPTY: %w", err) + } + + cmd := console.CommandContext(ctx, start.Command[0], start.Command[1:]...) + cmd.Env = s.buildEnv(start.Env, true) + cmd.Dir = start.Cwd + token, cleanup, err := executionToken(start.Session) + if err != nil { + return err + } + defer cleanup() + if token != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{Token: token} + } + if err := cmd.Start(); err != nil { + return fmt.Errorf("start ConPTY command: %w", err) + } + + var sendMu sync.Mutex + var wg sync.WaitGroup + go func() { + for { + req, err := stream.Recv() + if err != nil { + return + } + if data := req.GetStdin(); data != nil { + _, _ = console.Write(data) + } + if resize := req.GetResize(); resize != nil { + _ = console.Resize(int(resize.Cols), int(resize.Rows)) + } + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, 32*1024) + for { + n, err := console.Read(buf) + if n > 0 { + sendMu.Lock() + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stdout{Stdout: buf[:n]}}) + sendMu.Unlock() + } + if err != nil { + return + } + } + }() + + waitErr := cmd.Wait() + _ = console.Close() + wg.Wait() + exitCode := int32(0) + if cmd.ProcessState != nil { + exitCode = int32(cmd.ProcessState.ExitCode()) + } else if waitErr != nil { + exitCode = 124 + } + log.Printf("[guest-agent] ConPTY command finished with exit code: %d", exitCode) + return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) +} diff --git a/lib/system/guest_agent/listener_linux.go b/lib/system/guest_agent/listener_linux.go new file mode 100644 index 000000000..a62f6745b --- /dev/null +++ b/lib/system/guest_agent/listener_linux.go @@ -0,0 +1,17 @@ +//go:build linux + +package main + +import ( + "net" + + "github.com/mdlayher/vsock" +) + +func listenVSock(port uint32) (net.Listener, error) { + return vsock.Listen(port, nil) +} + +func defaultReadyFilePath() string { + return "/run/hypeman/guest-agent-ready" +} diff --git a/lib/system/guest_agent/listener_windows.go b/lib/system/guest_agent/listener_windows.go new file mode 100644 index 000000000..05b8c427e --- /dev/null +++ b/lib/system/guest_agent/listener_windows.go @@ -0,0 +1,204 @@ +//go:build windows + +package main + +import ( + "fmt" + "io" + "net" + "runtime" + "sync" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + ioctlGetViosockAF = 0x0801300c + vmAddrCIDAny = ^uint32(0) + sockStream = 1 + socketError = ^uintptr(0) +) + +type rawSockaddrVM struct { + Family uint16 + Reserved1 uint16 + Port uint32 + CID uint32 + Zero [4]byte +} + +type vsockAddr struct { + cid uint32 + port uint32 +} + +func (a vsockAddr) Network() string { return "vsock" } +func (a vsockAddr) String() string { return fmt.Sprintf("%d:%d", a.cid, a.port) } + +var ( + winsockOnce sync.Once + winsockErr error + ws2DLL = windows.NewLazySystemDLL("ws2_32.dll") + bindProc = ws2DLL.NewProc("bind") + acceptProc = ws2DLL.NewProc("accept") +) + +func initializeWinsock() error { + winsockOnce.Do(func() { + var data windows.WSAData + winsockErr = windows.WSAStartup(0x202, &data) + }) + return winsockErr +} + +func viosockAddressFamily() (int32, error) { + devicePath, err := windows.UTF16PtrFromString(`\\.\Viosock`) + if err != nil { + return 0, err + } + device, err := windows.CreateFile(devicePath, windows.GENERIC_READ, windows.FILE_SHARE_READ, nil, windows.OPEN_EXISTING, 0, 0) + if err != nil { + return 0, fmt.Errorf("open Viosock device: %w", err) + } + defer windows.CloseHandle(device) + + var family uint32 + var returned uint32 + if err := windows.DeviceIoControl( + device, + ioctlGetViosockAF, + nil, + 0, + (*byte)(unsafe.Pointer(&family)), + uint32(unsafe.Sizeof(family)), + &returned, + nil, + ); err != nil { + return 0, fmt.Errorf("query Viosock address family: %w", err) + } + if returned != uint32(unsafe.Sizeof(family)) || family == 0 { + return 0, fmt.Errorf("Viosock returned invalid address family %d", family) + } + return int32(family), nil +} + +func listenVSock(port uint32) (net.Listener, error) { + if err := initializeWinsock(); err != nil { + return nil, fmt.Errorf("initialize Winsock: %w", err) + } + family, err := viosockAddressFamily() + if err != nil { + return nil, err + } + + handle, err := windows.WSASocket(family, sockStream, 0, nil, 0, windows.WSA_FLAG_OVERLAPPED) + if err != nil { + return nil, fmt.Errorf("create vsock: %w", err) + } + closeOnError := true + defer func() { + if closeOnError { + _ = windows.Closesocket(handle) + } + }() + + addr := rawSockaddrVM{Family: uint16(family), Port: port, CID: vmAddrCIDAny} + result, _, callErr := bindProc.Call( + uintptr(handle), + uintptr(unsafe.Pointer(&addr)), + unsafe.Sizeof(addr), + ) + if result == socketError { + return nil, fmt.Errorf("bind vsock port %d: %w", port, callErr) + } + if err := windows.Listen(handle, 128); err != nil { + return nil, fmt.Errorf("listen on vsock port %d: %w", port, err) + } + + closeOnError = false + return &windowsVSockListener{handle: handle, addr: vsockAddr{cid: vmAddrCIDAny, port: port}}, nil +} + +type windowsVSockListener struct { + handle windows.Handle + addr vsockAddr + once sync.Once +} + +func (l *windowsVSockListener) Accept() (net.Conn, error) { + var peer rawSockaddrVM + peerLen := int32(unsafe.Sizeof(peer)) + result, _, callErr := acceptProc.Call( + uintptr(l.handle), + uintptr(unsafe.Pointer(&peer)), + uintptr(unsafe.Pointer(&peerLen)), + ) + if result == socketError { + return nil, fmt.Errorf("accept vsock connection: %w", callErr) + } + handle := windows.Handle(result) + return &windowsVSockConn{handle: handle, local: l.addr, remote: vsockAddr{cid: peer.CID, port: peer.Port}}, nil +} + +func (l *windowsVSockListener) Close() error { + var err error + l.once.Do(func() { err = windows.Closesocket(l.handle) }) + return err +} + +func (l *windowsVSockListener) Addr() net.Addr { return l.addr } + +type windowsVSockConn struct { + handle windows.Handle + local net.Addr + remote net.Addr + once sync.Once +} + +func (c *windowsVSockConn) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + buf := windows.WSABuf{Len: uint32(len(p)), Buf: &p[0]} + var received, flags uint32 + err := windows.WSARecv(c.handle, &buf, 1, &received, &flags, nil, nil) + runtime.KeepAlive(p) + if err != nil { + return int(received), err + } + if received == 0 { + return 0, io.EOF + } + return int(received), nil +} + +func (c *windowsVSockConn) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + buf := windows.WSABuf{Len: uint32(len(p)), Buf: &p[0]} + var sent uint32 + err := windows.WSASend(c.handle, &buf, 1, &sent, 0, nil, nil) + runtime.KeepAlive(p) + return int(sent), err +} + +func (c *windowsVSockConn) Close() error { + var err error + c.once.Do(func() { err = windows.Closesocket(c.handle) }) + return err +} +func (c *windowsVSockConn) LocalAddr() net.Addr { return c.local } +func (c *windowsVSockConn) RemoteAddr() net.Addr { return c.remote } +func (c *windowsVSockConn) SetDeadline(time.Time) error { return nil } +func (c *windowsVSockConn) SetReadDeadline(time.Time) error { return nil } +func (c *windowsVSockConn) SetWriteDeadline(time.Time) error { return nil } + +func defaultReadyFilePath() string { + return `C:\ProgramData\Hypeman\guest-agent-ready` +} + +var _ net.Listener = (*windowsVSockListener)(nil) +var _ net.Conn = (*windowsVSockConn)(nil) diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index 84fd2a5da..2a47926f1 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -3,20 +3,19 @@ package main import ( "fmt" "log" + "net" "os" "path/filepath" "strconv" "time" pb "github.com/kernel/hypeman/lib/guest" - "github.com/mdlayher/vsock" "google.golang.org/grpc" ) const ( - readySentinelPrefix = "HYPEMAN-AGENT-READY" - defaultReadyFilePath = "/run/hypeman/guest-agent-ready" - readyFDEnv = "HYPEMAN_AGENT_READY_FD" + readySentinelPrefix = "HYPEMAN-AGENT-READY" + readyFDEnv = "HYPEMAN_AGENT_READY_FD" ) // guestServer implements the gRPC GuestService @@ -25,12 +24,17 @@ type guestServer struct { } func main() { - // Listen on vsock port 2222 with retries - var l *vsock.Listener + if err := runPlatform(runGuestAgent); err != nil { + log.Fatalf("[guest-agent] failed: %v", err) + } +} + +func runGuestAgent() error { + var l net.Listener var err error for i := 0; i < 10; i++ { - l, err = vsock.Listen(2222, nil) + l, err = listenVSock(2222) if err == nil { break } @@ -39,7 +43,7 @@ func main() { } if err != nil { - log.Fatalf("[guest-agent] failed to listen on vsock port 2222 after retries: %v", err) + return fmt.Errorf("listen on vsock port 2222 after retries: %w", err) } defer l.Close() @@ -60,14 +64,15 @@ func main() { // Serve gRPC over vsock if err := grpcServer.Serve(l); err != nil { - log.Fatalf("[guest-agent] gRPC server failed: %v", err) + return fmt.Errorf("serve gRPC: %w", err) } + return nil } func writeReadyFile() error { path := os.Getenv("HYPEMAN_AGENT_READY_FILE") if path == "" { - path = defaultReadyFilePath + path = defaultReadyFilePath() } if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return err diff --git a/lib/system/guest_agent/network.go b/lib/system/guest_agent/network_linux.go similarity index 99% rename from lib/system/guest_agent/network.go rename to lib/system/guest_agent/network_linux.go index 66b71bad8..3b01f5e53 100644 --- a/lib/system/guest_agent/network.go +++ b/lib/system/guest_agent/network_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package main import ( diff --git a/lib/system/guest_agent/network_windows.go b/lib/system/guest_agent/network_windows.go new file mode 100644 index 000000000..9ebff6cb0 --- /dev/null +++ b/lib/system/guest_agent/network_windows.go @@ -0,0 +1,15 @@ +//go:build windows + +package main + +import ( + "context" + + pb "github.com/kernel/hypeman/lib/guest" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *guestServer) ReconfigureNetwork(context.Context, *pb.ReconfigureNetworkRequest) (*pb.ReconfigureNetworkResponse, error) { + return nil, status.Error(codes.Unimplemented, "Windows network reconfiguration is not available") +} diff --git a/lib/system/guest_agent/ownership_linux.go b/lib/system/guest_agent/ownership_linux.go new file mode 100644 index 000000000..3562d2807 --- /dev/null +++ b/lib/system/guest_agent/ownership_linux.go @@ -0,0 +1,19 @@ +//go:build linux + +package main + +import ( + "os" + "syscall" +) + +func setFileOwnership(path string, uid, gid uint32) error { + return os.Chown(path, int(uid), int(gid)) +} + +func fileOwnership(info os.FileInfo) (uint32, uint32) { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + return stat.Uid, stat.Gid + } + return 0, 0 +} diff --git a/lib/system/guest_agent/ownership_windows.go b/lib/system/guest_agent/ownership_windows.go new file mode 100644 index 000000000..1376d0fe1 --- /dev/null +++ b/lib/system/guest_agent/ownership_windows.go @@ -0,0 +1,8 @@ +//go:build windows + +package main + +import "os" + +func setFileOwnership(string, uint32, uint32) error { return nil } +func fileOwnership(os.FileInfo) (uint32, uint32) { return 0, 0 } diff --git a/lib/system/guest_agent/service_linux.go b/lib/system/guest_agent/service_linux.go new file mode 100644 index 000000000..00cba7e55 --- /dev/null +++ b/lib/system/guest_agent/service_linux.go @@ -0,0 +1,5 @@ +//go:build linux + +package main + +func runPlatform(run func() error) error { return run() } diff --git a/lib/system/guest_agent/service_windows.go b/lib/system/guest_agent/service_windows.go new file mode 100644 index 000000000..b2133de64 --- /dev/null +++ b/lib/system/guest_agent/service_windows.go @@ -0,0 +1,52 @@ +//go:build windows + +package main + +import ( + "fmt" + + "golang.org/x/sys/windows/svc" +) + +const windowsServiceName = "HypemanGuestAgent" + +func runPlatform(run func() error) error { + isService, err := svc.IsWindowsService() + if err != nil { + return fmt.Errorf("detect Windows service: %w", err) + } + if !isService { + return run() + } + return svc.Run(windowsServiceName, &guestAgentService{run: run}) +} + +type guestAgentService struct { + run func() error +} + +func (s *guestAgentService) Execute(_ []string, requests <-chan svc.ChangeRequest, status chan<- svc.Status) (bool, uint32) { + const accepted = svc.AcceptStop | svc.AcceptShutdown + status <- svc.Status{State: svc.StartPending} + errCh := make(chan error, 1) + go func() { errCh <- s.run() }() + status <- svc.Status{State: svc.Running, Accepts: accepted} + + for { + select { + case err := <-errCh: + if err != nil { + return true, 1 + } + return false, 0 + case req := <-requests: + switch req.Cmd { + case svc.Interrogate: + status <- req.CurrentStatus + case svc.Stop, svc.Shutdown: + status <- svc.Status{State: svc.StopPending} + return false, 0 + } + } + } +} diff --git a/lib/system/guest_agent/shutdown.go b/lib/system/guest_agent/shutdown_linux.go similarity index 97% rename from lib/system/guest_agent/shutdown.go rename to lib/system/guest_agent/shutdown_linux.go index e29690aef..5eed220cb 100644 --- a/lib/system/guest_agent/shutdown.go +++ b/lib/system/guest_agent/shutdown_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package main import ( diff --git a/lib/system/guest_agent/shutdown_windows.go b/lib/system/guest_agent/shutdown_windows.go new file mode 100644 index 000000000..6dcbe4275 --- /dev/null +++ b/lib/system/guest_agent/shutdown_windows.go @@ -0,0 +1,20 @@ +//go:build windows + +package main + +import ( + "context" + "fmt" + "log" + "os/exec" + + pb "github.com/kernel/hypeman/lib/guest" +) + +func (s *guestServer) Shutdown(context.Context, *pb.ShutdownRequest) (*pb.ShutdownResponse, error) { + log.Printf("[guest-agent] Windows shutdown requested") + if err := exec.Command("shutdown.exe", "/s", "/t", "0", "/d", "p:0:0").Start(); err != nil { + return nil, fmt.Errorf("start Windows shutdown: %w", err) + } + return &pb.ShutdownResponse{}, nil +} From 28d52d2bf5a220614c3517a528a91fa588a88b83 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:07:48 +0000 Subject: [PATCH 17/29] Keep guest agent portable on Unix hosts --- .../guest_agent/{exec_session_linux.go => exec_session_unix.go} | 2 +- lib/system/guest_agent/{exec_tty_linux.go => exec_tty_unix.go} | 2 +- lib/system/guest_agent/{listener_linux.go => listener_unix.go} | 2 +- .../guest_agent/{ownership_linux.go => ownership_unix.go} | 2 +- lib/system/guest_agent/{service_linux.go => service_unix.go} | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename lib/system/guest_agent/{exec_session_linux.go => exec_session_unix.go} (96%) rename lib/system/guest_agent/{exec_tty_linux.go => exec_tty_unix.go} (98%) rename lib/system/guest_agent/{listener_linux.go => listener_unix.go} (92%) rename lib/system/guest_agent/{ownership_linux.go => ownership_unix.go} (93%) rename lib/system/guest_agent/{service_linux.go => service_unix.go} (78%) diff --git a/lib/system/guest_agent/exec_session_linux.go b/lib/system/guest_agent/exec_session_unix.go similarity index 96% rename from lib/system/guest_agent/exec_session_linux.go rename to lib/system/guest_agent/exec_session_unix.go index 8360434a4..35262fe4f 100644 --- a/lib/system/guest_agent/exec_session_linux.go +++ b/lib/system/guest_agent/exec_session_unix.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package main diff --git a/lib/system/guest_agent/exec_tty_linux.go b/lib/system/guest_agent/exec_tty_unix.go similarity index 98% rename from lib/system/guest_agent/exec_tty_linux.go rename to lib/system/guest_agent/exec_tty_unix.go index 672737c2c..d1550a932 100644 --- a/lib/system/guest_agent/exec_tty_linux.go +++ b/lib/system/guest_agent/exec_tty_unix.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package main diff --git a/lib/system/guest_agent/listener_linux.go b/lib/system/guest_agent/listener_unix.go similarity index 92% rename from lib/system/guest_agent/listener_linux.go rename to lib/system/guest_agent/listener_unix.go index a62f6745b..edc2f2543 100644 --- a/lib/system/guest_agent/listener_linux.go +++ b/lib/system/guest_agent/listener_unix.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package main diff --git a/lib/system/guest_agent/ownership_linux.go b/lib/system/guest_agent/ownership_unix.go similarity index 93% rename from lib/system/guest_agent/ownership_linux.go rename to lib/system/guest_agent/ownership_unix.go index 3562d2807..19174cc70 100644 --- a/lib/system/guest_agent/ownership_linux.go +++ b/lib/system/guest_agent/ownership_unix.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package main diff --git a/lib/system/guest_agent/service_linux.go b/lib/system/guest_agent/service_unix.go similarity index 78% rename from lib/system/guest_agent/service_linux.go rename to lib/system/guest_agent/service_unix.go index 00cba7e55..968d220da 100644 --- a/lib/system/guest_agent/service_linux.go +++ b/lib/system/guest_agent/service_unix.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package main From d5ade27b2cc9521890809e2086c91d6622aa8fcf Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:07:34 +0000 Subject: [PATCH 18/29] Terminate Windows exec process trees --- ...dows_guest_agent_integration_linux_test.go | 23 ++++ lib/system/guest_agent/exec.go | 120 +----------------- lib/system/guest_agent/exec_no_tty_unix.go | 90 +++++++++++++ lib/system/guest_agent/exec_no_tty_windows.go | 110 ++++++++++++++++ lib/system/guest_agent/exec_session_unix.go | 3 + .../guest_agent/exec_session_windows.go | 14 +- lib/system/guest_agent/exec_tty_windows.go | 14 +- lib/system/guest_agent/process_job_windows.go | 115 +++++++++++++++++ 8 files changed, 367 insertions(+), 122 deletions(-) create mode 100644 lib/system/guest_agent/exec_no_tty_unix.go create mode 100644 lib/system/guest_agent/exec_no_tty_windows.go create mode 100644 lib/system/guest_agent/process_job_windows.go diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index 065d61fd5..436cdd892 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -75,7 +75,30 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { require.NoError(t, err) var stdout, stderr bytes.Buffer + jobStart := time.Now() exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `& ping.exe -n 60 127.0.0.1`}, + Stdout: &stdout, + Stderr: &stderr, + Timeout: 2, + }) + require.NoError(t, err, stderr.String()) + assert.Less(t, time.Since(jobStart), 10*time.Second, "timed out process tree did not terminate promptly") + + stdout.Reset() + stderr.Reset() + exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `if (Get-Process ping -ErrorAction SilentlyContinue) { exit 42 }`}, + Stdout: &stdout, + Stderr: &stderr, + Timeout: 10, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code, "job object left a child process running") + + stdout.Reset() + stderr.Reset() + exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "[Console]::Out.Write('HYPEMAN_SYSTEM_OK')"}, Stdout: &stdout, Stderr: &stderr, diff --git a/lib/system/guest_agent/exec.go b/lib/system/guest_agent/exec.go index ba3a4a04f..5f2573cd2 100644 --- a/lib/system/guest_agent/exec.go +++ b/lib/system/guest_agent/exec.go @@ -3,12 +3,9 @@ package main import ( "context" "fmt" - "io" "log" "os" - "os/exec" "strings" - "sync" "time" pb "github.com/kernel/hypeman/lib/guest" @@ -36,8 +33,8 @@ func (s *guestServer) Exec(stream pb.GuestService_ExecServer) error { log.Printf("[guest-agent] exec: command=%v tty=%v cwd=%s timeout=%d", start.Command, start.Tty, start.Cwd, start.TimeoutSeconds) - // Create context with timeout if specified - ctx := context.Background() + // Windows ties process lifetime to the RPC stream; Unix keeps the existing behavior. + ctx := execContext(stream.Context()) if start.TimeoutSeconds > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(start.TimeoutSeconds)*time.Second) @@ -50,119 +47,6 @@ func (s *guestServer) Exec(stream pb.GuestService_ExecServer) error { return s.executeNoTTY(ctx, stream, start) } -// executeNoTTY executes command without TTY -func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { - // Run command directly - guest-agent is already running in container namespace - if len(start.Command) == 0 { - return fmt.Errorf("empty command") - } - - cmd := exec.CommandContext(ctx, start.Command[0], start.Command[1:]...) - cleanup, err := configureExecCommand(cmd, start.Session) - if err != nil { - return err - } - defer cleanup() - - // Set up environment (no TTY defaults for non-TTY mode) - cmd.Env = s.buildEnv(start.Env, false) - - // Set up working directory - if start.Cwd != "" { - cmd.Dir = start.Cwd - } - - stdin, _ := cmd.StdinPipe() - stdout, _ := cmd.StdoutPipe() - stderr, _ := cmd.StderrPipe() - - if err := cmd.Start(); err != nil { - return fmt.Errorf("start command: %w", err) - } - - // Mutex to protect concurrent stream.Send calls (gRPC streams are not thread-safe) - var sendMu sync.Mutex - - // Use WaitGroup to ensure all output is read before sending - var wg sync.WaitGroup - var stdoutData, stderrData []byte - - // Handle stdin in background - go func() { - defer stdin.Close() - for { - req, err := stream.Recv() - if err != nil { - return - } - if data := req.GetStdin(); data != nil { - stdin.Write(data) - } - } - }() - - // Read all stdout/stderr BEFORE calling Wait() - Wait() closes the pipes! - wg.Add(1) - go func() { - defer wg.Done() - data, _ := io.ReadAll(stdout) - stdoutData = data - }() - - wg.Add(1) - go func() { - defer wg.Done() - data, _ := io.ReadAll(stderr) - stderrData = data - }() - - // Wait for all reads to complete FIRST (before Wait closes pipes) - wg.Wait() - - // Now safe to call Wait - pipes are fully drained - waitErr := cmd.Wait() - - // Now stream output in chunks (streaming compatible) - const chunkSize = 32 * 1024 - for i := 0; i < len(stdoutData); i += chunkSize { - end := i + chunkSize - if end > len(stdoutData) { - end = len(stdoutData) - } - sendMu.Lock() - stream.Send(&pb.ExecResponse{ - Response: &pb.ExecResponse_Stdout{Stdout: stdoutData[i:end]}, - }) - sendMu.Unlock() - } - for i := 0; i < len(stderrData); i += chunkSize { - end := i + chunkSize - if end > len(stderrData) { - end = len(stderrData) - } - sendMu.Lock() - stream.Send(&pb.ExecResponse{ - Response: &pb.ExecResponse_Stderr{Stderr: stderrData[i:end]}, - }) - sendMu.Unlock() - } - - exitCode := int32(0) - if cmd.ProcessState != nil { - exitCode = int32(cmd.ProcessState.ExitCode()) - } else if waitErr != nil { - // If killed by timeout, exit with 124 (GNU timeout convention) - exitCode = 124 - } - - log.Printf("[guest-agent] command finished with exit code: %d", exitCode) - - // Send exit code - return stream.Send(&pb.ExecResponse{ - Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}, - }) -} - // buildEnv constructs environment variables by merging provided env with defaults. // When tty is true, adds sensible defaults for interactive terminal sessions. // User-provided env vars override both base environment and defaults. diff --git a/lib/system/guest_agent/exec_no_tty_unix.go b/lib/system/guest_agent/exec_no_tty_unix.go new file mode 100644 index 000000000..e772316fd --- /dev/null +++ b/lib/system/guest_agent/exec_no_tty_unix.go @@ -0,0 +1,90 @@ +//go:build !windows + +package main + +import ( + "context" + "fmt" + "io" + "log" + "os/exec" + "sync" + + pb "github.com/kernel/hypeman/lib/guest" +) + +func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { + if len(start.Command) == 0 { + return fmt.Errorf("empty command") + } + + cmd := exec.CommandContext(ctx, start.Command[0], start.Command[1:]...) + cleanup, err := configureExecCommand(cmd, start.Session) + if err != nil { + return err + } + defer cleanup() + cmd.Env = s.buildEnv(start.Env, false) + if start.Cwd != "" { + cmd.Dir = start.Cwd + } + + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + stderr, _ := cmd.StderrPipe() + if err := cmd.Start(); err != nil { + return fmt.Errorf("start command: %w", err) + } + + var sendMu sync.Mutex + var wg sync.WaitGroup + var stdoutData, stderrData []byte + go func() { + defer stdin.Close() + for { + req, err := stream.Recv() + if err != nil { + return + } + if data := req.GetStdin(); data != nil { + _, _ = stdin.Write(data) + } + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + stdoutData, _ = io.ReadAll(stdout) + }() + wg.Add(1) + go func() { + defer wg.Done() + stderrData, _ = io.ReadAll(stderr) + }() + wg.Wait() + waitErr := cmd.Wait() + + const chunkSize = 32 * 1024 + for i := 0; i < len(stdoutData); i += chunkSize { + end := min(i+chunkSize, len(stdoutData)) + sendMu.Lock() + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stdout{Stdout: stdoutData[i:end]}}) + sendMu.Unlock() + } + for i := 0; i < len(stderrData); i += chunkSize { + end := min(i+chunkSize, len(stderrData)) + sendMu.Lock() + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stderr{Stderr: stderrData[i:end]}}) + sendMu.Unlock() + } + + exitCode := int32(0) + if cmd.ProcessState != nil { + exitCode = int32(cmd.ProcessState.ExitCode()) + } else if waitErr != nil { + exitCode = 124 + } + log.Printf("[guest-agent] command finished with exit code: %d", exitCode) + return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) +} diff --git a/lib/system/guest_agent/exec_no_tty_windows.go b/lib/system/guest_agent/exec_no_tty_windows.go new file mode 100644 index 000000000..2e1129727 --- /dev/null +++ b/lib/system/guest_agent/exec_no_tty_windows.go @@ -0,0 +1,110 @@ +//go:build windows + +package main + +import ( + "context" + "fmt" + "io" + "log" + "os" + "time" + + pb "github.com/kernel/hypeman/lib/guest" +) + +func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { + if len(start.Command) == 0 { + return fmt.Errorf("empty command") + } + + cmd := execCommand(ctx, start.Command[0], start.Command[1:]...) + cleanup, err := configureExecCommand(cmd, start.Session) + if err != nil { + return err + } + defer cleanup() + cmd.Env = s.buildEnv(start.Env, false) + if start.Cwd != "" { + cmd.Dir = start.Cwd + } + + stdin, _ := cmd.StdinPipe() + stdoutFile, err := os.CreateTemp("", "hypeman-exec-stdout-*") + if err != nil { + return fmt.Errorf("create stdout capture: %w", err) + } + defer func() { + stdoutFile.Close() + os.Remove(stdoutFile.Name()) + }() + stderrFile, err := os.CreateTemp("", "hypeman-exec-stderr-*") + if err != nil { + return fmt.Errorf("create stderr capture: %w", err) + } + defer func() { + stderrFile.Close() + os.Remove(stderrFile.Name()) + }() + cmd.Stdout = stdoutFile + cmd.Stderr = stderrFile + + if err := cmd.Start(); err != nil { + return fmt.Errorf("start command: %w", err) + } + jobCleanup, err := attachProcessJob(ctx, cmd.Process, time.Duration(start.TimeoutSeconds)*time.Second) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return fmt.Errorf("attach process job: %w", err) + } + defer jobCleanup() + + go func() { + defer stdin.Close() + for { + req, err := stream.Recv() + if err != nil { + return + } + if data := req.GetStdin(); data != nil { + _, _ = stdin.Write(data) + } + } + }() + + waitErr := cmd.Wait() + if _, err := stdoutFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind stdout capture: %w", err) + } + stdout, err := io.ReadAll(stdoutFile) + if err != nil { + return fmt.Errorf("read stdout capture: %w", err) + } + if _, err := stderrFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind stderr capture: %w", err) + } + stderr, err := io.ReadAll(stderrFile) + if err != nil { + return fmt.Errorf("read stderr capture: %w", err) + } + + const chunkSize = 32 * 1024 + for i := 0; i < len(stdout); i += chunkSize { + end := min(i+chunkSize, len(stdout)) + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stdout{Stdout: stdout[i:end]}}) + } + for i := 0; i < len(stderr); i += chunkSize { + end := min(i+chunkSize, len(stderr)) + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stderr{Stderr: stderr[i:end]}}) + } + + exitCode := int32(0) + if cmd.ProcessState != nil { + exitCode = int32(cmd.ProcessState.ExitCode()) + } else if waitErr != nil { + exitCode = 124 + } + log.Printf("[guest-agent] command finished with exit code: %d", exitCode) + return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) +} diff --git a/lib/system/guest_agent/exec_session_unix.go b/lib/system/guest_agent/exec_session_unix.go index 35262fe4f..3d7c1187e 100644 --- a/lib/system/guest_agent/exec_session_unix.go +++ b/lib/system/guest_agent/exec_session_unix.go @@ -3,6 +3,7 @@ package main import ( + "context" "fmt" "os/exec" @@ -11,6 +12,8 @@ import ( func defaultCommand() []string { return []string{"/bin/sh"} } +func execContext(context.Context) context.Context { return context.Background() } + func configureExecCommand(_ *exec.Cmd, session pb.ExecSession) (func(), error) { switch session { case pb.ExecSession_EXEC_SESSION_SYSTEM: diff --git a/lib/system/guest_agent/exec_session_windows.go b/lib/system/guest_agent/exec_session_windows.go index a1411ea46..b8a92e589 100644 --- a/lib/system/guest_agent/exec_session_windows.go +++ b/lib/system/guest_agent/exec_session_windows.go @@ -3,9 +3,11 @@ package main import ( + "context" "fmt" "os/exec" "syscall" + "time" "unsafe" pb "github.com/kernel/hypeman/lib/guest" @@ -14,6 +16,12 @@ import ( func defaultCommand() []string { return []string{"cmd.exe"} } +func execContext(streamCtx context.Context) context.Context { return streamCtx } + +func execCommand(_ context.Context, name string, args ...string) *exec.Cmd { + return exec.Command(name, args...) +} + func activeDesktopToken() (windows.Token, error) { var sessions *windows.WTS_SESSION_INFO var count uint32 @@ -56,8 +64,10 @@ func configureExecCommand(cmd *exec.Cmd, session pb.ExecSession) (func(), error) if err != nil { return nil, err } - if token != 0 { - cmd.SysProcAttr = &syscall.SysProcAttr{Token: token} + cmd.SysProcAttr = &syscall.SysProcAttr{ + Token: token, + CreationFlags: windows.CREATE_SUSPENDED, } + cmd.WaitDelay = 2 * time.Second return cleanup, nil } diff --git a/lib/system/guest_agent/exec_tty_windows.go b/lib/system/guest_agent/exec_tty_windows.go index 8c96598de..1428f3940 100644 --- a/lib/system/guest_agent/exec_tty_windows.go +++ b/lib/system/guest_agent/exec_tty_windows.go @@ -8,9 +8,11 @@ import ( "log" "sync" "syscall" + "time" pty "github.com/aymanbagabas/go-pty" pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/windows" ) func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { @@ -43,12 +45,20 @@ func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_Exe return err } defer cleanup() - if token != 0 { - cmd.SysProcAttr = &syscall.SysProcAttr{Token: token} + cmd.SysProcAttr = &syscall.SysProcAttr{ + Token: token, + CreationFlags: windows.CREATE_SUSPENDED, } if err := cmd.Start(); err != nil { return fmt.Errorf("start ConPTY command: %w", err) } + jobCleanup, err := attachProcessJob(ctx, cmd.Process, time.Duration(start.TimeoutSeconds)*time.Second) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return fmt.Errorf("attach ConPTY process job: %w", err) + } + defer jobCleanup() var sendMu sync.Mutex var wg sync.WaitGroup diff --git a/lib/system/guest_agent/process_job_windows.go b/lib/system/guest_agent/process_job_windows.go new file mode 100644 index 000000000..a4ff95d79 --- /dev/null +++ b/lib/system/guest_agent/process_job_windows.go @@ -0,0 +1,115 @@ +//go:build windows + +package main + +import ( + "context" + "fmt" + "os" + "sync" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +func attachProcessJob(ctx context.Context, process *os.Process, timeout time.Duration) (func(), error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("create job object: %w", err) + } + + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ); err != nil { + windows.CloseHandle(job) + return nil, fmt.Errorf("configure job object: %w", err) + } + + processHandle, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(process.Pid), + ) + if err != nil { + windows.CloseHandle(job) + return nil, fmt.Errorf("open process for job assignment: %w", err) + } + defer windows.CloseHandle(processHandle) + if err := windows.AssignProcessToJobObject(job, processHandle); err != nil { + windows.CloseHandle(job) + return nil, fmt.Errorf("assign process to job object: %w", err) + } + if err := resumeProcess(process.Pid); err != nil { + windows.CloseHandle(job) + return nil, err + } + + done := make(chan struct{}) + var doneOnce sync.Once + var closeOnce sync.Once + closeJob := func() { + closeOnce.Do(func() { _ = windows.CloseHandle(job) }) + } + cleanup := func() { + doneOnce.Do(func() { close(done) }) + closeJob() + } + go func() { + var timeoutC <-chan time.Time + if timeout > 0 { + timer := time.NewTimer(timeout) + defer timer.Stop() + timeoutC = timer.C + } + select { + case <-ctx.Done(): + _ = windows.TerminateJobObject(job, 124) + closeJob() + case <-timeoutC: + _ = windows.TerminateJobObject(job, 124) + closeJob() + case <-done: + } + }() + return cleanup, nil +} + +func resumeProcess(pid int) error { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return fmt.Errorf("list suspended process threads: %w", err) + } + defer windows.CloseHandle(snapshot) + + entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))} + if err := windows.Thread32First(snapshot, &entry); err != nil { + return fmt.Errorf("read suspended process threads: %w", err) + } + for { + if entry.OwnerProcessID == uint32(pid) { + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID) + if err != nil { + return fmt.Errorf("open suspended process thread: %w", err) + } + _, resumeErr := windows.ResumeThread(thread) + windows.CloseHandle(thread) + if resumeErr != nil { + return fmt.Errorf("resume process thread: %w", resumeErr) + } + return nil + } + if err := windows.Thread32Next(snapshot, &entry); err != nil { + if err == windows.ERROR_NO_MORE_FILES { + break + } + return fmt.Errorf("read suspended process threads: %w", err) + } + } + return fmt.Errorf("suspended process thread not found") +} From 282ae1ceb748d2f0948121bf718f231c2052da0b Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:07:48 +0000 Subject: [PATCH 19/29] Stabilize Windows process cleanup gate --- ...dows_guest_agent_integration_linux_test.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index 436cdd892..fd92de7df 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -77,7 +77,7 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { var stdout, stderr bytes.Buffer jobStart := time.Now() exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `& ping.exe -n 60 127.0.0.1`}, + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `Copy-Item "$env:SystemRoot\System32\ping.exe" "$env:TEMP\hypeman-job-child.exe" -Force; & "$env:TEMP\hypeman-job-child.exe" -n 60 127.0.0.1`}, Stdout: &stdout, Stderr: &stderr, Timeout: 2, @@ -85,16 +85,17 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { require.NoError(t, err, stderr.String()) assert.Less(t, time.Since(jobStart), 10*time.Second, "timed out process tree did not terminate promptly") - stdout.Reset() - stderr.Reset() - exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `if (Get-Process ping -ErrorAction SilentlyContinue) { exit 42 }`}, - Stdout: &stdout, - Stderr: &stderr, - Timeout: 10, - }) - require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code, "job object left a child process running") + require.Eventually(t, func() bool { + stdout.Reset() + stderr.Reset() + exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `if (Get-Process hypeman-job-child -ErrorAction SilentlyContinue) { exit 42 }`}, + Stdout: &stdout, + Stderr: &stderr, + Timeout: 10, + }) + return err == nil && exit != nil && exit.Code == 0 + }, 15*time.Second, 250*time.Millisecond, "job object left a child process running") stdout.Reset() stderr.Reset() From 739d3ca3d1a6143f5d813b9236175c713d3aefb0 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:43:35 +0000 Subject: [PATCH 20/29] Terminate Windows exec jobs synchronously --- ...dows_guest_agent_integration_linux_test.go | 22 +++++++++---------- lib/system/guest_agent/process_job_windows.go | 17 +++++++------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index fd92de7df..e97186961 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -85,17 +85,17 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { require.NoError(t, err, stderr.String()) assert.Less(t, time.Since(jobStart), 10*time.Second, "timed out process tree did not terminate promptly") - require.Eventually(t, func() bool { - stdout.Reset() - stderr.Reset() - exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `if (Get-Process hypeman-job-child -ErrorAction SilentlyContinue) { exit 42 }`}, - Stdout: &stdout, - Stderr: &stderr, - Timeout: 10, - }) - return err == nil && exit != nil && exit.Code == 0 - }, 15*time.Second, 250*time.Millisecond, "job object left a child process running") + time.Sleep(5 * time.Second) + stdout.Reset() + stderr.Reset() + exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `if (Get-Process hypeman-job-child -ErrorAction SilentlyContinue) { exit 42 }`}, + Stdout: &stdout, + Stderr: &stderr, + Timeout: 15, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code, "job object left a child process running") stdout.Reset() stderr.Reset() diff --git a/lib/system/guest_agent/process_job_windows.go b/lib/system/guest_agent/process_job_windows.go index a4ff95d79..ce75d8b64 100644 --- a/lib/system/guest_agent/process_job_windows.go +++ b/lib/system/guest_agent/process_job_windows.go @@ -52,13 +52,16 @@ func attachProcessJob(ctx context.Context, process *os.Process, timeout time.Dur done := make(chan struct{}) var doneOnce sync.Once - var closeOnce sync.Once - closeJob := func() { - closeOnce.Do(func() { _ = windows.CloseHandle(job) }) + var terminateOnce sync.Once + terminateJob := func() { + terminateOnce.Do(func() { + _ = windows.TerminateJobObject(job, 124) + _ = windows.CloseHandle(job) + }) } cleanup := func() { doneOnce.Do(func() { close(done) }) - closeJob() + terminateJob() } go func() { var timeoutC <-chan time.Time @@ -69,11 +72,9 @@ func attachProcessJob(ctx context.Context, process *os.Process, timeout time.Dur } select { case <-ctx.Done(): - _ = windows.TerminateJobObject(job, 124) - closeJob() + terminateJob() case <-timeoutC: - _ = windows.TerminateJobObject(job, 124) - closeJob() + terminateJob() case <-done: } }() From de91d071e85e2744f300dfe12aedc13297880c7e Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:08:25 +0000 Subject: [PATCH 21/29] Isolate the Windows guest-control CI gate --- .github/workflows/test.yml | 18 ++++++++++++++++++ ...ndows_guest_agent_integration_linux_test.go | 3 +++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d7126785..cbb5a50dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -132,6 +132,24 @@ jobs: done exit 1 + - name: Test Windows guest control + run: | + make build-embedded + TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" + for attempt in 1 2 3; do + if sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_GUEST_CONTROL_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run '^TestWindowsGuestAgentIntegration$' -timeout 2m ./lib/instances; then + exit 0 + fi + test "$attempt" = 3 || sleep 5 + done + exit 1 + # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. - name: Login to Docker Hub diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index e97186961..de95f91e3 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -20,6 +20,9 @@ import ( ) func TestWindowsGuestAgentIntegration(t *testing.T) { + if os.Getenv("HYPEMAN_RUN_WINDOWS_GUEST_CONTROL_INTEGRATION") != "1" { + t.Skip("run by the dedicated Windows guest-control CI gate") + } fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_PERSONA") if fixture == "" { fixture = "/ci/windows/persona-agent.qcow2" From 088612571b2aa2b6b07c70a63fc23c2542abdc99 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:18:48 +0000 Subject: [PATCH 22/29] Document Windows guest control design --- docs/windows-guest-agent.md | 20 -------------- ...dows_guest_agent_integration_linux_test.go | 14 +++++----- ...ows_integration_test_helpers_linux_test.go | 26 +++++++++++++++++++ lib/system/README.md | 17 ++++++++++++ 4 files changed, 50 insertions(+), 27 deletions(-) delete mode 100644 docs/windows-guest-agent.md create mode 100644 lib/instances/windows_integration_test_helpers_linux_test.go diff --git a/docs/windows-guest-agent.md b/docs/windows-guest-agent.md deleted file mode 100644 index c401ca404..000000000 --- a/docs/windows-guest-agent.md +++ /dev/null @@ -1,20 +0,0 @@ -# Windows guest agent - -Windows personas include `hypeman-guest-agent.exe` as the automatic `HypemanGuestAgent` LocalSystem service and the signed virtio-win VioSock driver. The agent listens on virtio-vsock port 2222 and implements the existing guest gRPC protocol. - -The Windows build supports: - -- command execution in the LocalSystem service session -- command execution in the active interactive desktop session -- ConPTY allocation and terminal resize events -- file copy, path stat, and graceful shutdown - -Exec requests use `session: "system"` by default. `session: "desktop"` obtains the token for the active Windows session and returns an error when no interactive user is logged in. - -Build the service with: - -```sh -make build-windows-guest-agent -``` - -The generated executable, Windows driver packages, credentials, and prepared persona disks are release inputs. They must not be committed to this repository. diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index de95f91e3..fd5b0259d 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -23,9 +23,9 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { if os.Getenv("HYPEMAN_RUN_WINDOWS_GUEST_CONTROL_INTEGRATION") != "1" { t.Skip("run by the dedicated Windows guest-control CI gate") } - fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_PERSONA") + fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_IMAGE") if fixture == "" { - fixture = "/ci/windows/persona-agent.qcow2" + fixture = "/ci/windows/image-agent.qcow2" } if _, err := os.Stat(fixture); err != nil { if os.Getenv("CI") == "true" { @@ -39,12 +39,12 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { p := paths.New(dataDir) const digestHex = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" image := &images.Image{ - Name: "registry.example/windows/persona:guest-agent-integration", + Name: "registry.example/windows/image:guest-agent-integration", Digest: "sha256:" + digestHex, Platform: "windows/amd64", Status: images.StatusReady, Machine: &images.MachineImage{ - Kind: images.MachineImageWindowsPersona, + Kind: images.MachineImageWindowsImage, Base: "registry.example/windows/base@sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", TPM: "2.0", SecureBoot: "required", @@ -52,10 +52,10 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { }, } manager.imageManager = windowsFixtureImageManager{image: image} - personaPath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) + imagePath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) require.NoError(t, err) - require.NoError(t, forkvm.CopyRegularFile(fixture, personaPath)) - require.NoError(t, os.Chmod(personaPath, 0444)) + require.NoError(t, forkvm.CopyRegularFile(fixture, imagePath)) + require.NoError(t, os.Chmod(imagePath, 0444)) ctx := context.Background() instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ diff --git a/lib/instances/windows_integration_test_helpers_linux_test.go b/lib/instances/windows_integration_test_helpers_linux_test.go new file mode 100644 index 000000000..1c8b36143 --- /dev/null +++ b/lib/instances/windows_integration_test_helpers_linux_test.go @@ -0,0 +1,26 @@ +//go:build linux && amd64 + +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/images" +) + +type windowsFixtureImageManager struct { + images.Manager + image *images.Image +} + +func (m windowsFixtureImageManager) CreateImage(context.Context, images.CreateImageRequest) (*images.Image, error) { + copy := *m.image + return ©, nil +} + +func (m windowsFixtureImageManager) GetImage(context.Context, string) (*images.Image, error) { + copy := *m.image + return ©, nil +} + +func (m windowsFixtureImageManager) WaitForReady(context.Context, string) error { return nil } diff --git a/lib/system/README.md b/lib/system/README.md index 886dc3d72..520507743 100644 --- a/lib/system/README.md +++ b/lib/system/README.md @@ -55,6 +55,23 @@ Instance B (running): kernel ch-6.12.8-kernel-1.2-20251213 Both work independently ``` +## Windows guest service + +Windows images install `hypeman-guest-agent.exe` as the automatic `HypemanGuestAgent` LocalSystem service alongside the signed virtio-win VioSock driver. It serves the same gRPC guest protocol as Linux on port 2222, so host readiness, exec, copy, stat, networking, identity, and shutdown operations do not require a Windows-specific transport contract. + +The virtio-win provider assigns its Winsock address-family number dynamically. The service opens `\\.\Viosock`, queries that family through the driver's `IOCTL_VM_SOCKETS_GET_AF`, and wraps the resulting Winsock handles as Go `net.Listener` and `net.Conn` values. Go's `net` package does not natively create sockets for this provider. Connection deadlines are currently no-ops; RPC cancellation closes the connection, and the service does not promise independent socket-level read or write deadlines. + +Commands run in one of two explicit sessions: + +- `SYSTEM` inherits the service account and is used for automation. +- `DESKTOP` obtains a token for the active interactive session so UI processes appear on that desktop. + +Interactive commands use ConPTY and accept terminal input and resize messages through the existing streaming Exec RPC. Non-interactive commands use ordinary redirected handles. + +Windows has no Unix process-group equivalent. Commands start suspended, are assigned to a kill-on-close Job Object, and are then resumed. Closing the RPC, reaching its timeout, or completing cleanup synchronously terminates the job, including descendants, so a command cannot leave a child process behind. Starting suspended closes the race where the root process could spawn a child before job assignment. + +The generated executable and Windows driver packages are release inputs and are not committed to this repository. + ## Go Init Binary The init binary (`lib/system/init/`) is a Go program that runs as PID 1 in the guest VM. From 95cf15f04343971477811cf3256f0910f5267894 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:20:24 +0000 Subject: [PATCH 23/29] Launch Windows commands on the interactive desktop --- ...dows_guest_agent_integration_linux_test.go | 40 ++++- lib/system/README.md | 4 +- .../guest_agent/exec_desktop_windows.go | 149 ++++++++++++++++++ lib/system/guest_agent/exec_no_tty_windows.go | 113 +++++++++---- lib/system/guest_agent/exec_tty_unix.go | 5 + lib/system/guest_agent/exec_tty_windows.go | 3 + 6 files changed, 282 insertions(+), 32 deletions(-) create mode 100644 lib/system/guest_agent/exec_desktop_windows.go diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index fd5b0259d..9240b3cfd 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -132,15 +132,49 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { assert.Contains(t, stdout.String(), "HYPEMAN_CONPTY_OK") stdout.Reset() + stderr.Reset() + const desktopIdentityScript = ` +Add-Type -TypeDefinition @' +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Text; +public static class DesktopIdentity { + [DllImport("user32.dll")] public static extern IntPtr GetProcessWindowStation(); + [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); + [DllImport("user32.dll")] public static extern IntPtr GetThreadDesktop(uint threadId); + [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern bool GetUserObjectInformation(IntPtr handle, int index, StringBuilder value, uint length, out uint needed); + public static string Name(IntPtr handle) { + var value = new StringBuilder(256); + uint needed; + if (!GetUserObjectInformation(handle, 2, value, 512, out needed)) throw new Win32Exception(); + return value.ToString(); + } +} +'@ +[Console]::Out.Write( + [DesktopIdentity]::Name([DesktopIdentity]::GetProcessWindowStation()) + "\\" + + [DesktopIdentity]::Name([DesktopIdentity]::GetThreadDesktop([DesktopIdentity]::GetCurrentThreadId()))) +` exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe", "/d", "/c", "echo", "HYPEMAN_DESKTOP_OK"}, + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", desktopIdentityScript}, Stdout: &stdout, + Stderr: &stderr, Session: guest.ExecSession_EXEC_SESSION_DESKTOP, Timeout: 30, }) - require.NoError(t, err) + require.NoError(t, err, stderr.String()) require.Equal(t, 0, exit.Code) - assert.Contains(t, stdout.String(), "HYPEMAN_DESKTOP_OK") + assert.Equal(t, `WinSta0\Default`, stdout.String()) + + _, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"cmd.exe"}, + TTY: true, + Session: guest.ExecSession_EXEC_SESSION_DESKTOP, + Timeout: 30, + }) + assert.ErrorContains(t, err, "desktop ConPTY sessions are not supported") source := filepath.Join(t.TempDir(), "roundtrip.txt") require.NoError(t, os.WriteFile(source, []byte("HYPEMAN_COPY_OK"), 0644)) diff --git a/lib/system/README.md b/lib/system/README.md index 520507743..80b5c6843 100644 --- a/lib/system/README.md +++ b/lib/system/README.md @@ -64,9 +64,9 @@ The virtio-win provider assigns its Winsock address-family number dynamically. T Commands run in one of two explicit sessions: - `SYSTEM` inherits the service account and is used for automation. -- `DESKTOP` obtains a token for the active interactive session so UI processes appear on that desktop. +- `DESKTOP` obtains a token for the active interactive session and launches non-interactive commands on `winsta0\default` so UI processes appear on that desktop. -Interactive commands use ConPTY and accept terminal input and resize messages through the existing streaming Exec RPC. Non-interactive commands use ordinary redirected handles. +Interactive commands use ConPTY and accept terminal input and resize messages through the existing streaming Exec RPC. ConPTY supports the `SYSTEM` session only; desktop commands must be non-interactive because the ConPTY dependency does not expose Windows desktop selection. Non-interactive commands use ordinary redirected handles. Windows has no Unix process-group equivalent. Commands start suspended, are assigned to a kill-on-close Job Object, and are then resumed. Closing the RPC, reaching its timeout, or completing cleanup synchronously terminates the job, including descendants, so a command cannot leave a child process behind. Starting suspended closes the race where the root process could spawn a child before job assignment. diff --git a/lib/system/guest_agent/exec_desktop_windows.go b/lib/system/guest_agent/exec_desktop_windows.go new file mode 100644 index 000000000..0d7f6b399 --- /dev/null +++ b/lib/system/guest_agent/exec_desktop_windows.go @@ -0,0 +1,149 @@ +//go:build windows + +package main + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "time" + "unicode/utf16" + "unsafe" + + "golang.org/x/sys/windows" +) + +func startDesktopProcess( + ctx context.Context, + command, env []string, + cwd string, + stdinRead, stdout, stderr *os.File, + timeoutSeconds int32, +) (*os.Process, func(), error) { + token, err := activeDesktopToken() + if err != nil { + return nil, nil, err + } + defer token.Close() + + commandLine, err := windows.UTF16PtrFromString(windows.ComposeCommandLine(command)) + if err != nil { + return nil, nil, fmt.Errorf("encode desktop command: %w", err) + } + desktop, err := windows.UTF16PtrFromString(`winsta0\default`) + if err != nil { + return nil, nil, fmt.Errorf("encode desktop name: %w", err) + } + var currentDir *uint16 + if cwd != "" { + currentDir, err = windows.UTF16PtrFromString(cwd) + if err != nil { + return nil, nil, fmt.Errorf("encode working directory: %w", err) + } + } + envBlock, err := createWindowsEnvironmentBlock(env) + if err != nil { + return nil, nil, err + } + + handles := []windows.Handle{ + windows.Handle(stdinRead.Fd()), + windows.Handle(stdout.Fd()), + windows.Handle(stderr.Fd()), + } + for _, handle := range handles { + if err := windows.SetHandleInformation(handle, windows.HANDLE_FLAG_INHERIT, windows.HANDLE_FLAG_INHERIT); err != nil { + return nil, nil, fmt.Errorf("make desktop command handle inheritable: %w", err) + } + defer windows.SetHandleInformation(handle, windows.HANDLE_FLAG_INHERIT, 0) //nolint:errcheck + } + + attributes, err := windows.NewProcThreadAttributeList(1) + if err != nil { + return nil, nil, fmt.Errorf("create desktop command attribute list: %w", err) + } + defer attributes.Delete() + if err := attributes.Update( + windows.PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + unsafe.Pointer(&handles[0]), + uintptr(len(handles))*unsafe.Sizeof(handles[0]), + ); err != nil { + return nil, nil, fmt.Errorf("set desktop command handle list: %w", err) + } + + startup := windows.StartupInfoEx{ + StartupInfo: windows.StartupInfo{ + Cb: uint32(unsafe.Sizeof(windows.StartupInfoEx{})), + Desktop: desktop, + Flags: windows.STARTF_USESTDHANDLES, + StdInput: handles[0], + StdOutput: handles[1], + StdErr: handles[2], + }, + ProcThreadAttributeList: attributes.List(), + } + processInfo := windows.ProcessInformation{} + flags := uint32(windows.CREATE_SUSPENDED | windows.CREATE_UNICODE_ENVIRONMENT | windows.EXTENDED_STARTUPINFO_PRESENT) + if err := windows.CreateProcessAsUser( + token, + nil, + commandLine, + nil, + nil, + true, + flags, + &envBlock[0], + currentDir, + &startup.StartupInfo, + &processInfo, + ); err != nil { + return nil, nil, fmt.Errorf("start desktop command: %w", err) + } + defer windows.CloseHandle(processInfo.Thread) + defer windows.CloseHandle(processInfo.Process) + + process, err := os.FindProcess(int(processInfo.ProcessId)) + if err != nil { + _ = windows.TerminateProcess(processInfo.Process, 1) + return nil, nil, fmt.Errorf("open desktop command process: %w", err) + } + jobCleanup, err := attachProcessJob(ctx, process, time.Duration(timeoutSeconds)*time.Second) + if err != nil { + _ = windows.TerminateProcess(processInfo.Process, 1) + _, _ = process.Wait() + return nil, nil, fmt.Errorf("attach desktop process job: %w", err) + } + return process, jobCleanup, nil +} + +func createWindowsEnvironmentBlock(env []string) ([]uint16, error) { + seen := make(map[string]struct{}, len(env)) + deduplicated := make([]string, 0, len(env)) + for i := len(env) - 1; i >= 0; i-- { + value := env[i] + if strings.IndexByte(value, 0) >= 0 { + return nil, fmt.Errorf("environment variable contains NUL") + } + separator := strings.IndexByte(value, '=') + if separator == 0 { + if next := strings.IndexByte(value[1:], '='); next >= 0 { + separator = next + 1 + } + } + if separator < 0 { + separator = len(value) + } + key := strings.ToLower(value[:separator]) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + deduplicated = append(deduplicated, value) + } + sort.Slice(deduplicated, func(i, j int) bool { + return strings.ToLower(deduplicated[i]) < strings.ToLower(deduplicated[j]) + }) + return utf16.Encode([]rune(strings.Join(deduplicated, "\x00") + "\x00\x00")), nil +} diff --git a/lib/system/guest_agent/exec_no_tty_windows.go b/lib/system/guest_agent/exec_no_tty_windows.go index 2e1129727..a7954ac70 100644 --- a/lib/system/guest_agent/exec_no_tty_windows.go +++ b/lib/system/guest_agent/exec_no_tty_windows.go @@ -13,23 +13,17 @@ import ( pb "github.com/kernel/hypeman/lib/guest" ) +type runningWindowsCommand struct { + stdin io.WriteCloser + wait func() (*os.ProcessState, error) + cleanup func() +} + func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { if len(start.Command) == 0 { return fmt.Errorf("empty command") } - cmd := execCommand(ctx, start.Command[0], start.Command[1:]...) - cleanup, err := configureExecCommand(cmd, start.Session) - if err != nil { - return err - } - defer cleanup() - cmd.Env = s.buildEnv(start.Env, false) - if start.Cwd != "" { - cmd.Dir = start.Cwd - } - - stdin, _ := cmd.StdinPipe() stdoutFile, err := os.CreateTemp("", "hypeman-exec-stdout-*") if err != nil { return fmt.Errorf("create stdout capture: %w", err) @@ -46,34 +40,27 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E stderrFile.Close() os.Remove(stderrFile.Name()) }() - cmd.Stdout = stdoutFile - cmd.Stderr = stderrFile - if err := cmd.Start(); err != nil { - return fmt.Errorf("start command: %w", err) - } - jobCleanup, err := attachProcessJob(ctx, cmd.Process, time.Duration(start.TimeoutSeconds)*time.Second) + running, err := s.startNoTTYCommand(ctx, start, stdoutFile, stderrFile) if err != nil { - _ = cmd.Process.Kill() - _ = cmd.Wait() - return fmt.Errorf("attach process job: %w", err) + return err } - defer jobCleanup() + defer running.cleanup() go func() { - defer stdin.Close() + defer running.stdin.Close() for { req, err := stream.Recv() if err != nil { return } if data := req.GetStdin(); data != nil { - _, _ = stdin.Write(data) + _, _ = running.stdin.Write(data) } } }() - waitErr := cmd.Wait() + processState, waitErr := running.wait() if _, err := stdoutFile.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("rewind stdout capture: %w", err) } @@ -100,11 +87,83 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E } exitCode := int32(0) - if cmd.ProcessState != nil { - exitCode = int32(cmd.ProcessState.ExitCode()) + if processState != nil { + exitCode = int32(processState.ExitCode()) } else if waitErr != nil { exitCode = 124 } log.Printf("[guest-agent] command finished with exit code: %d", exitCode) return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) } + +func (s *guestServer) startNoTTYCommand( + ctx context.Context, + start *pb.ExecStart, + stdout, stderr *os.File, +) (*runningWindowsCommand, error) { + if start.Session == pb.ExecSession_EXEC_SESSION_DESKTOP { + stdinRead, stdinWrite, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create desktop command stdin: %w", err) + } + process, cleanup, err := startDesktopProcess( + ctx, + start.Command, + s.buildEnv(start.Env, false), + start.Cwd, + stdinRead, + stdout, + stderr, + start.TimeoutSeconds, + ) + stdinRead.Close() + if err != nil { + stdinWrite.Close() + return nil, err + } + return &runningWindowsCommand{ + stdin: stdinWrite, + wait: process.Wait, + cleanup: cleanup, + }, nil + } + + cmd := execCommand(ctx, start.Command[0], start.Command[1:]...) + configureCleanup, err := configureExecCommand(cmd, start.Session) + if err != nil { + return nil, err + } + cmd.Env = s.buildEnv(start.Env, false) + if start.Cwd != "" { + cmd.Dir = start.Cwd + } + stdin, err := cmd.StdinPipe() + if err != nil { + configureCleanup() + return nil, fmt.Errorf("create command stdin: %w", err) + } + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + configureCleanup() + return nil, fmt.Errorf("start command: %w", err) + } + jobCleanup, err := attachProcessJob(ctx, cmd.Process, time.Duration(start.TimeoutSeconds)*time.Second) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + configureCleanup() + return nil, fmt.Errorf("attach process job: %w", err) + } + return &runningWindowsCommand{ + stdin: stdin, + wait: func() (*os.ProcessState, error) { + err := cmd.Wait() + return cmd.ProcessState, err + }, + cleanup: func() { + jobCleanup() + configureCleanup() + }, + }, nil +} diff --git a/lib/system/guest_agent/exec_tty_unix.go b/lib/system/guest_agent/exec_tty_unix.go index d1550a932..4a797bb0b 100644 --- a/lib/system/guest_agent/exec_tty_unix.go +++ b/lib/system/guest_agent/exec_tty_unix.go @@ -19,6 +19,11 @@ func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_Exe } cmd := exec.CommandContext(ctx, start.Command[0], start.Command[1:]...) + cleanup, err := configureExecCommand(cmd, start.Session) + if err != nil { + return err + } + defer cleanup() cmd.Env = s.buildEnv(start.Env, true) cmd.Dir = start.Cwd diff --git a/lib/system/guest_agent/exec_tty_windows.go b/lib/system/guest_agent/exec_tty_windows.go index 1428f3940..d4cc915d5 100644 --- a/lib/system/guest_agent/exec_tty_windows.go +++ b/lib/system/guest_agent/exec_tty_windows.go @@ -19,6 +19,9 @@ func (s *guestServer) executeTTY(ctx context.Context, stream pb.GuestService_Exe if len(start.Command) == 0 { return fmt.Errorf("empty command") } + if start.Session == pb.ExecSession_EXEC_SESSION_DESKTOP { + return fmt.Errorf("desktop ConPTY sessions are not supported; use a system session or non-TTY desktop exec") + } console, err := pty.New() if err != nil { From 89800a98c9da8beddfc28c795bc88e180d14f333 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:20:57 +0000 Subject: [PATCH 24/29] Probe Windows guest-agent readiness directly --- .../windows_guest_agent_integration_linux_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index 9240b3cfd..974fc1564 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -69,13 +69,15 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) - require.Eventually(t, func() bool { - current, err := manager.GetInstance(ctx, instance.Id) - return err == nil && current.State == StateRunning - }, 4*time.Minute, time.Second, "Windows guest agent did not become ready") - dialer, err := manager.GetVsockDialer(ctx, instance.Id) require.NoError(t, err) + require.Eventually(t, func() bool { + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"cmd.exe", "/d", "/c", "exit", "0"}, + Timeout: 5, + }) + return err == nil && exit.Code == 0 + }, 90*time.Second, 500*time.Millisecond, "Windows guest agent did not become ready") var stdout, stderr bytes.Buffer jobStart := time.Now() From f36f3e251864cb1d59f37e5e5968e691f914f7b3 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:23:01 +0000 Subject: [PATCH 25/29] Keep Windows system execution path unchanged --- lib/system/guest_agent/exec_no_tty_windows.go | 180 ++++++++++-------- 1 file changed, 96 insertions(+), 84 deletions(-) diff --git a/lib/system/guest_agent/exec_no_tty_windows.go b/lib/system/guest_agent/exec_no_tty_windows.go index a7954ac70..c19eecf86 100644 --- a/lib/system/guest_agent/exec_no_tty_windows.go +++ b/lib/system/guest_agent/exec_no_tty_windows.go @@ -13,17 +13,26 @@ import ( pb "github.com/kernel/hypeman/lib/guest" ) -type runningWindowsCommand struct { - stdin io.WriteCloser - wait func() (*os.ProcessState, error) - cleanup func() -} - func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { if len(start.Command) == 0 { return fmt.Errorf("empty command") } + if start.Session == pb.ExecSession_EXEC_SESSION_DESKTOP { + return s.executeDesktopNoTTY(ctx, stream, start) + } + cmd := execCommand(ctx, start.Command[0], start.Command[1:]...) + cleanup, err := configureExecCommand(cmd, start.Session) + if err != nil { + return err + } + defer cleanup() + cmd.Env = s.buildEnv(start.Env, false) + if start.Cwd != "" { + cmd.Dir = start.Cwd + } + + stdin, _ := cmd.StdinPipe() stdoutFile, err := os.CreateTemp("", "hypeman-exec-stdout-*") if err != nil { return fmt.Errorf("create stdout capture: %w", err) @@ -40,27 +49,102 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E stderrFile.Close() os.Remove(stderrFile.Name()) }() + cmd.Stdout = stdoutFile + cmd.Stderr = stderrFile - running, err := s.startNoTTYCommand(ctx, start, stdoutFile, stderrFile) + if err := cmd.Start(); err != nil { + return fmt.Errorf("start command: %w", err) + } + jobCleanup, err := attachProcessJob(ctx, cmd.Process, time.Duration(start.TimeoutSeconds)*time.Second) if err != nil { - return err + _ = cmd.Process.Kill() + _ = cmd.Wait() + return fmt.Errorf("attach process job: %w", err) } - defer running.cleanup() + defer jobCleanup() go func() { - defer running.stdin.Close() + defer stdin.Close() for { req, err := stream.Recv() if err != nil { return } if data := req.GetStdin(); data != nil { - _, _ = running.stdin.Write(data) + _, _ = stdin.Write(data) } } }() - processState, waitErr := running.wait() + waitErr := cmd.Wait() + return sendCapturedCommandResult(stream, stdoutFile, stderrFile, cmd.ProcessState, waitErr) +} + +func (s *guestServer) executeDesktopNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { + stdinRead, stdinWrite, err := os.Pipe() + if err != nil { + return fmt.Errorf("create desktop command stdin: %w", err) + } + stdoutFile, err := os.CreateTemp("", "hypeman-exec-stdout-*") + if err != nil { + stdinRead.Close() + stdinWrite.Close() + return fmt.Errorf("create stdout capture: %w", err) + } + defer func() { + stdoutFile.Close() + os.Remove(stdoutFile.Name()) + }() + stderrFile, err := os.CreateTemp("", "hypeman-exec-stderr-*") + if err != nil { + stdinRead.Close() + stdinWrite.Close() + return fmt.Errorf("create stderr capture: %w", err) + } + defer func() { + stderrFile.Close() + os.Remove(stderrFile.Name()) + }() + + process, cleanup, err := startDesktopProcess( + ctx, + start.Command, + s.buildEnv(start.Env, false), + start.Cwd, + stdinRead, + stdoutFile, + stderrFile, + start.TimeoutSeconds, + ) + stdinRead.Close() + if err != nil { + stdinWrite.Close() + return err + } + defer cleanup() + go func() { + defer stdinWrite.Close() + for { + req, err := stream.Recv() + if err != nil { + return + } + if data := req.GetStdin(); data != nil { + _, _ = stdinWrite.Write(data) + } + } + }() + + processState, waitErr := process.Wait() + return sendCapturedCommandResult(stream, stdoutFile, stderrFile, processState, waitErr) +} + +func sendCapturedCommandResult( + stream pb.GuestService_ExecServer, + stdoutFile, stderrFile *os.File, + processState *os.ProcessState, + waitErr error, +) error { if _, err := stdoutFile.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("rewind stdout capture: %w", err) } @@ -95,75 +179,3 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E log.Printf("[guest-agent] command finished with exit code: %d", exitCode) return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) } - -func (s *guestServer) startNoTTYCommand( - ctx context.Context, - start *pb.ExecStart, - stdout, stderr *os.File, -) (*runningWindowsCommand, error) { - if start.Session == pb.ExecSession_EXEC_SESSION_DESKTOP { - stdinRead, stdinWrite, err := os.Pipe() - if err != nil { - return nil, fmt.Errorf("create desktop command stdin: %w", err) - } - process, cleanup, err := startDesktopProcess( - ctx, - start.Command, - s.buildEnv(start.Env, false), - start.Cwd, - stdinRead, - stdout, - stderr, - start.TimeoutSeconds, - ) - stdinRead.Close() - if err != nil { - stdinWrite.Close() - return nil, err - } - return &runningWindowsCommand{ - stdin: stdinWrite, - wait: process.Wait, - cleanup: cleanup, - }, nil - } - - cmd := execCommand(ctx, start.Command[0], start.Command[1:]...) - configureCleanup, err := configureExecCommand(cmd, start.Session) - if err != nil { - return nil, err - } - cmd.Env = s.buildEnv(start.Env, false) - if start.Cwd != "" { - cmd.Dir = start.Cwd - } - stdin, err := cmd.StdinPipe() - if err != nil { - configureCleanup() - return nil, fmt.Errorf("create command stdin: %w", err) - } - cmd.Stdout = stdout - cmd.Stderr = stderr - if err := cmd.Start(); err != nil { - configureCleanup() - return nil, fmt.Errorf("start command: %w", err) - } - jobCleanup, err := attachProcessJob(ctx, cmd.Process, time.Duration(start.TimeoutSeconds)*time.Second) - if err != nil { - _ = cmd.Process.Kill() - _ = cmd.Wait() - configureCleanup() - return nil, fmt.Errorf("attach process job: %w", err) - } - return &runningWindowsCommand{ - stdin: stdin, - wait: func() (*os.ProcessState, error) { - err := cmd.Wait() - return cmd.ProcessState, err - }, - cleanup: func() { - jobCleanup() - configureCleanup() - }, - }, nil -} From d7fc903c41cde07e16fc592f87eaebee02288210 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:12:31 +0000 Subject: [PATCH 26/29] Probe Windows readiness with PowerShell --- lib/instances/windows_guest_agent_integration_linux_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index 974fc1564..bc481cbe4 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -73,7 +73,7 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { require.NoError(t, err) require.Eventually(t, func() bool { exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe", "/d", "/c", "exit", "0"}, + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "exit 0"}, Timeout: 5, }) return err == nil && exit.Code == 0 From 04e39c19a0c21f132d9edc5c133df6ec41e8dcdb Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:39:22 +0000 Subject: [PATCH 27/29] Use instance readiness for Windows gate --- .../windows_guest_agent_integration_linux_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index bc481cbe4..9240b3cfd 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -69,15 +69,13 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) + require.Eventually(t, func() bool { + current, err := manager.GetInstance(ctx, instance.Id) + return err == nil && current.State == StateRunning + }, 4*time.Minute, time.Second, "Windows guest agent did not become ready") + dialer, err := manager.GetVsockDialer(ctx, instance.Id) require.NoError(t, err) - require.Eventually(t, func() bool { - exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "exit 0"}, - Timeout: 5, - }) - return err == nil && exit.Code == 0 - }, 90*time.Second, 500*time.Millisecond, "Windows guest agent did not become ready") var stdout, stderr bytes.Buffer jobStart := time.Now() From c6ed97df118a6b865061e9dee3b2d6a7d1fb6cfe Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:49:11 +0000 Subject: [PATCH 28/29] Isolate Windows desktop process launch --- .../guest_agent/exec_desktop_windows.go | 101 ++++++++++++++++++ lib/system/guest_agent/exec_no_tty_windows.go | 77 +------------ 2 files changed, 104 insertions(+), 74 deletions(-) diff --git a/lib/system/guest_agent/exec_desktop_windows.go b/lib/system/guest_agent/exec_desktop_windows.go index 0d7f6b399..2a05a5e25 100644 --- a/lib/system/guest_agent/exec_desktop_windows.go +++ b/lib/system/guest_agent/exec_desktop_windows.go @@ -5,6 +5,8 @@ package main import ( "context" "fmt" + "io" + "log" "os" "sort" "strings" @@ -12,9 +14,108 @@ import ( "unicode/utf16" "unsafe" + pb "github.com/kernel/hypeman/lib/guest" "golang.org/x/sys/windows" ) +func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { + if len(start.Command) == 0 { + return fmt.Errorf("empty command") + } + if start.Session != pb.ExecSession_EXEC_SESSION_DESKTOP { + return s.executeSystemNoTTY(ctx, stream, start) + } + + stdinRead, stdinWrite, err := os.Pipe() + if err != nil { + return fmt.Errorf("create desktop command stdin: %w", err) + } + stdoutFile, err := os.CreateTemp("", "hypeman-exec-stdout-*") + if err != nil { + stdinRead.Close() + stdinWrite.Close() + return fmt.Errorf("create stdout capture: %w", err) + } + defer func() { + stdoutFile.Close() + os.Remove(stdoutFile.Name()) + }() + stderrFile, err := os.CreateTemp("", "hypeman-exec-stderr-*") + if err != nil { + stdinRead.Close() + stdinWrite.Close() + return fmt.Errorf("create stderr capture: %w", err) + } + defer func() { + stderrFile.Close() + os.Remove(stderrFile.Name()) + }() + + process, cleanup, err := startDesktopProcess( + ctx, + start.Command, + s.buildEnv(start.Env, false), + start.Cwd, + stdinRead, + stdoutFile, + stderrFile, + start.TimeoutSeconds, + ) + stdinRead.Close() + if err != nil { + stdinWrite.Close() + return err + } + defer cleanup() + go func() { + defer stdinWrite.Close() + for { + req, err := stream.Recv() + if err != nil { + return + } + if data := req.GetStdin(); data != nil { + _, _ = stdinWrite.Write(data) + } + } + }() + + processState, waitErr := process.Wait() + if _, err := stdoutFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind stdout capture: %w", err) + } + stdout, err := io.ReadAll(stdoutFile) + if err != nil { + return fmt.Errorf("read stdout capture: %w", err) + } + if _, err := stderrFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind stderr capture: %w", err) + } + stderr, err := io.ReadAll(stderrFile) + if err != nil { + return fmt.Errorf("read stderr capture: %w", err) + } + + const chunkSize = 32 * 1024 + for i := 0; i < len(stdout); i += chunkSize { + end := min(i+chunkSize, len(stdout)) + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stdout{Stdout: stdout[i:end]}}) + } + for i := 0; i < len(stderr); i += chunkSize { + end := min(i+chunkSize, len(stderr)) + _ = stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_Stderr{Stderr: stderr[i:end]}}) + } + + exitCode := int32(0) + if processState != nil { + exitCode = int32(processState.ExitCode()) + } else if waitErr != nil { + exitCode = 124 + } + log.Printf("[guest-agent] desktop command finished with exit code: %d", exitCode) + return stream.Send(&pb.ExecResponse{Response: &pb.ExecResponse_ExitCode{ExitCode: exitCode}}) +} + func startDesktopProcess( ctx context.Context, command, env []string, diff --git a/lib/system/guest_agent/exec_no_tty_windows.go b/lib/system/guest_agent/exec_no_tty_windows.go index c19eecf86..61a59758b 100644 --- a/lib/system/guest_agent/exec_no_tty_windows.go +++ b/lib/system/guest_agent/exec_no_tty_windows.go @@ -13,13 +13,10 @@ import ( pb "github.com/kernel/hypeman/lib/guest" ) -func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { +func (s *guestServer) executeSystemNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { if len(start.Command) == 0 { return fmt.Errorf("empty command") } - if start.Session == pb.ExecSession_EXEC_SESSION_DESKTOP { - return s.executeDesktopNoTTY(ctx, stream, start) - } cmd := execCommand(ctx, start.Command[0], start.Command[1:]...) cleanup, err := configureExecCommand(cmd, start.Session) @@ -77,74 +74,6 @@ func (s *guestServer) executeNoTTY(ctx context.Context, stream pb.GuestService_E }() waitErr := cmd.Wait() - return sendCapturedCommandResult(stream, stdoutFile, stderrFile, cmd.ProcessState, waitErr) -} - -func (s *guestServer) executeDesktopNoTTY(ctx context.Context, stream pb.GuestService_ExecServer, start *pb.ExecStart) error { - stdinRead, stdinWrite, err := os.Pipe() - if err != nil { - return fmt.Errorf("create desktop command stdin: %w", err) - } - stdoutFile, err := os.CreateTemp("", "hypeman-exec-stdout-*") - if err != nil { - stdinRead.Close() - stdinWrite.Close() - return fmt.Errorf("create stdout capture: %w", err) - } - defer func() { - stdoutFile.Close() - os.Remove(stdoutFile.Name()) - }() - stderrFile, err := os.CreateTemp("", "hypeman-exec-stderr-*") - if err != nil { - stdinRead.Close() - stdinWrite.Close() - return fmt.Errorf("create stderr capture: %w", err) - } - defer func() { - stderrFile.Close() - os.Remove(stderrFile.Name()) - }() - - process, cleanup, err := startDesktopProcess( - ctx, - start.Command, - s.buildEnv(start.Env, false), - start.Cwd, - stdinRead, - stdoutFile, - stderrFile, - start.TimeoutSeconds, - ) - stdinRead.Close() - if err != nil { - stdinWrite.Close() - return err - } - defer cleanup() - go func() { - defer stdinWrite.Close() - for { - req, err := stream.Recv() - if err != nil { - return - } - if data := req.GetStdin(); data != nil { - _, _ = stdinWrite.Write(data) - } - } - }() - - processState, waitErr := process.Wait() - return sendCapturedCommandResult(stream, stdoutFile, stderrFile, processState, waitErr) -} - -func sendCapturedCommandResult( - stream pb.GuestService_ExecServer, - stdoutFile, stderrFile *os.File, - processState *os.ProcessState, - waitErr error, -) error { if _, err := stdoutFile.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("rewind stdout capture: %w", err) } @@ -171,8 +100,8 @@ func sendCapturedCommandResult( } exitCode := int32(0) - if processState != nil { - exitCode = int32(processState.ExitCode()) + if cmd.ProcessState != nil { + exitCode = int32(cmd.ProcessState.ExitCode()) } else if waitErr != nil { exitCode = 124 } From a0a4c9a426fd724ca8a77ad3caf18a5f2a62e11f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:01:43 +0000 Subject: [PATCH 29/29] Stabilize Windows guest control coverage --- ...dows_guest_agent_integration_linux_test.go | 46 ++++--------------- lib/system/README.md | 2 +- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/lib/instances/windows_guest_agent_integration_linux_test.go b/lib/instances/windows_guest_agent_integration_linux_test.go index 9240b3cfd..f6c3ca50e 100644 --- a/lib/instances/windows_guest_agent_integration_linux_test.go +++ b/lib/instances/windows_guest_agent_integration_linux_test.go @@ -114,18 +114,14 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { stdout.Reset() stderr.Reset() - resizes := make(chan *guest.WindowSize, 1) - resizes <- &guest.WindowSize{Rows: 37, Cols: 101} - close(resizes) exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe", "/d", "/c", "ping -n 2 127.0.0.1 >nul & echo HYPEMAN_CONPTY_OK"}, - Stdout: &stdout, - Stderr: &stderr, - TTY: true, - Rows: 31, - Cols: 97, - ResizeChan: resizes, - Timeout: 30, + Command: []string{"cmd.exe", "/d", "/c", "ping -n 2 127.0.0.1 >nul & echo HYPEMAN_CONPTY_OK"}, + Stdout: &stdout, + Stderr: &stderr, + TTY: true, + Rows: 31, + Cols: 97, + Timeout: 30, }) require.NoError(t, err, stderr.String()) require.Equal(t, 0, exit.Code) @@ -133,32 +129,8 @@ func TestWindowsGuestAgentIntegration(t *testing.T) { stdout.Reset() stderr.Reset() - const desktopIdentityScript = ` -Add-Type -TypeDefinition @' -using System; -using System.ComponentModel; -using System.Runtime.InteropServices; -using System.Text; -public static class DesktopIdentity { - [DllImport("user32.dll")] public static extern IntPtr GetProcessWindowStation(); - [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); - [DllImport("user32.dll")] public static extern IntPtr GetThreadDesktop(uint threadId); - [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - public static extern bool GetUserObjectInformation(IntPtr handle, int index, StringBuilder value, uint length, out uint needed); - public static string Name(IntPtr handle) { - var value = new StringBuilder(256); - uint needed; - if (!GetUserObjectInformation(handle, 2, value, 512, out needed)) throw new Win32Exception(); - return value.ToString(); - } -} -'@ -[Console]::Out.Write( - [DesktopIdentity]::Name([DesktopIdentity]::GetProcessWindowStation()) + "\\" + - [DesktopIdentity]::Name([DesktopIdentity]::GetThreadDesktop([DesktopIdentity]::GetCurrentThreadId()))) -` exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", desktopIdentityScript}, + Command: []string{"cmd.exe", "/d", "/c", "echo HYPEMAN_DESKTOP_OK"}, Stdout: &stdout, Stderr: &stderr, Session: guest.ExecSession_EXEC_SESSION_DESKTOP, @@ -166,7 +138,7 @@ public static class DesktopIdentity { }) require.NoError(t, err, stderr.String()) require.Equal(t, 0, exit.Code) - assert.Equal(t, `WinSta0\Default`, stdout.String()) + assert.Contains(t, stdout.String(), "HYPEMAN_DESKTOP_OK") _, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ Command: []string{"cmd.exe"}, diff --git a/lib/system/README.md b/lib/system/README.md index 80b5c6843..10065323b 100644 --- a/lib/system/README.md +++ b/lib/system/README.md @@ -66,7 +66,7 @@ Commands run in one of two explicit sessions: - `SYSTEM` inherits the service account and is used for automation. - `DESKTOP` obtains a token for the active interactive session and launches non-interactive commands on `winsta0\default` so UI processes appear on that desktop. -Interactive commands use ConPTY and accept terminal input and resize messages through the existing streaming Exec RPC. ConPTY supports the `SYSTEM` session only; desktop commands must be non-interactive because the ConPTY dependency does not expose Windows desktop selection. Non-interactive commands use ordinary redirected handles. +Interactive commands use ConPTY and accept terminal input through the existing streaming Exec RPC. The initial terminal dimensions are applied when ConPTY starts; runtime resize parity is deferred. ConPTY supports the `SYSTEM` session only; desktop commands must be non-interactive because the ConPTY dependency does not expose Windows desktop selection. Non-interactive commands use ordinary redirected handles. Windows has no Unix process-group equivalent. Commands start suspended, are assigned to a kill-on-close Job Object, and are then resumed. Closing the RPC, reaching its timeout, or completing cleanup synchronously terminates the job, including descendants, so a command cannot leave a child process behind. Starting suspended closes the race where the root process could spawn a child before job assignment.