diff --git a/internal/layers/wcow_mount.go b/internal/layers/wcow_mount.go index c5ee9cbabb..672606f262 100644 --- a/internal/layers/wcow_mount.go +++ b/internal/layers/wcow_mount.go @@ -488,9 +488,10 @@ func mountHypervIsolatedBlockCIMLayers(ctx context.Context, l *wcowBlockCIMLayer hostPath := filepath.Join(l.scratchLayerPath, "sandbox.vhdx") + // Refs format scratch vhds for c-wcow cases only. scsiMount, err := vm.SCSIManager.AddVirtualDisk(ctx, hostPath, false, vm.ID(), "", &scsi.MountConfig{ - // TODO(ambarve): Add SCSI config to format the scratch in guest + FormatWithRefs: vm.HasConfidentialPolicy(), }) if err != nil { return nil, nil, fmt.Errorf("failed to add SCSI scratch VHD: %w", err) diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index c19d54e89a..3e69e6cba7 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -27,6 +27,10 @@ const ( // ResourceTypeMappedVirtualDisk is the modify resource type for mapped // virtual disks ResourceTypeMappedVirtualDisk guestrequest.ResourceType = "MappedVirtualDisk" + // ResourceTypeMappedVirtualDiskForContainerScratch is the modify resource type + // specifically for refs formatting and mounting scratch vhds for c-wcow cases only. + ResourceTypeMappedVirtualDiskForContainerScratch guestrequest.ResourceType = "MappedVirtualDiskForContainerScratch" + ResourceTypeWCOWBlockCims guestrequest.ResourceType = "WCOWBlockCims" // ResourceTypeNetwork is the modify resource type for the `NetworkAdapterV2` // device. ResourceTypeNetwork guestrequest.ResourceType = "Network" @@ -51,12 +55,6 @@ const ( ResourceTypeSecurityPolicy guestrequest.ResourceType = "SecurityPolicy" // ResourceTypePolicyFragment is the modify resource type for injecting policy fragments. ResourceTypePolicyFragment guestrequest.ResourceType = "SecurityPolicyFragment" - // ResourceTypeWCOWBlockCims is the modify resource type for mounting block cims for hyperv - // wcow containers. - ResourceTypeWCOWBlockCims guestrequest.ResourceType = "WCOWBlockCims" - // ResourceTypeMappedVirtualDiskForContainerScratch is the modify resource type - // specifically for refs formatting and mounting scratch vhds for c-wcow cases only. - ResourceTypeMappedVirtualDiskForContainerScratch guestrequest.ResourceType = "MappedVirtualDiskForContainerScratch" ) // This class is used by a modify request to add or remove a combined layers diff --git a/internal/uvm/scsi/backend.go b/internal/uvm/scsi/backend.go index 6219a15172..6ec4208d8a 100644 --- a/internal/uvm/scsi/backend.go +++ b/internal/uvm/scsi/backend.go @@ -170,6 +170,13 @@ func mountRequest(controller, lun uint, path string, config *mountConfig, osType ResourceType: guestresource.ResourceTypeMappedVirtualDisk, RequestType: guestrequest.RequestTypeAdd, } + // This option is set only for cwcow scratch disk mount requests + // where we need to format the disk with refs. + // For refs the scratch disk size should > 40 GB. + if config.formatWithRefs { + req.ResourceType = guestresource.ResourceTypeMappedVirtualDiskForContainerScratch + } + switch osType { case "windows": // We don't check config.readOnly here, as that will still result in the overall attachment being read-only. @@ -185,6 +192,7 @@ func mountRequest(controller, lun uint, path string, config *mountConfig, osType ContainerPath: path, Lun: int32(lun), } + case "linux": req.Settings = guestresource.LCOWMappedVirtualDisk{ MountPath: path, diff --git a/internal/uvm/scsi/manager.go b/internal/uvm/scsi/manager.go index ff8374038e..272ea9c992 100644 --- a/internal/uvm/scsi/manager.go +++ b/internal/uvm/scsi/manager.go @@ -86,6 +86,9 @@ type MountConfig struct { // BlockDev indicates if the device should be mounted as a block device. // This is only supported for LCOW. BlockDev bool + // FormatWithRefs indicates to refs format the disk. + // This is only supported for CWCOW scratch disks. + FormatWithRefs bool } // Mount represents a SCSI device that has been attached to a VM, and potentially @@ -162,6 +165,7 @@ func (m *Manager) AddVirtualDisk( ensureFilesystem: mc.EnsureFilesystem, filesystem: mc.Filesystem, blockDev: mc.BlockDev, + formatWithRefs: mc.FormatWithRefs, } } return m.add(ctx, diff --git a/internal/uvm/scsi/mount.go b/internal/uvm/scsi/mount.go index cf64f4ec42..e9e6fed324 100644 --- a/internal/uvm/scsi/mount.go +++ b/internal/uvm/scsi/mount.go @@ -45,6 +45,7 @@ type mountConfig struct { options []string ensureFilesystem bool filesystem string + formatWithRefs bool } func (mm *mountManager) mount(ctx context.Context, controller, lun uint, path string, c *mountConfig) (_ string, err error) { diff --git a/internal/wclayer/cim/mount.go b/internal/wclayer/cim/mount.go index 56d0d0ac7d..89fa13ccdc 100644 --- a/internal/wclayer/cim/mount.go +++ b/internal/wclayer/cim/mount.go @@ -108,6 +108,7 @@ func MergeMountBlockCIMLayer(ctx context.Context, mergedLayer *cimfs.BlockCIM, p if err != nil { return "", fmt.Errorf("generated cim mount GUID: %w", err) } + return cimfs.MountMergedBlockCIMs(mergedLayer, parentLayers, mountFlags, volumeGUID) } diff --git a/internal/windevice/devicequery_test.go b/internal/windevice/devicequery_test.go index 288e6c996f..b45bd83320 100644 --- a/internal/windevice/devicequery_test.go +++ b/internal/windevice/devicequery_test.go @@ -4,12 +4,19 @@ package windevice import ( "context" + "errors" + "fmt" + "log" "path/filepath" + "strings" + "syscall" "testing" "time" "github.com/Microsoft/go-winio/vhd" + "github.com/Microsoft/hcsshim/internal/fsformatter" "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc/mgr" ) func TestGetDeviceInterfaceInstances(t *testing.T) { @@ -78,3 +85,137 @@ func TestGetDeviceInterfaceInstances(t *testing.T) { t.Fatalf("expected interface lists to have same length") } } + +const ( + diskSizeInGB = 35 + defaultVHDxBlockSizeMB = 1 +) + +// startFsformatterDriver checks if fsformatter driver +// has already been loaded and starts the service. +// Returns syscall.ERROR_FILE_NOT_FOUND if driver +// is not loaded. +func startFsformatterDriver() error { + m, err := mgr.Connect() + if err != nil { + log.Fatalf("Failed to connect to service manager: %v", err) + } + defer func() { + _ = m.Disconnect() + }() + + // Ensure fsformatter driver is loaded by querying for the service. + serviceName := "kernelfsformatter" + s, err := m.OpenService(serviceName) + if err != nil { + return syscall.ERROR_FILE_NOT_FOUND + } + defer s.Close() + + _, err = s.Query() + if err != nil { + return syscall.ERROR_FILE_NOT_FOUND + } + + err = s.Start() + if err != nil && !strings.Contains(err.Error(), "An instance of the service is already running") { + return fmt.Errorf("Failed to start service: %w", err) + } + + return nil +} + +func TestFormatVHDXToReFS(t *testing.T) { + // Ensure fsformatter service is loaded and started + err := startFsformatterDriver() + if err != nil { + // if driver is not loaded already, skip. + if errors.Is(err, syscall.ERROR_FILE_NOT_FOUND) { + t.Skip() + } + t.Fatalf("Failed to start service: %v", err) + } + + ctx := context.Background() + initialInterfacesList, err := getDeviceInterfaceInstancesByClass(ctx, &devClassDiskGUID, false) + if err != nil { + t.Fatalf("failed to get initial disk interfaces: %v", err) + } + // We expect to see only one device initially + if len(initialInterfacesList) != 1 { + t.Fatalf("unexpected number of initial disk interfaces: %v", len(initialInterfacesList)) + } + t.Logf("initial interface list: %+v\n", initialInterfacesList) + + // Create a fixed VHDX of 31 GB (refs needs size to be > 30GB) + tempDir := t.TempDir() + vhdxPath := filepath.Join(tempDir, "test.vhdx") + if err := vhd.CreateVhdx(vhdxPath, diskSizeInGB, defaultVHDxBlockSizeMB); err != nil { + t.Fatalf("failed to create VHDX: %v", err) + } + + diskHandle, err := vhd.OpenVirtualDisk(vhdxPath, vhd.VirtualDiskAccessNone, vhd.OpenVirtualDiskFlagNone) + if err != nil { + t.Fatalf("failed to open VHD handle: %s", err) + } + t.Cleanup(func() { + if closeErr := windows.CloseHandle(windows.Handle(diskHandle)); closeErr != nil { + t.Logf("Failed to close VHD handle: %s", closeErr) + } + }) + + err = vhd.AttachVirtualDisk(diskHandle, vhd.AttachVirtualDiskFlagNone, &vhd.AttachVirtualDiskParameters{Version: 1}) + if err != nil { + t.Fatalf("failed to attach VHD: %s", err) + } + t.Cleanup(func() { + if detachErr := vhd.DetachVirtualDisk(diskHandle); detachErr != nil { + t.Logf("failed to detach vhd: %s", detachErr) + } + }) + // Disks might take time to show up. Add a small delay + time.Sleep(1 * time.Second) + + interfaceListAfterVHDAttach, err := getDeviceInterfaceInstancesByClass(ctx, &devClassDiskGUID, false) + if err != nil { + t.Fatalf("failed to get initial disk interfaces: %v", err) + } + t.Logf("interface list after attaching VHD: %+v\n", interfaceListAfterVHDAttach) + + if len(initialInterfacesList) != (len(interfaceListAfterVHDAttach) - 1) { + t.Fatalf("expected to find exactly 1 new interface in the returned interfaces list") + } + + for _, iPath := range interfaceListAfterVHDAttach { + // Take only the newly attached vhdx + if iPath == initialInterfacesList[0] { + continue + } + utf16Path, err := windows.UTF16PtrFromString(iPath) + if err != nil { + t.Fatalf("failed to convert interface path [%s] to utf16: %v", iPath, err) + } + + handle, err := windows.CreateFile(utf16Path, windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + nil, windows.OPEN_EXISTING, 0, 0) + if err != nil { + t.Fatalf("failed to get handle to interface path [%s]: %v", iPath, err) + } + defer windows.Close(handle) + + deviceNumber, err := getStorageDeviceNumber(ctx, handle) + if err != nil { + t.Fatalf("failed to get physical device number: %v", err) + } + diskPath := fmt.Sprintf(fsformatter.VirtualDevObjectPathFormat, deviceNumber.DeviceNumber) + t.Logf("diskPath %v", diskPath) + + // Invoke refs formatter and ensure it passes. + mountedVolumePath, err := fsformatter.InvokeFsFormatter(ctx, diskPath) + if err != nil { + t.Fatalf("invoking refsFormatter failed: %v", err) + } + t.Logf("mountedVolumePath %v", mountedVolumePath) + } +}