diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index 7811c9eabf..b41e4a1b15 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" @@ -18,6 +19,7 @@ import ( "github.com/Microsoft/hcsshim/pkg/annotations" eventstypes "github.com/containerd/containerd/api/events" task "github.com/containerd/containerd/api/runtime/task/v2" + "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/v2/core/runtime" "github.com/containerd/errdefs" "github.com/opencontainers/runtime-spec/specs-go" @@ -25,6 +27,56 @@ import ( "golang.org/x/sync/errgroup" ) +// initializeWCOWBootFiles handles the initialization of boot files for WCOW VMs +func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs []*types.Mount, s *specs.Spec) error { + var ( + layerFolders []string + err error + ) + + if s.Windows != nil { + layerFolders = s.Windows.LayerFolders + } + + wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, rootfs, layerFolders) + if err != nil { + return err + } + + if wopts.SecurityPolicyEnabled { + if wopts.BootFiles.BootType != uvm.BlockCIMBoot { + return fmt.Errorf("security policy (confidential mode) only works with block CIM based layers") + } + // we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs + wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath() + wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath() + } else if wopts.BootFiles.BootType == uvm.BlockCIMBoot { + // Supporting hyperv isolation with block CIM layers requires changes in + // the image pull path to prepare the EFI VHD. But more importantly, since both the + // container and UtilityVM files will be in the same CIM, the early boot + // code (bootmgr, winload etc.) will need to support reading the UVM OS + // files from `UtilityVM/Files` inside the CIM. Once that support is added + // we can enable this. + return fmt.Errorf("hyperv isolation is not supported with block CIM layers yet") + } + + // writable EFI VHD is a valid config for both confidential and regular hyperv + // isolated WCOW. Override the default value here if required. + if wopts.WritableEFI { + // Make a copy of EFI VHD, we can't risk the UVM accidentally modifying + // the original VHD. make copy next to the scratch VHD, this assumes that + // the scratch is located in the separate directory dedicated for this + // UVM. + writableEFIVHDPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.BootFiles.BlockCIMFiles.EFIVHDPath)) + if err := copyfile.CopyFile(ctx, wopts.BootFiles.BlockCIMFiles.EFIVHDPath, writableEFIVHDPath, false); err != nil { + return fmt.Errorf("failed to copy EFI VHD at %s: %w", writableEFIVHDPath, err) + } + wopts.BootFiles.BlockCIMFiles.EFIVHDPath = writableEFIVHDPath + } + + return nil +} + // shimPod represents the logical grouping of all tasks in a single set of // shared namespaces. The pod sandbox (container) is represented by the task // that matches the `shimPod.ID()` @@ -123,16 +175,11 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques return nil, err } case *uvm.OptionsWCOW: - var layerFolders []string - if s.Windows != nil { - layerFolders = s.Windows.LayerFolders - } wopts := (opts).(*uvm.OptionsWCOW) - wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders) + err = initializeWCOWBootFiles(ctx, wopts, req.Rootfs, s) if err != nil { return nil, err } - parent, err = uvm.CreateWCOW(ctx, wopts) if err != nil { return nil, err diff --git a/internal/fsformatter/formatter_driver.go b/internal/fsformatter/formatter_driver.go index 14f743cf0b..7317e4c779 100644 --- a/internal/fsformatter/formatter_driver.go +++ b/internal/fsformatter/formatter_driver.go @@ -60,12 +60,17 @@ func (filesystemType kernelFormatVolumeFilesystemTypes) String() string { type kernelFormatVolumeFormatInputBufferFlags uint32 -const kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000) +const ( + kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000) + kernelFormatVolumeFormatInputBufferFlagSuperFloppy = kernelFormatVolumeFormatInputBufferFlags(0x00000001) +) func (flag kernelFormatVolumeFormatInputBufferFlags) String() string { switch flag { case kernelFormatVolumeFormatInputBufferFlagNone: return "kernelFormatVolumeFormatInputBufferFlagNone" + case kernelFormatVolumeFormatInputBufferFlagSuperFloppy: + return "kernelFormatVolumeFormatInputBufferFlagSuperFloppy" default: return "Unknown" } @@ -211,7 +216,7 @@ func KmFmtCreateFormatInputBuffer(diskPath string) *KernelFormatVolumeFormatInpu inputBuffer := (*KernelFormatVolumeFormatInputBuffer)(unsafe.Pointer(&buf[0])) inputBuffer.Size = uint64(bufferSize) - inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagNone + inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagSuperFloppy inputBuffer.FsParameters.FileSystemType = kernelFormatVolumeFilesystemTypeRefs inputBuffer.FsParameters.VolumeLabelLength = 0 diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go index e321d4e092..689e02bb66 100644 --- a/internal/gcs-sidecar/handlers.go +++ b/internal/gcs-sidecar/handlers.go @@ -338,7 +338,11 @@ func (b *Bridge) modifySettings(req *request) (err error) { case guestresource.ResourceTypeWCOWBlockCims: // This is request to mount the merged cim at given volumeGUID - wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWBlockCIMMounts) + if modifyGuestSettingsRequest.RequestType == guestrequest.RequestTypeRemove { + return fmt.Errorf("not implemented") + } + + wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWBlockCIMMounts) log.G(ctx).Tracef("WCOWBlockCIMMounts { %v}", wcowBlockCimMounts) // The block device takes some time to show up. Wait for a few seconds. @@ -386,6 +390,11 @@ func (b *Bridge) modifySettings(req *request) (err error) { return nil case guestresource.ResourceTypeCWCOWCombinedLayers: + + if modifyGuestSettingsRequest.RequestType == guestrequest.RequestTypeRemove { + return fmt.Errorf("not implemented") + } + settings := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWCombinedLayers) containerID := settings.ContainerID log.G(ctx).Tracef("CWCOWCombinedLayers:: ContainerID: %v, ContainerRootPath: %v, Layers: %v, ScratchPath: %v", diff --git a/internal/gcs-sidecar/uvm.go b/internal/gcs-sidecar/uvm.go index 4fccdf3df6..9578e0273c 100644 --- a/internal/gcs-sidecar/uvm.go +++ b/internal/gcs-sidecar/uvm.go @@ -106,7 +106,7 @@ func unmarshalContainerModifySettings(req *request) (_ *prot.ContainerModifySett modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk case guestresource.ResourceTypeWCOWBlockCims: - wcowBlockCimMounts := &guestresource.WCOWBlockCIMMounts{} + wcowBlockCimMounts := &guestresource.CWCOWBlockCIMMounts{} if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowBlockCimMounts); err != nil { return nil, fmt.Errorf("invalid ResourceTypeWCOWBlockCims request: %w", err) } diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 6dddc6f41d..c0ebdc2e7b 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -456,9 +456,9 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM // containing the files is exposed via UVM_SECURITY_CONTEXT_DIR env var. // It may be an error to have a security policy but not expose it to the // container as in that case it can never be checked as correct by a verifier. - if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.UVMSecurityPolicyEnv, true) { + if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWSecurityPolicyEnv, true) { encodedPolicy := h.securityPolicyEnforcer.EncodedSecurityPolicy() - hostAMDCert := settings.OCISpecification.Annotations[annotations.HostAMDCertificate] + hostAMDCert := settings.OCISpecification.Annotations[annotations.LCOWHostAMDCertificate] if len(encodedPolicy) > 0 || len(hostAMDCert) > 0 || len(h.uvmReferenceInfo) > 0 { // Use os.MkdirTemp to make sure that the directory is unique. securityContextDir, err := os.MkdirTemp(settings.OCISpecification.Root.Path, securitypolicy.SecurityContextDirTemplate) diff --git a/internal/layers/wcow_mount.go b/internal/layers/wcow_mount.go index 9df9f199eb..c5ee9cbabb 100644 --- a/internal/layers/wcow_mount.go +++ b/internal/layers/wcow_mount.go @@ -43,7 +43,7 @@ func MountWCOWLayers(ctx context.Context, containerID string, vm *uvm.UtilityVM, if vm == nil { return mountProcessIsolatedBlockCIMLayers(ctx, containerID, l) } - return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with block cim layers") + return mountHypervIsolatedBlockCIMLayers(ctx, l, vm, containerID) default: return nil, nil, fmt.Errorf("invalid layer type %T", wl) } @@ -440,7 +440,8 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm }) } - err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS) + // containerID isn't required when using non-confidential pods (WCIFS based layers can't run confidential pods) + err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.WCIFS, "") if err != nil { return nil, nil, err } @@ -454,6 +455,90 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm }, nil } +func mountHypervIsolatedBlockCIMLayers(ctx context.Context, l *wcowBlockCIMLayers, vm *uvm.UtilityVM, containerID string) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) { + ctx, span := oc.StartSpan(ctx, "mountHyperVIsolatedBlockCIMLayers") + defer func() { + oc.SetSpanStatus(span, err) + span.End() + }() + + rcl := &resources.ResourceCloserList{} + defer func() { + if err != nil { + if rErr := rcl.Release(ctx); rErr != nil { + log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr) + } + } + }() + + log.G(ctx).WithFields(logrus.Fields{ + "scratch": l.scratchLayerPath, + "merged layer": l.mergedLayer, + "parent layers": l.parentLayers, + }).Debug("mounting hyperv isolated block CIM layers") + + mountedCIMs, err := vm.MountBlockCIMs(ctx, l.mergedLayer, l.parentLayers, containerID) + if err != nil { + return nil, nil, fmt.Errorf("failed to mount block CIMs in UVM: %w", err) + } + rcl.Add(mountedCIMs) + + // mount the CIM inside UVM now + log.G(ctx).WithField("volume", mountedCIMs.MountedVolumePath()).Debug("mounted blockCIM layers for hyperV isolated container") + + hostPath := filepath.Join(l.scratchLayerPath, "sandbox.vhdx") + + scsiMount, err := vm.SCSIManager.AddVirtualDisk(ctx, hostPath, false, vm.ID(), "", + &scsi.MountConfig{ + // TODO(ambarve): Add SCSI config to format the scratch in guest + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to add SCSI scratch VHD: %w", err) + } + containerScratchPathInUVM := scsiMount.GuestPath() + rcl.Add(scsiMount) + + log.G(ctx).WithFields(logrus.Fields{ + "hostPath": hostPath, + "uvmPath": containerScratchPathInUVM, + }).Debug("mounted scratch VHD") + + mountedCIMLayerID, err := cimlayer.LayerID(mountedCIMs.MountedVolumePath()) + if err != nil { + return nil, nil, fmt.Errorf("failed to get layer ID for mounted block CIM: %w", err) + } + + ml := &MountedWCOWLayers{ + RootFS: containerScratchPathInUVM, + MountedLayerPaths: []MountedWCOWLayer{ + { + LayerID: mountedCIMLayerID, + MountedPath: mountedCIMs.MountedVolumePath(), + }, + }, + } + + hcsLayers := []hcsschema.Layer{ + { + Id: mountedCIMLayerID, + Path: filepath.Join(mountedCIMs.MountedVolumePath(), "Files"), + }, + } + + err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.UnionFS, containerID) + if err != nil { + return nil, nil, err + } + log.G(ctx).Debug("hcsshim::mountHyperVIsolatedBlockCIMLayers Succeeded") + + return ml, &wcowIsolatedWCIFSLayerCloser{ + uvm: vm, + guestCombinedLayersPath: ml.RootFS, + scratchMount: scsiMount, + layerClosers: []resources.ResourceCloser{rcl}, + }, nil +} + // Mount the sandbox vhd to a user friendly path. func MountSandboxVolume(ctx context.Context, hostPath, volumeName string) (err error) { log.G(ctx).WithFields(logrus.Fields{ diff --git a/internal/layers/wcow_parse.go b/internal/layers/wcow_parse.go index 486bb1c0da..4700db75bf 100644 --- a/internal/layers/wcow_parse.go +++ b/internal/layers/wcow_parse.go @@ -217,33 +217,7 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers, } } -// GetWCOWUVMBootFilesFromLayers prepares the UVM boot files from the rootfs or layerFolders. -func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) { - var parentLayers []string - var scratchLayer string - var err error - - if err = validateRootfsAndLayers(rootfs, layerFolders); err != nil { - return nil, err - } - - if len(layerFolders) > 0 { - parentLayers = layerFolders[:len(layerFolders)-1] - scratchLayer = layerFolders[len(layerFolders)-1] - } else { - m := rootfs[0] - switch m.Type { - case legacyMountType: - parentLayers, err = getOptionAsArray(m, parentLayerPathsFlag) - if err != nil { - return nil, err - } - scratchLayer = m.Source - default: - return nil, fmt.Errorf("mount type '%s' is not supported for UVM boot", m.Type) - } - } - +func getVmbFSBootFiles(ctx context.Context, scratchLayer string, parentLayers []string) (*uvm.WCOWBootFiles, error) { uvmFolder, err := uvmfolder.LocateUVMFolder(ctx, parentLayers) if err != nil { return nil, fmt.Errorf("failed to locate utility VM folder from layer folders: %w", err) @@ -252,6 +226,11 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l // In order for the UVM sandbox.vhdx not to collide with the actual // nested Argon sandbox.vhdx we append the \vm folder to the last // entry in the list. + // TODO(ambarve): This probably is a bug, we should make a separate `vm` directory + // only if we are creating a standalone task. If we are starting a separate pod, + // containerd snapshotter already creates a separate directory for the pod. We + // won't have to worry about the name collision in that case. Keeping this for + // backward compatibility/avoid breaking existing code. scratchLayer = filepath.Join(scratchLayer, "vm") scratchVHDPath := filepath.Join(scratchLayer, "sandbox.vhdx") if err = os.MkdirAll(scratchLayer, 0777); err != nil { @@ -264,6 +243,7 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l return nil, err } } + return &uvm.WCOWBootFiles{ BootType: uvm.VmbFSBoot, VmbFSFiles: &uvm.VmbFSBootFiles{ @@ -273,3 +253,38 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l }, }, nil } + +func getBlockCIMBootFiles(ctx context.Context, wl *wcowBlockCIMLayers) (*uvm.WCOWBootFiles, error) { + scratchVHDPath := filepath.Join(wl.scratchLayerPath, "sandbox.vhdx") + + // block CIM based layers don't support multiple layers in the UVM, pick the base layer and continue + efiVHDPath := filepath.Join(filepath.Dir(wl.parentLayers[0].BlockPath), "boot.vhd") + + return &uvm.WCOWBootFiles{ + BootType: uvm.BlockCIMBoot, + BlockCIMFiles: &uvm.BlockCIMBootFiles{ + BootCIMVHDPath: wl.parentLayers[0].BlockPath, // This should be a block CIM with VHD footer attached. + EFIVHDPath: efiVHDPath, + ScratchVHDPath: scratchVHDPath, + }, + }, nil +} + +// GetWCOWUVMBootFilesFromLayers prepares the UVM boot files from the rootfs or layerFolders. +func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) { + var err error + + parsedWCOWLayers, err := ParseWCOWLayers(rootfs, layerFolders) + if err != nil { + return nil, err + } + + switch wl := parsedWCOWLayers.(type) { + case *wcowWCIFSLayers: + return getVmbFSBootFiles(ctx, wl.scratchLayerPath, wl.layerPaths) + case *wcowBlockCIMLayers: + return getBlockCIMBootFiles(ctx, wl) + default: + return nil, fmt.Errorf("unsupported layer format for UVM boot: %T", parsedWCOWLayers) + } +} diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index f0d9402e1d..03afc95e50 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -5,6 +5,7 @@ package oci import ( "context" "errors" + "fmt" "maps" "strconv" @@ -189,13 +190,13 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str } } -// handleSecurityPolicy handles parsing SecurityPolicy and NoSecurityHardware and setting -// implied options from the results. Both LCOW only, not WCOW. -func handleSecurityPolicy(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.SecurityPolicy = ParseAnnotationsString(a, annotations.SecurityPolicy, lopts.SecurityPolicy) +// handleLCOWSecurityPolicy handles parsing SecurityPolicy and NoSecurityHardware and setting +// implied options from the results for LCOW. +func handleLCOWSecurityPolicy(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { + lopts.SecurityPolicy = ParseAnnotationsString(a, annotations.LCOWSecurityPolicy, lopts.SecurityPolicy) // allow actual isolated boot etc to be ignored if we have no hardware. Required for dev // this is not a security issue as the attestation will fail without a genuine report - noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false) + noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.LCOWNoSecurityHardware, false) // if there is a security policy (and SNP) we currently boot in a way that doesn't support any boot options // this might change if the building of the vmgs file were to be done on demand but that is likely @@ -224,10 +225,38 @@ func handleSecurityPolicy(ctx context.Context, a map[string]string, lopts *uvm.O if len(lopts.SecurityPolicy) > 0 { // will only be false if explicitly set false by the annotation. We will otherwise default to true when there is a security policy - lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, a, annotations.EncryptedScratchDisk, true) + lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, a, annotations.LCOWEncryptedScratchDisk, true) } } +// handleWCOWSecurityPolicy handles parsing confidential pods related options and setting +// implied options from the results for WCOW. +func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) error { + wopts.SecurityPolicy = ParseAnnotationsString(a, annotations.WCOWSecurityPolicy, wopts.SecurityPolicy) + if len(wopts.SecurityPolicy) == 0 { + return nil + } + wopts.SecurityPolicyEnabled = true + + // overcommit isn't allowed when running in confidential mode and minimum of 2GB memory is required. + // We can change default values here, but if user provided specific values in annotations we should error out. + wopts.MemorySizeInMB = ParseAnnotationsUint64(ctx, a, annotations.MemorySizeInMB, 2048) + if wopts.MemorySizeInMB < 2048 { + return fmt.Errorf("minimum 2048MB of memory is required for confidential pods, got: %d", wopts.MemorySizeInMB) + } + + wopts.AllowOvercommit = ParseAnnotationsBool(ctx, a, annotations.AllowOvercommit, false) + if wopts.AllowOvercommit { + return fmt.Errorf("allow overcommit MUST be false for confidential pods") + } + + wopts.SecurityPolicyEnforcer = ParseAnnotationsString(a, annotations.WCOWSecurityPolicyEnforcer, wopts.SecurityPolicyEnforcer) + wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false) + wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) + wopts.IsolationType = ParseAnnotationsString(a, annotations.WCOWIsolationType, "") + return nil +} + func parseDevices(ctx context.Context, specWindows *specs.Windows) []uvm.VPCIDeviceID { if specWindows == nil || specWindows.Devices == nil { return nil @@ -310,10 +339,10 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( lopts.VPCIEnabled = ParseAnnotationsBool(ctx, s.Annotations, annotations.VPCIEnabled, lopts.VPCIEnabled) lopts.ExtraVSockPorts = ParseAnnotationCommaSeparatedUint32(ctx, s.Annotations, iannotations.ExtraVSockPorts, lopts.ExtraVSockPorts) handleAnnotationBootFilesPath(ctx, s.Annotations, lopts) - lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, s.Annotations, annotations.EncryptedScratchDisk, lopts.EnableScratchEncryption) - lopts.SecurityPolicy = ParseAnnotationsString(s.Annotations, annotations.SecurityPolicy, lopts.SecurityPolicy) - lopts.SecurityPolicyEnforcer = ParseAnnotationsString(s.Annotations, annotations.SecurityPolicyEnforcer, lopts.SecurityPolicyEnforcer) - lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.UVMReferenceInfoFile, lopts.UVMReferenceInfoFile) + lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, s.Annotations, annotations.LCOWEncryptedScratchDisk, lopts.EnableScratchEncryption) + lopts.SecurityPolicy = ParseAnnotationsString(s.Annotations, annotations.LCOWSecurityPolicy, lopts.SecurityPolicy) + lopts.SecurityPolicyEnforcer = ParseAnnotationsString(s.Annotations, annotations.LCOWSecurityPolicyEnforcer, lopts.SecurityPolicyEnforcer) + lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.LCOWReferenceInfoFile, lopts.UVMReferenceInfoFile) lopts.KernelBootOptions = ParseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService) lopts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, lopts.ConsolePipe) @@ -323,15 +352,15 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( // SecurityPolicy is very sensitive to other settings and will silently change those that are incompatible. // Eg VMPem device count, overridden kernel option cannot be respected. - handleSecurityPolicy(ctx, s.Annotations, lopts) + handleLCOWSecurityPolicy(ctx, s.Annotations, lopts) // override the default GuestState and DmVerityRootFs filenames if specified - lopts.GuestStateFile = ParseAnnotationsString(s.Annotations, annotations.GuestStateFile, lopts.GuestStateFile) + lopts.GuestStateFile = ParseAnnotationsString(s.Annotations, annotations.LCOWGuestStateFile, lopts.GuestStateFile) lopts.DmVerityRootFsVhd = ParseAnnotationsString(s.Annotations, annotations.DmVerityRootFsVhd, lopts.DmVerityRootFsVhd) lopts.DmVerityMode = ParseAnnotationsBool(ctx, s.Annotations, annotations.DmVerityMode, lopts.DmVerityMode) lopts.DmVerityCreateArgs = ParseAnnotationsString(s.Annotations, annotations.DmVerityCreateArgs, lopts.DmVerityCreateArgs) // Set HclEnabled if specified. Else default to a null pointer, which is omitted from the resulting JSON. - lopts.HclEnabled = ParseAnnotationsNullableBool(ctx, s.Annotations, annotations.HclEnabled) + lopts.HclEnabled = ParseAnnotationsNullableBool(ctx, s.Annotations, annotations.LCOWHclEnabled) // Add devices on the spec to the UVM's options lopts.AssignedDevices = parseDevices(ctx, s.Windows) @@ -346,6 +375,15 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( wopts.NoInheritHostTimezone = ParseAnnotationsBool(ctx, s.Annotations, annotations.NoInheritHostTimezone, wopts.NoInheritHostTimezone) wopts.AdditionalRegistryKeys = append(wopts.AdditionalRegistryKeys, parseAdditionalRegistryValues(ctx, s.Annotations)...) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) + + // Writable EFI is valid for both confidential and regular Hyper-V isolated WCOW. + wopts.WritableEFI = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWWritableEFI, wopts.WritableEFI) + + // Handle WCOW security policy settings + if err := handleWCOWSecurityPolicy(ctx, s.Annotations, wopts); err != nil { + return nil, err + } + return wopts, nil } return nil, errors.New("cannot create UVM opts spec is not LCOW or WCOW") diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 6b82c94de5..4a4ceed5c2 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -122,6 +122,156 @@ func Test_SpecToUVMCreateOptions_Default_WCOW(t *testing.T) { } } +func Test_SpecToUVMCreateOptions_WCOW_Confidential_Defaults(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if !wopts.SecurityPolicyEnabled { + t.Fatal("SecurityPolicyEnabled should be true when WCOWSecurityPolicy is set") + } + // Writable EFI should default to false unless explicitly enabled + if wopts.WritableEFI { + t.Fatal("WritableEFI should default to false when not specified") + } + if wopts.MemorySizeInMB != 2048 { + t.Fatalf("expected MemorySizeInMB to default to 2048, got %d", wopts.MemorySizeInMB) + } + if wopts.AllowOvercommit { + t.Fatal("AllowOvercommit must be false for confidential pods by default") + } + if wopts.DisableSecureBoot { + t.Fatal("DisableSecureBoot should default to false when not specified") + } + if wopts.SecurityPolicyEnforcer != "" { + t.Fatalf("expected empty SecurityPolicyEnforcer by default, got %q", wopts.SecurityPolicyEnforcer) + } + if wopts.GuestStateFilePath != uvm.GetDefaultConfidentialVMGSPath() { + t.Fatalf("expected GuestStateFilePath to default to %q, got %q", uvm.GetDefaultConfidentialVMGSPath(), wopts.GuestStateFilePath) + } + if wopts.IsolationType != "" { + t.Fatalf("expected empty IsolationType by default, got %q", wopts.IsolationType) + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_WritableEFI_Enabled(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.WCOWWritableEFI: "true", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if !wopts.WritableEFI { + t.Fatal("WritableEFI should be true when WCOWWritableEFI annotation is set to true") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_NonConfidential_WritableEFI_Enabled(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + // No WCOWSecurityPolicy means non-confidential path + annotations.WCOWWritableEFI: "true", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + // WritableEFI should be respected for non-confidential as well + if !wopts.WritableEFI { + t.Fatal("WritableEFI should be true for non-confidential when WCOWWritableEFI is set") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnLowMemory(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.MemorySizeInMB: "1024", + }, + } + + if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { + t.Fatal("expected error for confidential pods with MemorySizeInMB < 2048, got nil") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnAllowOvercommit(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.AllowOvercommit: "true", + }, + } + + if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { + t.Fatal("expected error for confidential pods when AllowOvercommit=true, got nil") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_Overrides(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.MemorySizeInMB: "4096", + annotations.AllowOvercommit: "false", + annotations.WCOWSecurityPolicyEnforcer: "rego", + annotations.WCOWDisableSecureBoot: "true", + annotations.WCOWGuestStateFile: "C:\\custom\\cwcow.vmgs", + annotations.WCOWIsolationType: "VirtualizationBasedSecurity", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if wopts.MemorySizeInMB != 4096 { + t.Fatalf("expected MemorySizeInMB=4096, got %d", wopts.MemorySizeInMB) + } + if wopts.AllowOvercommit { + t.Fatal("AllowOvercommit should be false when explicitly set to false") + } + if wopts.SecurityPolicyEnforcer != "rego" { + t.Fatalf("expected SecurityPolicyEnforcer=rego, got %q", wopts.SecurityPolicyEnforcer) + } + if !wopts.DisableSecureBoot { + t.Fatal("DisableSecureBoot should be true when specified") + } + if wopts.GuestStateFilePath != "C:\\custom\\cwcow.vmgs" { + t.Fatalf("expected GuestStateFilePath to match override, got %q", wopts.GuestStateFilePath) + } + if wopts.IsolationType != "VirtualizationBasedSecurity" { + t.Fatalf("expected IsolationType=VirtualizationBasedSecurity, got %q", wopts.IsolationType) + } +} + func Test_SpecToUVMCreateOptions_Common(t *testing.T) { cpugroupid := "1" lowmmiogap := 1024 diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index b848017c13..c19d54e89a 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -73,9 +73,10 @@ type LCOWCombinedLayers struct { } type WCOWCombinedLayers struct { - ContainerRootPath string `json:"ContainerRootPath,omitempty"` - Layers []hcsschema.Layer `json:"Layers,omitempty"` - ScratchPath string `json:"ScratchPath,omitempty"` + ContainerRootPath string `json:"ContainerRootPath,omitempty"` + Layers []hcsschema.Layer `json:"Layers,omitempty"` + ScratchPath string `json:"ScratchPath,omitempty"` + FilterType hcsschema.FileSystemFilterType `json:"FilterType,omitempty"` } type CWCOWCombinedLayers struct { @@ -113,11 +114,12 @@ type BlockCIMDevice struct { Lun int32 } -type WCOWBlockCIMMounts struct { +type CWCOWBlockCIMMounts struct { // BlockCIMs should be ordered from merged CIM followed by Layer n .. layer 1 - BlockCIMs []BlockCIMDevice `json:"BlockCIMs,omitempty"` - VolumeGUID guid.GUID `json:"VolumeGUID,omitempty"` - MountFlags uint32 `json:"MountFlags,omitempty"` + BlockCIMs []BlockCIMDevice `json:"BlockCIMs,omitempty"` + VolumeGUID guid.GUID `json:"VolumeGUID,omitempty"` + MountFlags uint32 `json:"MountFlags,omitempty"` + ContainerID string `json:"ContainerID,omitempty"` } type WCOWMappedVirtualDisk struct { diff --git a/internal/uvm/cimfs.go b/internal/uvm/cimfs.go new file mode 100644 index 0000000000..c1e8704883 --- /dev/null +++ b/internal/uvm/cimfs.go @@ -0,0 +1,167 @@ +//go:build windows +// +build windows + +package uvm + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + "github.com/Microsoft/hcsshim/internal/uvm/scsi" + "github.com/Microsoft/hcsshim/pkg/cimfs" + "github.com/sirupsen/logrus" +) + +type UVMMountedBlockCIMs struct { + scsiMounts []*scsi.Mount + // Volume GUID, inside the UVM this is the volume at which the CIMs are mounted + volumeGUID guid.GUID + // SCSI mounts are already ref counted, we only need to ref count the mounted merged CIMs + refCount uint32 + // UVM in which these mounts are done + host *UtilityVM + // reference key used for ref counting + refKey string +} + +func (umb *UVMMountedBlockCIMs) MountedVolumePath() string { + return fmt.Sprintf(cimfs.VolumePathFormat, umb.volumeGUID) +} + +func (umb *UVMMountedBlockCIMs) Release(ctx context.Context) error { + umb.host.blockCIMMountLock.Lock() + defer umb.host.blockCIMMountLock.Unlock() + + if umb.refCount > 1 { + umb.refCount-- + return nil + } + + guestReq := guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeWCOWBlockCims, + RequestType: guestrequest.RequestTypeRemove, + Settings: &guestresource.CWCOWBlockCIMMounts{ + VolumeGUID: umb.volumeGUID, + }, + } + if err := umb.host.GuestRequest(ctx, guestReq); err != nil { + return fmt.Errorf("failed to mount the cim: %w", err) + } + + for i := len(umb.scsiMounts) - 1; i >= 0; i-- { + if err := umb.scsiMounts[i].Release(ctx); err != nil { + return err + } + } + + // Remove from cache when ref count reaches zero + delete(umb.host.blockCIMMounts, umb.refKey) + return nil +} + +// mergedCIM can be nil, +// sourceCIMs MUST be in the top to bottom order +func (uvm *UtilityVM) MountBlockCIMs(ctx context.Context, mergedCIM *cimfs.BlockCIM, sourceCIMs []*cimfs.BlockCIM, containerID string) (_ *UVMMountedBlockCIMs, retErr error) { + if len(sourceCIMs) < 1 { + return nil, fmt.Errorf("at least 1 source CIM is required") + } + + uvm.blockCIMMountLock.Lock() + defer uvm.blockCIMMountLock.Unlock() + + layersToAttach := sourceCIMs + if mergedCIM != nil { + layersToAttach = append([]*cimfs.BlockCIM{mergedCIM}, sourceCIMs...) + } + + // The block path of the mergedCIM (or the block path of the sourceCIM when there + // is just 1 layer) is sufficient to uniquely identify a block CIM mount done in + // the UVM. We use that to check if we have already mounted this set of CIMs in + // the UVM. + if mountedCIMs, ok := uvm.blockCIMMounts[layersToAttach[0].BlockPath]; ok { + mountedCIMs.refCount++ + return mountedCIMs, nil + } + + volumeGUID, err := guid.NewV4() + if err != nil { + return nil, fmt.Errorf("generated cim mount GUID: %w", err) + } + + // TODO(ambarve): When inbox GCS adds support for mounting block CIMs, we should + // use the appropriate request type for confidential vs regular pods as inbox GCS + // may not understand the CWCOWBlockCIMMounts type. + settings := &guestresource.CWCOWBlockCIMMounts{ + BlockCIMs: []guestresource.BlockCIMDevice{}, + VolumeGUID: volumeGUID, + MountFlags: cimfs.CimMountBlockDeviceCim, + ContainerID: containerID, + } + + umb := &UVMMountedBlockCIMs{ + volumeGUID: volumeGUID, + scsiMounts: []*scsi.Mount{}, + refCount: 1, + host: uvm, + refKey: layersToAttach[0].BlockPath, + } + + // Cleanup function to release all SCSI mounts on error + defer func() { + if retErr != nil { + for _, mount := range umb.scsiMounts { + if cErr := mount.Release(ctx); cErr != nil { + log.G(ctx).WithFields(logrus.Fields{ + "mount": mount, + "original error": retErr, + }).Debugf("failure during SCSI mount cleanup: %s", cErr) + } + } + } + }() + + for _, bcim := range layersToAttach { + sm, err := uvm.SCSIManager.AddVirtualDisk(ctx, bcim.BlockPath, true, uvm.ID(), "", nil) + if err != nil { + return nil, fmt.Errorf("failed to attach block CIM %s: %w", bcim.BlockPath, err) + } + + hasher := sha256.New() + hasher.Write([]byte(bcim.BlockPath)) + layerDigest := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) + + log.G(ctx).WithFields(logrus.Fields{ + "block path": bcim.BlockPath, + "cim name": bcim.CimName, + "layer digest": layerDigest, + "scsi controller": sm.Controller(), + "scsi LUN": sm.LUN(), + }).Debugf("attached block CIM VHD") + + settings.BlockCIMs = append(settings.BlockCIMs, guestresource.BlockCIMDevice{ + CimName: bcim.CimName, + Lun: int32(sm.LUN()), + }) + umb.scsiMounts = append(umb.scsiMounts, sm) + } + + guestReq := guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeWCOWBlockCims, + RequestType: guestrequest.RequestTypeAdd, + Settings: settings, + } + if err := uvm.GuestRequest(ctx, guestReq); err != nil { + return nil, fmt.Errorf("failed to mount the cim: %w", err) + } + + // Add to cache for future reference counting + uvm.blockCIMMounts[layersToAttach[0].BlockPath] = umb + + return umb, nil +} diff --git a/internal/uvm/combine_layers.go b/internal/uvm/combine_layers.go index 468139c0f7..b577f706f3 100644 --- a/internal/uvm/combine_layers.go +++ b/internal/uvm/combine_layers.go @@ -14,21 +14,41 @@ import ( // container file system. // // Note: `layerPaths` and `containerRootPath` are paths from within the UVM. -func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcsschema.Layer, containerRootPath string) error { +func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcsschema.Layer, containerRootPath string, filterType hcsschema.FileSystemFilterType, containerID string) error { if uvm.operatingSystem != "windows" { return errNotSupported } - msr := &hcsschema.ModifySettingRequest{ - GuestRequest: guestrequest.ModificationRequest{ - ResourceType: guestresource.ResourceTypeCombinedLayers, - RequestType: guestrequest.RequestTypeAdd, - Settings: guestresource.WCOWCombinedLayers{ - ContainerRootPath: containerRootPath, - Layers: layerPaths, + + var modifyRequest *hcsschema.ModifySettingRequest + if uvm.HasConfidentialPolicy() { + modifyRequest = &hcsschema.ModifySettingRequest{ + GuestRequest: guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeCWCOWCombinedLayers, + RequestType: guestrequest.RequestTypeAdd, + Settings: guestresource.CWCOWCombinedLayers{ + ContainerID: containerID, + CombinedLayers: guestresource.WCOWCombinedLayers{ + ContainerRootPath: containerRootPath, + Layers: layerPaths, + FilterType: filterType, + }, + }, }, - }, + } + } else { + modifyRequest = &hcsschema.ModifySettingRequest{ + GuestRequest: guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeCombinedLayers, + RequestType: guestrequest.RequestTypeAdd, + Settings: guestresource.WCOWCombinedLayers{ + ContainerRootPath: containerRootPath, + Layers: layerPaths, + FilterType: filterType, + }, + }, + } } - return uvm.modify(ctx, msr) + return uvm.modify(ctx, modifyRequest) } // CombineLayersLCOW combines `layerPaths` and optionally `scratchPath` into an diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index b1dbfdd138..77c77f5e02 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -4,6 +4,7 @@ package uvm import ( "context" + "encoding/base64" "fmt" "maps" "os" @@ -27,17 +28,20 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm/scsi" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" ) type ConfidentialWCOWOptions struct { - GuestStateFilePath string // The vmgs file path - SecurityPolicyEnabled bool // Set when there is a security policy to apply on actual SNP hardware, use this rathen than checking the string length - SecurityPolicy string // Optional security policy + GuestStateFilePath string // The vmgs file path + SecurityPolicyEnabled bool // Set when there is a security policy to apply on actual SNP hardware, use this rathen than checking the string length + SecurityPolicy string // Optional security policy + SecurityPolicyEnforcer string // Set which security policy enforcer to use (open door or rego). This allows for better fallback mechanic. /* Below options are only included for testing/debugging purposes - shouldn't be used in regular scenarios */ IsolationType string DisableSecureBoot bool FirmwareParameters string + WritableEFI bool } // OptionsWCOW are the set of options passed to CreateWCOW() to create a utility vm. @@ -57,6 +61,22 @@ type OptionsWCOW struct { AdditionalRegistryKeys []hcsschema.RegistryValue } +func defaultConfidentialWCOWOSBootFilesPath() string { + return filepath.Join(filepath.Dir(os.Args[0]), "WindowsBootFiles", "confidential") +} + +func GetDefaultConfidentialVMGSPath() string { + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.vmgs") +} + +func GetDefaultConfidentialBootCIMPath() string { + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "rootfs.vhd") +} + +func GetDefaultConfidentialEFIPath() string { + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "efi.vhd") +} + // NewDefaultOptionsWCOW creates the default options for a bootable version of // WCOW. The caller `MUST` set the `BootFiles` on the returned value. // @@ -221,6 +241,9 @@ func prepareCommonConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWC ServiceTable: make(map[string]hcsschema.HvSocketServiceConfig), }, }, + VirtualSmb: &hcsschema.VirtualSmb{ + DirectFileMappingInMB: 1024, // Sensible default, but could be a tuning parameter somewhere + }, }, }, } @@ -315,12 +338,23 @@ func prepareSecurityConfigDoc(ctx context.Context, uvm *UtilityVM, opts *Options } } + policyDigest, err := securitypolicy.NewSecurityPolicyDigest(opts.SecurityPolicy) + if err != nil { + return nil, err + } + + // HCS API expect a base64 encoded string as LaunchData. Internally it + // decodes it to bytes. SEV later returns the decoded byte blob as HostData + // field of the report. + hostData := base64.StdEncoding.EncodeToString(policyDigest) + enableHCL := true doc.VirtualMachine.SecuritySettings = &hcsschema.SecuritySettings{ EnableTpm: false, Isolation: &hcsschema.IsolationSettings{ IsolationType: "SecureNestedPaging", HclEnabled: &enableHCL, + LaunchData: hostData, }, } @@ -351,26 +385,37 @@ func prepareSecurityConfigDoc(ctx context.Context, uvm *UtilityVM, opts *Options Minor: 0, } + // TODO(ambarve): only scratch VHD is unique per VM, EFI & Boot CIM VHDs are + // shared across UVMs, so we don't need to assign VM group access to them every + // time. It should have been done once while deploying the package. + if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.EFIVHDPath); err != nil { + return nil, errors.Wrap(err, "failed to grant vm access to EFI VHD") + } + if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.BootCIMVHDPath); err != nil { - return nil, errors.Wrap(err, "failed to grant vm access to boot CIM VHD") + return nil, errors.Wrap(err, "failed to grant vm access to Boot CIM VHD") } - if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.EFIVHDPath); err != nil { - return nil, errors.Wrap(err, "failed to grant vm access to EFI VHD") + if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.GuestStateFilePath); err != nil { + return nil, errors.Wrap(err, "failed to grant vm access to guest state file") } if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.ScratchVHDPath); err != nil { return nil, errors.Wrap(err, "failed to grant vm access to scratch VHD") } + // boot depends on scratch being attached at LUN 0, it MUST ALWAYS remain at LUN 0 doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["0"] = hcsschema.Attachment{ Path: opts.BootFiles.BlockCIMFiles.ScratchVHDPath, Type_: "VirtualDisk", } + doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["1"] = hcsschema.Attachment{ - Path: opts.BootFiles.BlockCIMFiles.EFIVHDPath, - Type_: "VirtualDisk", + Path: opts.BootFiles.BlockCIMFiles.EFIVHDPath, + Type_: "VirtualDisk", + ReadOnly: !opts.WritableEFI, } + doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["2"] = hcsschema.Attachment{ Path: opts.BootFiles.BlockCIMFiles.BootCIMVHDPath, Type_: "VirtualDisk", @@ -397,16 +442,11 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW) (* vsmbOpts := uvm.DefaultVSMBOptions(true) vsmbOpts.TakeBackupPrivilege = true - doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{ - DirectFileMappingInMB: 1024, // Sensible default, but could be a tuning parameter somewhere - Shares: []hcsschema.VirtualSmbShare{ - { - Name: "os", - Path: opts.BootFiles.VmbFSFiles.OSFilesPath, - Options: vsmbOpts, - }, - }, - } + doc.VirtualMachine.Devices.VirtualSmb.Shares = []hcsschema.VirtualSmbShare{{ + Name: "os", + Path: opts.BootFiles.VmbFSFiles.OSFilesPath, + Options: vsmbOpts, + }} doc.VirtualMachine.Chipset = &hcsschema.Chipset{ Uefi: &hcsschema.Uefi{ diff --git a/internal/uvm/security_policy.go b/internal/uvm/security_policy.go index 195b93d3ce..950179acc4 100644 --- a/internal/uvm/security_policy.go +++ b/internal/uvm/security_policy.go @@ -120,3 +120,14 @@ func (uvm *UtilityVM) InjectPolicyFragment(ctx context.Context, fragment *ctrdta } return uvm.modify(ctx, mod) } + +// returns if this instance of the UtilityVM is created with confidential policy +func (uvm *UtilityVM) HasConfidentialPolicy() bool { + switch opts := uvm.createOpts.(type) { + case *OptionsWCOW: + return opts.SecurityPolicyEnabled + case *OptionsLCOW: + return opts.SecurityPolicyEnabled + } + return false +} diff --git a/internal/uvm/types.go b/internal/uvm/types.go index 150b204999..7686e608f6 100644 --- a/internal/uvm/types.go +++ b/internal/uvm/types.go @@ -142,6 +142,10 @@ type UtilityVM struct { // LCOW only. Indicates whether to use policy based routing when configuring net interfaces in the guest. policyBasedRouting bool + + // ref counting for block CIMs + blockCIMMounts map[string]*UVMMountedBlockCIMs + blockCIMMountLock sync.Mutex } func (uvm *UtilityVM) ScratchEncryptionEnabled() bool { diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 31406fa1b9..abc0a2772d 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -121,40 +121,58 @@ const ( // Only applies in SNP mode. DmVerityRootFsVhd = "io.microsoft.virtualmachine.lcow.dmverity-rootfs-vhd" - // EncryptedScratchDisk indicates whether or not the container scratch disks + // LCOWEncryptedScratchDisk indicates whether or not the container scratch disks // should be encrypted or not. // // LCOW only. - EncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + LCOWEncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + // Deprecated: use [LCOWEncryptedScratchDisk] instead. + EncryptedScratchDisk = LCOWEncryptedScratchDisk - // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. - GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" + // LCOWGuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. + LCOWGuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" + // Deprecated: use [LCOWGuestStateFile] instead. + GuestStateFile = LCOWGuestStateFile - // HclEnabled specifies whether to enable the host compatibility layer. - HclEnabled = "io.microsoft.virtualmachine.lcow.hcl-enabled" + // LCOWHclEnabled specifies whether to enable the host compatibility layer. + LCOWHclEnabled = "io.microsoft.virtualmachine.lcow.hcl-enabled" + // Deprecated: use [LCOWHclEnabled] instead. + HclEnabled = LCOWHclEnabled - // HostAMDCertificate specifies the filename of the AMD certificates to be passed to UVM. + // LCOWHostAMDCertificate specifies the filename of the AMD certificates to be passed to UVM. // The certificate is expected to be located in the same directory as the shim executable. - HostAMDCertificate = "io.microsoft.virtualmachine.lcow.amd-certificate" + LCOWHostAMDCertificate = "io.microsoft.virtualmachine.lcow.amd-certificate" + // Deprecated: use [LCOWHostAMDCertificate] instead. + HostAMDCertificate = LCOWHostAMDCertificate - // NoSecurityHardware allows us, when it is set to true, to do testing and development without requiring SNP hardware. - NoSecurityHardware = "io.microsoft.virtualmachine.lcow.no_security_hardware" + // LCOWNoSecurityHardware allows us, when it is set to true, to do testing and development without requiring SNP hardware. + LCOWNoSecurityHardware = "io.microsoft.virtualmachine.lcow.no_security_hardware" + // Deprecated: use [LCOWNoSecurityHardware] instead. + NoSecurityHardware = LCOWNoSecurityHardware - // SecurityPolicy is used to specify a security policy for opengcs to enforce. - SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // LCOWSecurityPolicy is used to specify a security policy for opengcs to enforce. + LCOWSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // Deprecated: use [LCOWSecurityPolicy] instead. + SecurityPolicy = LCOWSecurityPolicy - // SecurityPolicyEnforcer is used to specify which enforcer to initialize (open-door, standard or rego). + // LCOWSecurityPolicyEnforcer is used to specify which enforcer to initialize (open-door, standard or rego). // This allows for better fallback mechanics. - SecurityPolicyEnforcer = "io.microsoft.virtualmachine.lcow.enforcer" + LCOWSecurityPolicyEnforcer = "io.microsoft.virtualmachine.lcow.enforcer" + // Deprecated: use [LCOWSecurityPolicyEnforcer] instead. + SecurityPolicyEnforcer = LCOWSecurityPolicyEnforcer - // UVMSecurityPolicyEnv specifies if confidential containers' related information + // LCOWSecurityPolicyEnv specifies if confidential containers' related information // should be written to containers' rootfs. The filenames and location are defined // by securitypolicy.PolicyFilename, securitypolicy.HostAMDCertFilename and // securitypolicy.ReferenceInfoFilename. - UVMSecurityPolicyEnv = "io.microsoft.virtualmachine.lcow.securitypolicy.env" - - // UVMReferenceInfoFile specifies the filename of a signed UVM reference file to be passed to UVM. - UVMReferenceInfoFile = "io.microsoft.virtualmachine.lcow.uvm-reference-info-file" + LCOWSecurityPolicyEnv = "io.microsoft.virtualmachine.lcow.securitypolicy.env" + // Deprecated: use [LCOWSecurityPolicyEnv] instead. + UVMSecurityPolicyEnv = LCOWSecurityPolicyEnv + + // LCOWReferenceInfoFile specifies the filename of a signed UVM reference file to be passed to UVM. + LCOWReferenceInfoFile = "io.microsoft.virtualmachine.lcow.uvm-reference-info-file" + // Deprecated: use [LCOWReferenceInfoFile] instead. + UVMReferenceInfoFile = LCOWReferenceInfoFile ) // WCOW container annotations. @@ -184,6 +202,33 @@ const ( WCOWProcessDumpCount = "io.microsoft.wcow.processdumpcount" ) +// WCOW confidential container related annotations +const ( + // WCOWGuestStateFile allows overriding the default VMGS path. + WCOWGuestStateFile = "io.microsoft.virtualmachine.wcow.gueststatefile" + + // WCOWSecurityPolicy is used to specify a security policy for WCOW containers to enforce. + WCOWSecurityPolicy = "io.microsoft.virtualmachine.wcow.securitypolicy" + + // WCOWSecurityPolicyEnforcer is used to specify which enforcer to + // initialize for WCOW (open-door, standard or rego). This allows for better + // fallback mechanics. + WCOWSecurityPolicyEnforcer = "io.microsoft.virtualmachine.wcow.enforcer" + + // WCOWIsolationType allows overriding isolation type of a confidential pod. + // Default is "SecureNestedPaging" and valid override values are + // "VirtualizationBasedSecurity" and "GuestStateOnly" + WCOWIsolationType = "io.microsoft.virtualmachine.wcow.isolation_type" + + // Allows disabling secure boot for testing and debugging scenarios, secure boot doesn't apply to confidential LCOW so + // this is a WCOW only config + WCOWDisableSecureBoot = "io.microsoft.virtualmachine.wcow.no_secure_boot" + + // Attaches the EFI/boot VHD in the writable mode (instead of the default read-only mode). This is usually required + // when debugging boot to capture bootstat traces. + WCOWWritableEFI = "io.microsoft.virtualmachine.wcow.writable_efi" +) + // WCOW host process container annotations. const ( // HostProcessInheritUser indicates whether to ignore the username passed in to run a host process diff --git a/pkg/cimfs/mount_cim.go b/pkg/cimfs/mount_cim.go index e9c305f604..12fa2da036 100644 --- a/pkg/cimfs/mount_cim.go +++ b/pkg/cimfs/mount_cim.go @@ -31,13 +31,17 @@ func (e *MountError) Error() string { return s } +const ( + VolumePathFormat = "\\\\?\\Volume{%s}\\" +) + // Mount mounts the given cim at a volume with given GUID. Returns the full volume // path if mount is successful. func Mount(cimPath string, volumeGUID guid.GUID, mountFlags uint32) (string, error) { if err := winapi.CimMountImage(filepath.Dir(cimPath), filepath.Base(cimPath), mountFlags, &volumeGUID); err != nil { return "", &MountError{Cim: cimPath, Op: "Mount", VolumeGUID: volumeGUID, Err: err} } - return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil + return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil } // Unmount unmounts the cim at mounted at path `volumePath`. @@ -116,7 +120,7 @@ func MountMergedBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM, mountFlag if err := winapi.CimMergeMountImage(uint32(len(cimsToMerge)), &cimsToMerge[0], mountFlags, &volumeGUID); err != nil { return "", &MountError{Cim: filepath.Join(mergedCIM.BlockPath, mergedCIM.CimName), Op: "MountMerged", Err: err} } - return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil + return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil } // Mounts a verified block CIM with the provided root hash. The root hash is usually