From d7d4e41c1ee8625e50435558fdf18fbd78b5c5d8 Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 27 Aug 2025 11:23:10 -0400 Subject: [PATCH 1/6] Support for starting confidential pods We already added support for starting confidential UVMs. This commit adds support for running confidential pods (via containerd) by including a non-empty security policy in the pod config annotations. The files (VHDs & guest statef file) required for starting such a confidential UVM MUST be available in a directory named `WindowsBootFiles/confidential` that is located next to the shim binary. Since some LCOW & WCOW annotations have conflicting/confusing names, existing LCOW annotation vars are renamed to have the "LCOW" prefix. Signed-off-by: Amit Barve --- cmd/containerd-shim-runhcs-v1/pod.go | 43 +++++++++-- internal/guest/runtime/hcsv2/uvm.go | 4 +- internal/layers/wcow_parse.go | 69 +++++++++++------- internal/oci/uvm.go | 61 ++++++++++++---- internal/oci/uvm_test.go | 105 +++++++++++++++++++++++++++ internal/uvm/create_wcow.go | 41 +++++++---- pkg/annotations/annotations.go | 79 +++++++++++++++----- 7 files changed, 322 insertions(+), 80 deletions(-) diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index 7811c9eabf..ccd9fbe2eb 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -18,6 +18,7 @@ import ( "github.com/Microsoft/hcsshim/pkg/annotations" eventstypes "github.com/containerd/containerd/api/events" task "github.com/containerd/containerd/api/runtime/task/v2" + "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/v2/core/runtime" "github.com/containerd/errdefs" "github.com/opencontainers/runtime-spec/specs-go" @@ -25,6 +26,41 @@ import ( "golang.org/x/sync/errgroup" ) +// initializeWCOWBootFiles handles the initialization of boot files for WCOW VMs +func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs []*types.Mount, s *specs.Spec) error { + var ( + layerFolders []string + err error + ) + + if s.Windows != nil { + layerFolders = s.Windows.LayerFolders + } + + wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, rootfs, layerFolders) + if err != nil { + return err + } + + if wopts.SecurityPolicyEnabled { + if wopts.BootFiles.BootType != uvm.BlockCIMBoot { + return fmt.Errorf("security policy (confidential mode) only works with block CIM based layers") + } + // we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs + wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath() + wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath() + } else if wopts.BootFiles.BootType == uvm.BlockCIMBoot { + // Supporting hyperv isolation with block CIM layers requires changes in + // the image pull path to prepare the EFI VHD. But more importantly, since both the + // container and UtilityVM files will be in the same CIM, the early boot + // code (bootmgr, winload etc.) will need to support reading the UVM OS + // files from `UtilityVM/Files` inside the CIM. Once that support is added + // we can enable this. + return fmt.Errorf("hyperv isolation is not supported with block CIM layers yet") + } + return nil +} + // shimPod represents the logical grouping of all tasks in a single set of // shared namespaces. The pod sandbox (container) is represented by the task // that matches the `shimPod.ID()` @@ -123,16 +159,11 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques return nil, err } case *uvm.OptionsWCOW: - var layerFolders []string - if s.Windows != nil { - layerFolders = s.Windows.LayerFolders - } wopts := (opts).(*uvm.OptionsWCOW) - wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders) + err = initializeWCOWBootFiles(ctx, wopts, req.Rootfs, s) if err != nil { return nil, err } - parent, err = uvm.CreateWCOW(ctx, wopts) if err != nil { return nil, err diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 6dddc6f41d..c0ebdc2e7b 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -456,9 +456,9 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM // containing the files is exposed via UVM_SECURITY_CONTEXT_DIR env var. // It may be an error to have a security policy but not expose it to the // container as in that case it can never be checked as correct by a verifier. - if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.UVMSecurityPolicyEnv, true) { + if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWSecurityPolicyEnv, true) { encodedPolicy := h.securityPolicyEnforcer.EncodedSecurityPolicy() - hostAMDCert := settings.OCISpecification.Annotations[annotations.HostAMDCertificate] + hostAMDCert := settings.OCISpecification.Annotations[annotations.LCOWHostAMDCertificate] if len(encodedPolicy) > 0 || len(hostAMDCert) > 0 || len(h.uvmReferenceInfo) > 0 { // Use os.MkdirTemp to make sure that the directory is unique. securityContextDir, err := os.MkdirTemp(settings.OCISpecification.Root.Path, securitypolicy.SecurityContextDirTemplate) diff --git a/internal/layers/wcow_parse.go b/internal/layers/wcow_parse.go index 486bb1c0da..4700db75bf 100644 --- a/internal/layers/wcow_parse.go +++ b/internal/layers/wcow_parse.go @@ -217,33 +217,7 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers, } } -// GetWCOWUVMBootFilesFromLayers prepares the UVM boot files from the rootfs or layerFolders. -func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) { - var parentLayers []string - var scratchLayer string - var err error - - if err = validateRootfsAndLayers(rootfs, layerFolders); err != nil { - return nil, err - } - - if len(layerFolders) > 0 { - parentLayers = layerFolders[:len(layerFolders)-1] - scratchLayer = layerFolders[len(layerFolders)-1] - } else { - m := rootfs[0] - switch m.Type { - case legacyMountType: - parentLayers, err = getOptionAsArray(m, parentLayerPathsFlag) - if err != nil { - return nil, err - } - scratchLayer = m.Source - default: - return nil, fmt.Errorf("mount type '%s' is not supported for UVM boot", m.Type) - } - } - +func getVmbFSBootFiles(ctx context.Context, scratchLayer string, parentLayers []string) (*uvm.WCOWBootFiles, error) { uvmFolder, err := uvmfolder.LocateUVMFolder(ctx, parentLayers) if err != nil { return nil, fmt.Errorf("failed to locate utility VM folder from layer folders: %w", err) @@ -252,6 +226,11 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l // In order for the UVM sandbox.vhdx not to collide with the actual // nested Argon sandbox.vhdx we append the \vm folder to the last // entry in the list. + // TODO(ambarve): This probably is a bug, we should make a separate `vm` directory + // only if we are creating a standalone task. If we are starting a separate pod, + // containerd snapshotter already creates a separate directory for the pod. We + // won't have to worry about the name collision in that case. Keeping this for + // backward compatibility/avoid breaking existing code. scratchLayer = filepath.Join(scratchLayer, "vm") scratchVHDPath := filepath.Join(scratchLayer, "sandbox.vhdx") if err = os.MkdirAll(scratchLayer, 0777); err != nil { @@ -264,6 +243,7 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l return nil, err } } + return &uvm.WCOWBootFiles{ BootType: uvm.VmbFSBoot, VmbFSFiles: &uvm.VmbFSBootFiles{ @@ -273,3 +253,38 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l }, }, nil } + +func getBlockCIMBootFiles(ctx context.Context, wl *wcowBlockCIMLayers) (*uvm.WCOWBootFiles, error) { + scratchVHDPath := filepath.Join(wl.scratchLayerPath, "sandbox.vhdx") + + // block CIM based layers don't support multiple layers in the UVM, pick the base layer and continue + efiVHDPath := filepath.Join(filepath.Dir(wl.parentLayers[0].BlockPath), "boot.vhd") + + return &uvm.WCOWBootFiles{ + BootType: uvm.BlockCIMBoot, + BlockCIMFiles: &uvm.BlockCIMBootFiles{ + BootCIMVHDPath: wl.parentLayers[0].BlockPath, // This should be a block CIM with VHD footer attached. + EFIVHDPath: efiVHDPath, + ScratchVHDPath: scratchVHDPath, + }, + }, nil +} + +// GetWCOWUVMBootFilesFromLayers prepares the UVM boot files from the rootfs or layerFolders. +func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) { + var err error + + parsedWCOWLayers, err := ParseWCOWLayers(rootfs, layerFolders) + if err != nil { + return nil, err + } + + switch wl := parsedWCOWLayers.(type) { + case *wcowWCIFSLayers: + return getVmbFSBootFiles(ctx, wl.scratchLayerPath, wl.layerPaths) + case *wcowBlockCIMLayers: + return getBlockCIMBootFiles(ctx, wl) + default: + return nil, fmt.Errorf("unsupported layer format for UVM boot: %T", parsedWCOWLayers) + } +} diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index f0d9402e1d..d1c1e75734 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -5,6 +5,7 @@ package oci import ( "context" "errors" + "fmt" "maps" "strconv" @@ -189,13 +190,13 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str } } -// handleSecurityPolicy handles parsing SecurityPolicy and NoSecurityHardware and setting -// implied options from the results. Both LCOW only, not WCOW. -func handleSecurityPolicy(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.SecurityPolicy = ParseAnnotationsString(a, annotations.SecurityPolicy, lopts.SecurityPolicy) +// handleLCOWSecurityPolicy handles parsing SecurityPolicy and NoSecurityHardware and setting +// implied options from the results for LCOW. +func handleLCOWSecurityPolicy(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { + lopts.SecurityPolicy = ParseAnnotationsString(a, annotations.LCOWSecurityPolicy, lopts.SecurityPolicy) // allow actual isolated boot etc to be ignored if we have no hardware. Required for dev // this is not a security issue as the attestation will fail without a genuine report - noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false) + noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.LCOWNoSecurityHardware, false) // if there is a security policy (and SNP) we currently boot in a way that doesn't support any boot options // this might change if the building of the vmgs file were to be done on demand but that is likely @@ -224,10 +225,38 @@ func handleSecurityPolicy(ctx context.Context, a map[string]string, lopts *uvm.O if len(lopts.SecurityPolicy) > 0 { // will only be false if explicitly set false by the annotation. We will otherwise default to true when there is a security policy - lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, a, annotations.EncryptedScratchDisk, true) + lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, a, annotations.LCOWEncryptedScratchDisk, true) } } +// handleWCOWSecurityPolicy handles parsing confidential pods related options and setting +// implied options from the results for WCOW. +func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) error { + wopts.SecurityPolicy = ParseAnnotationsString(a, annotations.WCOWSecurityPolicy, wopts.SecurityPolicy) + if len(wopts.SecurityPolicy) == 0 { + return nil + } + wopts.SecurityPolicyEnabled = true + + // overcommit isn't allowed when running in confidential mode and minimum of 2GB memory is required. + // We can change default values here, but if user provided specific values in annotations we should error out. + wopts.MemorySizeInMB = ParseAnnotationsUint64(ctx, a, annotations.MemorySizeInMB, 2048) + if wopts.MemorySizeInMB < 2048 { + return fmt.Errorf("minimum 2048MB of memory is required for confidential pods, got: %d", wopts.MemorySizeInMB) + } + + wopts.AllowOvercommit = ParseAnnotationsBool(ctx, a, annotations.AllowOvercommit, false) + if wopts.AllowOvercommit { + return fmt.Errorf("allow overcommit MUST be false for confidential pods") + } + + wopts.SecurityPolicyEnforcer = ParseAnnotationsString(a, annotations.WCOWSecurityPolicyEnforcer, wopts.SecurityPolicyEnforcer) + wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false) + wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) + wopts.IsolationType = ParseAnnotationsString(a, annotations.WCOWIsolationType, "") + return nil +} + func parseDevices(ctx context.Context, specWindows *specs.Windows) []uvm.VPCIDeviceID { if specWindows == nil || specWindows.Devices == nil { return nil @@ -310,10 +339,10 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( lopts.VPCIEnabled = ParseAnnotationsBool(ctx, s.Annotations, annotations.VPCIEnabled, lopts.VPCIEnabled) lopts.ExtraVSockPorts = ParseAnnotationCommaSeparatedUint32(ctx, s.Annotations, iannotations.ExtraVSockPorts, lopts.ExtraVSockPorts) handleAnnotationBootFilesPath(ctx, s.Annotations, lopts) - lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, s.Annotations, annotations.EncryptedScratchDisk, lopts.EnableScratchEncryption) - lopts.SecurityPolicy = ParseAnnotationsString(s.Annotations, annotations.SecurityPolicy, lopts.SecurityPolicy) - lopts.SecurityPolicyEnforcer = ParseAnnotationsString(s.Annotations, annotations.SecurityPolicyEnforcer, lopts.SecurityPolicyEnforcer) - lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.UVMReferenceInfoFile, lopts.UVMReferenceInfoFile) + lopts.EnableScratchEncryption = ParseAnnotationsBool(ctx, s.Annotations, annotations.LCOWEncryptedScratchDisk, lopts.EnableScratchEncryption) + lopts.SecurityPolicy = ParseAnnotationsString(s.Annotations, annotations.LCOWSecurityPolicy, lopts.SecurityPolicy) + lopts.SecurityPolicyEnforcer = ParseAnnotationsString(s.Annotations, annotations.LCOWSecurityPolicyEnforcer, lopts.SecurityPolicyEnforcer) + lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.LCOWReferenceInfoFile, lopts.UVMReferenceInfoFile) lopts.KernelBootOptions = ParseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService) lopts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, lopts.ConsolePipe) @@ -323,15 +352,15 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( // SecurityPolicy is very sensitive to other settings and will silently change those that are incompatible. // Eg VMPem device count, overridden kernel option cannot be respected. - handleSecurityPolicy(ctx, s.Annotations, lopts) + handleLCOWSecurityPolicy(ctx, s.Annotations, lopts) // override the default GuestState and DmVerityRootFs filenames if specified - lopts.GuestStateFile = ParseAnnotationsString(s.Annotations, annotations.GuestStateFile, lopts.GuestStateFile) + lopts.GuestStateFile = ParseAnnotationsString(s.Annotations, annotations.LCOWGuestStateFile, lopts.GuestStateFile) lopts.DmVerityRootFsVhd = ParseAnnotationsString(s.Annotations, annotations.DmVerityRootFsVhd, lopts.DmVerityRootFsVhd) lopts.DmVerityMode = ParseAnnotationsBool(ctx, s.Annotations, annotations.DmVerityMode, lopts.DmVerityMode) lopts.DmVerityCreateArgs = ParseAnnotationsString(s.Annotations, annotations.DmVerityCreateArgs, lopts.DmVerityCreateArgs) // Set HclEnabled if specified. Else default to a null pointer, which is omitted from the resulting JSON. - lopts.HclEnabled = ParseAnnotationsNullableBool(ctx, s.Annotations, annotations.HclEnabled) + lopts.HclEnabled = ParseAnnotationsNullableBool(ctx, s.Annotations, annotations.LCOWHclEnabled) // Add devices on the spec to the UVM's options lopts.AssignedDevices = parseDevices(ctx, s.Windows) @@ -346,6 +375,12 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( wopts.NoInheritHostTimezone = ParseAnnotationsBool(ctx, s.Annotations, annotations.NoInheritHostTimezone, wopts.NoInheritHostTimezone) wopts.AdditionalRegistryKeys = append(wopts.AdditionalRegistryKeys, parseAdditionalRegistryValues(ctx, s.Annotations)...) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) + + // Handle WCOW security policy settings + if err := handleWCOWSecurityPolicy(ctx, s.Annotations, wopts); err != nil { + return nil, err + } + return wopts, nil } return nil, errors.New("cannot create UVM opts spec is not LCOW or WCOW") diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 6b82c94de5..188507ae75 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -122,6 +122,111 @@ func Test_SpecToUVMCreateOptions_Default_WCOW(t *testing.T) { } } +func Test_SpecToUVMCreateOptions_WCOW_Confidential_Defaults(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if !wopts.SecurityPolicyEnabled { + t.Fatal("SecurityPolicyEnabled should be true when WCOWSecurityPolicy is set") + } + if wopts.MemorySizeInMB != 2048 { + t.Fatalf("expected MemorySizeInMB to default to 2048, got %d", wopts.MemorySizeInMB) + } + if wopts.AllowOvercommit { + t.Fatal("AllowOvercommit must be false for confidential pods by default") + } + if wopts.DisableSecureBoot { + t.Fatal("DisableSecureBoot should default to false when not specified") + } + if wopts.SecurityPolicyEnforcer != "" { + t.Fatalf("expected empty SecurityPolicyEnforcer by default, got %q", wopts.SecurityPolicyEnforcer) + } + if wopts.GuestStateFilePath != uvm.GetDefaultConfidentialVMGSPath() { + t.Fatalf("expected GuestStateFilePath to default to %q, got %q", uvm.GetDefaultConfidentialVMGSPath(), wopts.GuestStateFilePath) + } + if wopts.IsolationType != "" { + t.Fatalf("expected empty IsolationType by default, got %q", wopts.IsolationType) + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnLowMemory(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.MemorySizeInMB: "1024", + }, + } + + if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { + t.Fatal("expected error for confidential pods with MemorySizeInMB < 2048, got nil") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnAllowOvercommit(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.AllowOvercommit: "true", + }, + } + + if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { + t.Fatal("expected error for confidential pods when AllowOvercommit=true, got nil") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_Overrides(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.MemorySizeInMB: "4096", + annotations.AllowOvercommit: "false", + annotations.WCOWSecurityPolicyEnforcer: "rego", + annotations.WCOWDisableSecureBoot: "true", + annotations.WCOWGuestStateFile: "C:\\custom\\cwcow.vmgs", + annotations.WCOWIsolationType: "VirtualizationBasedSecurity", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if wopts.MemorySizeInMB != 4096 { + t.Fatalf("expected MemorySizeInMB=4096, got %d", wopts.MemorySizeInMB) + } + if wopts.AllowOvercommit { + t.Fatal("AllowOvercommit should be false when explicitly set to false") + } + if wopts.SecurityPolicyEnforcer != "rego" { + t.Fatalf("expected SecurityPolicyEnforcer=rego, got %q", wopts.SecurityPolicyEnforcer) + } + if !wopts.DisableSecureBoot { + t.Fatal("DisableSecureBoot should be true when specified") + } + if wopts.GuestStateFilePath != "C:\\custom\\cwcow.vmgs" { + t.Fatalf("expected GuestStateFilePath to match override, got %q", wopts.GuestStateFilePath) + } + if wopts.IsolationType != "VirtualizationBasedSecurity" { + t.Fatalf("expected IsolationType=VirtualizationBasedSecurity, got %q", wopts.IsolationType) + } +} + func Test_SpecToUVMCreateOptions_Common(t *testing.T) { cpugroupid := "1" lowmmiogap := 1024 diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index b1dbfdd138..4069ae00b0 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -30,9 +30,10 @@ import ( ) type ConfidentialWCOWOptions struct { - GuestStateFilePath string // The vmgs file path - SecurityPolicyEnabled bool // Set when there is a security policy to apply on actual SNP hardware, use this rathen than checking the string length - SecurityPolicy string // Optional security policy + GuestStateFilePath string // The vmgs file path + SecurityPolicyEnabled bool // Set when there is a security policy to apply on actual SNP hardware, use this rathen than checking the string length + SecurityPolicy string // Optional security policy + SecurityPolicyEnforcer string // Set which security policy enforcer to use (open door or rego). This allows for better fallback mechanic. /* Below options are only included for testing/debugging purposes - shouldn't be used in regular scenarios */ IsolationType string @@ -57,6 +58,22 @@ type OptionsWCOW struct { AdditionalRegistryKeys []hcsschema.RegistryValue } +func defaultConfidentialWCOWOSBootFilesPath() string { + return filepath.Join(filepath.Dir(os.Args[0]), "WindowsBootFiles", "confidential") +} + +func GetDefaultConfidentialVMGSPath() string { + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.vmgs") +} + +func GetDefaultConfidentialBootCIMPath() string { + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "rootfs.vhd") +} + +func GetDefaultConfidentialEFIPath() string { + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "efi.vhd") +} + // NewDefaultOptionsWCOW creates the default options for a bootable version of // WCOW. The caller `MUST` set the `BootFiles` on the returned value. // @@ -221,6 +238,9 @@ func prepareCommonConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWC ServiceTable: make(map[string]hcsschema.HvSocketServiceConfig), }, }, + VirtualSmb: &hcsschema.VirtualSmb{ + DirectFileMappingInMB: 1024, // Sensible default, but could be a tuning parameter somewhere + }, }, }, } @@ -397,16 +417,11 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW) (* vsmbOpts := uvm.DefaultVSMBOptions(true) vsmbOpts.TakeBackupPrivilege = true - doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{ - DirectFileMappingInMB: 1024, // Sensible default, but could be a tuning parameter somewhere - Shares: []hcsschema.VirtualSmbShare{ - { - Name: "os", - Path: opts.BootFiles.VmbFSFiles.OSFilesPath, - Options: vsmbOpts, - }, - }, - } + doc.VirtualMachine.Devices.VirtualSmb.Shares = []hcsschema.VirtualSmbShare{{ + Name: "os", + Path: opts.BootFiles.VmbFSFiles.OSFilesPath, + Options: vsmbOpts, + }} doc.VirtualMachine.Chipset = &hcsschema.Chipset{ Uefi: &hcsschema.Uefi{ diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 31406fa1b9..c4ebd4f2b8 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -121,40 +121,58 @@ const ( // Only applies in SNP mode. DmVerityRootFsVhd = "io.microsoft.virtualmachine.lcow.dmverity-rootfs-vhd" - // EncryptedScratchDisk indicates whether or not the container scratch disks + // LCOWEncryptedScratchDisk indicates whether or not the container scratch disks // should be encrypted or not. // // LCOW only. - EncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + LCOWEncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + // Deprecated: use [LCOWEncryptedScratchDisk] instead. + EncryptedScratchDisk = LCOWEncryptedScratchDisk - // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. - GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" + // LCOWGuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. + LCOWGuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" + // Deprecated: use [LCOWGuestStateFile] instead. + GuestStateFile = LCOWGuestStateFile - // HclEnabled specifies whether to enable the host compatibility layer. - HclEnabled = "io.microsoft.virtualmachine.lcow.hcl-enabled" + // LCOWHclEnabled specifies whether to enable the host compatibility layer. + LCOWHclEnabled = "io.microsoft.virtualmachine.lcow.hcl-enabled" + // Deprecated: use [LCOWHclEnabled] instead. + HclEnabled = LCOWHclEnabled - // HostAMDCertificate specifies the filename of the AMD certificates to be passed to UVM. + // LCOWHostAMDCertificate specifies the filename of the AMD certificates to be passed to UVM. // The certificate is expected to be located in the same directory as the shim executable. - HostAMDCertificate = "io.microsoft.virtualmachine.lcow.amd-certificate" + LCOWHostAMDCertificate = "io.microsoft.virtualmachine.lcow.amd-certificate" + // Deprecated: use [LCOWHostAMDCertificate] instead. + HostAMDCertificate = LCOWHostAMDCertificate - // NoSecurityHardware allows us, when it is set to true, to do testing and development without requiring SNP hardware. - NoSecurityHardware = "io.microsoft.virtualmachine.lcow.no_security_hardware" + // LCOWNoSecurityHardware allows us, when it is set to true, to do testing and development without requiring SNP hardware. + LCOWNoSecurityHardware = "io.microsoft.virtualmachine.lcow.no_security_hardware" + // Deprecated: use [LCOWNoSecurityHardware] instead. + NoSecurityHardware = LCOWNoSecurityHardware - // SecurityPolicy is used to specify a security policy for opengcs to enforce. - SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // LCOWSecurityPolicy is used to specify a security policy for opengcs to enforce. + LCOWSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // Deprecated: use [LCOWSecurityPolicy] instead. + SecurityPolicy = LCOWSecurityPolicy - // SecurityPolicyEnforcer is used to specify which enforcer to initialize (open-door, standard or rego). + // LCOWSecurityPolicyEnforcer is used to specify which enforcer to initialize (open-door, standard or rego). // This allows for better fallback mechanics. - SecurityPolicyEnforcer = "io.microsoft.virtualmachine.lcow.enforcer" + LCOWSecurityPolicyEnforcer = "io.microsoft.virtualmachine.lcow.enforcer" + // Deprecated: use [LCOWSecurityPolicyEnforcer] instead. + SecurityPolicyEnforcer = LCOWSecurityPolicyEnforcer - // UVMSecurityPolicyEnv specifies if confidential containers' related information + // LCOWSecurityPolicyEnv specifies if confidential containers' related information // should be written to containers' rootfs. The filenames and location are defined // by securitypolicy.PolicyFilename, securitypolicy.HostAMDCertFilename and // securitypolicy.ReferenceInfoFilename. - UVMSecurityPolicyEnv = "io.microsoft.virtualmachine.lcow.securitypolicy.env" - - // UVMReferenceInfoFile specifies the filename of a signed UVM reference file to be passed to UVM. - UVMReferenceInfoFile = "io.microsoft.virtualmachine.lcow.uvm-reference-info-file" + LCOWSecurityPolicyEnv = "io.microsoft.virtualmachine.lcow.securitypolicy.env" + // Deprecated: use [LCOWSecurityPolicyEnv] instead. + UVMSecurityPolicyEnv = LCOWSecurityPolicyEnv + + // LCOWReferenceInfoFile specifies the filename of a signed UVM reference file to be passed to UVM. + LCOWReferenceInfoFile = "io.microsoft.virtualmachine.lcow.uvm-reference-info-file" + // Deprecated: use [LCOWReferenceInfoFile] instead. + UVMReferenceInfoFile = LCOWReferenceInfoFile ) // WCOW container annotations. @@ -184,6 +202,29 @@ const ( WCOWProcessDumpCount = "io.microsoft.wcow.processdumpcount" ) +// WCOW confidential container related annotations +const ( + // WCOWGuestStateFile allows overriding the default VMGS path. + WCOWGuestStateFile = "io.microsoft.virtualmachine.wcow.gueststatefile" + + // WCOWSecurityPolicy is used to specify a security policy for WCOW containers to enforce. + WCOWSecurityPolicy = "io.microsoft.virtualmachine.wcow.securitypolicy" + + // WCOWSecurityPolicyEnforcer is used to specify which enforcer to + // initialize for WCOW (open-door, standard or rego). This allows for better + // fallback mechanics. + WCOWSecurityPolicyEnforcer = "io.microsoft.virtualmachine.wcow.enforcer" + + // WCOWIsolationType allows overriding isolation type of a confidential pod. + // Default is "SecureNestedPaging" and valid override values are + // "VirtualizationBasedSecurity" and "GuestStateOnly" + WCOWIsolationType = "io.microsoft.virtualmachine.wcow.isolation_type" + + // Allows disabling secure boot for testing and debugging scenarios, secure boot doesn't apply to confidential LCOW so + // this is a WCOW only config + WCOWDisableSecureBoot = "io.microsoft.virtualmachine.wcow.no_secure_boot" +) + // WCOW host process container annotations. const ( // HostProcessInheritUser indicates whether to ignore the username passed in to run a host process From 304f2cea19d0b446993874fb884748ca4b0198c1 Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 27 Aug 2025 11:24:12 -0400 Subject: [PATCH 2/6] Support for starting confidential containers Adds support for mounting block CIM based container image layers inside the UVM. (ResourceTypeWCOWBlockCims is only supported by sidecar GCS at the moment. Inbox GCS will add this support soon.) Signed-off-by: Amit Barve --- internal/layers/wcow_mount.go | 89 +++++++++- internal/protocol/guestresource/resources.go | 7 +- internal/uvm/cimfs.go | 163 +++++++++++++++++++ internal/uvm/combine_layers.go | 3 +- internal/uvm/types.go | 4 + pkg/cimfs/mount_cim.go | 8 +- 6 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 internal/uvm/cimfs.go diff --git a/internal/layers/wcow_mount.go b/internal/layers/wcow_mount.go index 9df9f199eb..f20217459e 100644 --- a/internal/layers/wcow_mount.go +++ b/internal/layers/wcow_mount.go @@ -43,7 +43,7 @@ func MountWCOWLayers(ctx context.Context, containerID string, vm *uvm.UtilityVM, if vm == nil { return mountProcessIsolatedBlockCIMLayers(ctx, containerID, l) } - return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with block cim layers") + return mountHypervIsolatedBlockCIMLayers(ctx, l, vm, containerID) default: return nil, nil, fmt.Errorf("invalid layer type %T", wl) } @@ -440,7 +440,7 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm }) } - err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS) + err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.WCIFS) if err != nil { return nil, nil, err } @@ -454,6 +454,91 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm }, nil } +func mountHypervIsolatedBlockCIMLayers(ctx context.Context, l *wcowBlockCIMLayers, vm *uvm.UtilityVM, containerID string) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) { + ctx, span := oc.StartSpan(ctx, "mountHyperVIsolatedBlockCIMLayers") + defer func() { + oc.SetSpanStatus(span, err) + span.End() + }() + + rcl := &resources.ResourceCloserList{} + defer func() { + if err != nil { + if rErr := rcl.Release(ctx); rErr != nil { + log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr) + } + } + }() + + log.G(ctx).WithFields(logrus.Fields{ + "scratch": l.scratchLayerPath, + "merged layer": l.mergedLayer, + "parent layers": l.parentLayers, + }).Debug("mounting hyperv isolated block CIM layers") + + mountedCIMs, err := vm.MountBlockCIMs(ctx, l.mergedLayer, l.parentLayers) + if err != nil { + return nil, nil, fmt.Errorf("failed to mount block CIMs in UVM: %w", err) + } + rcl.Add(mountedCIMs) + + // mount the CIM inside UVM now + log.G(ctx).WithField("volume", mountedCIMs.MountedVolumePath()).Debug("mounted blockCIM layers for hyperV isolated container") + + hostPath := filepath.Join(l.scratchLayerPath, "sandbox.vhdx") + + scsiMount, err := vm.SCSIManager.AddVirtualDisk(ctx, hostPath, false, vm.ID(), "", + &scsi.MountConfig{ + // TODO(ambarve): Add SCSI config to format the scratch in guest + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to add SCSI scratch VHD: %w", err) + } + containerScratchPathInUVM := scsiMount.GuestPath() + rcl.Add(scsiMount) + + log.G(ctx).WithFields(logrus.Fields{ + "hostPath": hostPath, + "uvmPath": containerScratchPathInUVM, + }).Debug("mounted scratch VHD") + + mountedCIMLayerID, err := cimlayer.LayerID(mountedCIMs.MountedVolumePath()) + if err != nil { + return nil, nil, fmt.Errorf("failed to get layer ID for mounted block CIM: %w", err) + } + + ml := &MountedWCOWLayers{ + RootFS: containerScratchPathInUVM, + MountedLayerPaths: []MountedWCOWLayer{ + { + LayerID: mountedCIMLayerID, + MountedPath: mountedCIMs.MountedVolumePath(), + }, + }, + } + + hcsLayers := []hcsschema.Layer{ + { + Id: mountedCIMLayerID, + Path: filepath.Join(mountedCIMs.MountedVolumePath(), "Files"), + }, + } + + // TODO(ambarve): Do we need CWCOW specific request type here? + err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.UnionFS) + if err != nil { + return nil, nil, err + } + log.G(ctx).Debug("hcsshim::mountHyperVIsolatedBlockCIMLayers Succeeded") + + return ml, &wcowIsolatedWCIFSLayerCloser{ + uvm: vm, + guestCombinedLayersPath: ml.RootFS, + scratchMount: scsiMount, + layerClosers: []resources.ResourceCloser{rcl}, + }, nil +} + // Mount the sandbox vhd to a user friendly path. func MountSandboxVolume(ctx context.Context, hostPath, volumeName string) (err error) { log.G(ctx).WithFields(logrus.Fields{ diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index b848017c13..7508668980 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -73,9 +73,10 @@ type LCOWCombinedLayers struct { } type WCOWCombinedLayers struct { - ContainerRootPath string `json:"ContainerRootPath,omitempty"` - Layers []hcsschema.Layer `json:"Layers,omitempty"` - ScratchPath string `json:"ScratchPath,omitempty"` + ContainerRootPath string `json:"ContainerRootPath,omitempty"` + Layers []hcsschema.Layer `json:"Layers,omitempty"` + ScratchPath string `json:"ScratchPath,omitempty"` + FilterType hcsschema.FileSystemFilterType `json:"FilterType,omitempty"` } type CWCOWCombinedLayers struct { diff --git a/internal/uvm/cimfs.go b/internal/uvm/cimfs.go new file mode 100644 index 0000000000..92cd45b8d7 --- /dev/null +++ b/internal/uvm/cimfs.go @@ -0,0 +1,163 @@ +//go:build windows +// +build windows + +package uvm + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + "github.com/Microsoft/hcsshim/internal/uvm/scsi" + "github.com/Microsoft/hcsshim/pkg/cimfs" + "github.com/sirupsen/logrus" +) + +type UVMMountedBlockCIMs struct { + scsiMounts []*scsi.Mount + // Volume GUID, inside the UVM this is the volume at which the CIMs are mounted + volumeGUID guid.GUID + // SCSI mounts are already ref counted, we only need to ref count the mounted merged CIMs + refCount uint32 + // UVM in which these mounts are done + host *UtilityVM + // reference key used for ref counting + refKey string +} + +func (umb *UVMMountedBlockCIMs) MountedVolumePath() string { + return fmt.Sprintf(cimfs.VolumePathFormat, umb.volumeGUID) +} + +func (umb *UVMMountedBlockCIMs) Release(ctx context.Context) error { + umb.host.blockCIMMountLock.Lock() + defer umb.host.blockCIMMountLock.Unlock() + + if umb.refCount > 1 { + umb.refCount-- + return nil + } + + guestReq := guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeWCOWBlockCims, + RequestType: guestrequest.RequestTypeRemove, + Settings: &guestresource.WCOWBlockCIMMounts{ + VolumeGUID: umb.volumeGUID, + }, + } + if err := umb.host.GuestRequest(ctx, guestReq); err != nil { + return fmt.Errorf("failed to mount the cim: %w", err) + } + + for i := len(umb.scsiMounts) - 1; i >= 0; i-- { + if err := umb.scsiMounts[i].Release(ctx); err != nil { + return err + } + } + + // Remove from cache when ref count reaches zero + delete(umb.host.blockCIMMounts, umb.refKey) + return nil +} + +// mergedCIM can be nil, +// sourceCIMs MUST be in the top to bottom order +func (uvm *UtilityVM) MountBlockCIMs(ctx context.Context, mergedCIM *cimfs.BlockCIM, sourceCIMs []*cimfs.BlockCIM) (_ *UVMMountedBlockCIMs, retErr error) { + if len(sourceCIMs) < 1 { + return nil, fmt.Errorf("at least 1 source CIM is required") + } + + uvm.blockCIMMountLock.Lock() + defer uvm.blockCIMMountLock.Unlock() + + layersToAttach := sourceCIMs + if mergedCIM != nil { + layersToAttach = append([]*cimfs.BlockCIM{mergedCIM}, sourceCIMs...) + } + + // The block path of the mergedCIM (or the block path of the sourceCIM when there + // is just 1 layer) is sufficient to uniquely identify a block CIM mount done in + // the UVM. We use that to check if we have already mounted this set of CIMs in + // the UVM. + if mountedCIMs, ok := uvm.blockCIMMounts[layersToAttach[0].BlockPath]; ok { + mountedCIMs.refCount++ + return mountedCIMs, nil + } + + volumeGUID, err := guid.NewV4() + if err != nil { + return nil, fmt.Errorf("generated cim mount GUID: %w", err) + } + + settings := &guestresource.WCOWBlockCIMMounts{ + BlockCIMs: []guestresource.BlockCIMDevice{}, + VolumeGUID: volumeGUID, + MountFlags: cimfs.CimMountBlockDeviceCim, + } + + umb := &UVMMountedBlockCIMs{ + volumeGUID: volumeGUID, + scsiMounts: []*scsi.Mount{}, + refCount: 1, + host: uvm, + refKey: layersToAttach[0].BlockPath, + } + + // Cleanup function to release all SCSI mounts on error + defer func() { + if retErr != nil { + for _, mount := range umb.scsiMounts { + if cErr := mount.Release(ctx); cErr != nil { + log.G(ctx).WithFields(logrus.Fields{ + "mount": mount, + "original error": retErr, + }).Debugf("failure during SCSI mount cleanup: %s", cErr) + } + } + } + }() + + for _, bcim := range layersToAttach { + sm, err := uvm.SCSIManager.AddVirtualDisk(ctx, bcim.BlockPath, true, uvm.ID(), "", nil) + if err != nil { + return nil, fmt.Errorf("failed to attach block CIM %s: %w", bcim.BlockPath, err) + } + + hasher := sha256.New() + hasher.Write([]byte(bcim.BlockPath)) + layerDigest := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) + + log.G(ctx).WithFields(logrus.Fields{ + "block path": bcim.BlockPath, + "cim name": bcim.CimName, + "layer digest": layerDigest, + "scsi controller": sm.Controller(), + "scsi LUN": sm.LUN(), + }).Debugf("attached block CIM VHD") + + settings.BlockCIMs = append(settings.BlockCIMs, guestresource.BlockCIMDevice{ + CimName: bcim.CimName, + Lun: int32(sm.LUN()), + }) + umb.scsiMounts = append(umb.scsiMounts, sm) + } + + guestReq := guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeWCOWBlockCims, + RequestType: guestrequest.RequestTypeAdd, + Settings: settings, + } + if err := uvm.GuestRequest(ctx, guestReq); err != nil { + return nil, fmt.Errorf("failed to mount the cim: %w", err) + } + + // Add to cache for future reference counting + uvm.blockCIMMounts[layersToAttach[0].BlockPath] = umb + + return umb, nil +} diff --git a/internal/uvm/combine_layers.go b/internal/uvm/combine_layers.go index 468139c0f7..c3b79622f4 100644 --- a/internal/uvm/combine_layers.go +++ b/internal/uvm/combine_layers.go @@ -14,7 +14,7 @@ import ( // container file system. // // Note: `layerPaths` and `containerRootPath` are paths from within the UVM. -func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcsschema.Layer, containerRootPath string) error { +func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcsschema.Layer, containerRootPath string, filterType hcsschema.FileSystemFilterType) error { if uvm.operatingSystem != "windows" { return errNotSupported } @@ -25,6 +25,7 @@ func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcssch Settings: guestresource.WCOWCombinedLayers{ ContainerRootPath: containerRootPath, Layers: layerPaths, + FilterType: filterType, }, }, } diff --git a/internal/uvm/types.go b/internal/uvm/types.go index 150b204999..7686e608f6 100644 --- a/internal/uvm/types.go +++ b/internal/uvm/types.go @@ -142,6 +142,10 @@ type UtilityVM struct { // LCOW only. Indicates whether to use policy based routing when configuring net interfaces in the guest. policyBasedRouting bool + + // ref counting for block CIMs + blockCIMMounts map[string]*UVMMountedBlockCIMs + blockCIMMountLock sync.Mutex } func (uvm *UtilityVM) ScratchEncryptionEnabled() bool { diff --git a/pkg/cimfs/mount_cim.go b/pkg/cimfs/mount_cim.go index e9c305f604..12fa2da036 100644 --- a/pkg/cimfs/mount_cim.go +++ b/pkg/cimfs/mount_cim.go @@ -31,13 +31,17 @@ func (e *MountError) Error() string { return s } +const ( + VolumePathFormat = "\\\\?\\Volume{%s}\\" +) + // Mount mounts the given cim at a volume with given GUID. Returns the full volume // path if mount is successful. func Mount(cimPath string, volumeGUID guid.GUID, mountFlags uint32) (string, error) { if err := winapi.CimMountImage(filepath.Dir(cimPath), filepath.Base(cimPath), mountFlags, &volumeGUID); err != nil { return "", &MountError{Cim: cimPath, Op: "Mount", VolumeGUID: volumeGUID, Err: err} } - return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil + return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil } // Unmount unmounts the cim at mounted at path `volumePath`. @@ -116,7 +120,7 @@ func MountMergedBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM, mountFlag if err := winapi.CimMergeMountImage(uint32(len(cimsToMerge)), &cimsToMerge[0], mountFlags, &volumeGUID); err != nil { return "", &MountError{Cim: filepath.Join(mergedCIM.BlockPath, mergedCIM.CimName), Op: "MountMerged", Err: err} } - return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil + return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil } // Mounts a verified block CIM with the provided root hash. The root hash is usually From 61b1bd497aa8301a205cce0fd3ccc428e589bb80 Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 27 Aug 2025 11:24:22 -0400 Subject: [PATCH 3/6] Include policy digest in the host data for confidential UVM When the policy digest is included in the host data field of the UVM config, the SNP hardware is able to directly access that and include that in the attestation report. Signed-off-by: Amit Barve --- internal/uvm/create_wcow.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 4069ae00b0..8585dba100 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -4,6 +4,7 @@ package uvm import ( "context" + "encoding/base64" "fmt" "maps" "os" @@ -27,6 +28,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm/scsi" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" ) type ConfidentialWCOWOptions struct { @@ -335,12 +337,23 @@ func prepareSecurityConfigDoc(ctx context.Context, uvm *UtilityVM, opts *Options } } + policyDigest, err := securitypolicy.NewSecurityPolicyDigest(opts.SecurityPolicy) + if err != nil { + return nil, err + } + + // HCS API expect a base64 encoded string as LaunchData. Internally it + // decodes it to bytes. SEV later returns the decoded byte blob as HostData + // field of the report. + hostData := base64.StdEncoding.EncodeToString(policyDigest) + enableHCL := true doc.VirtualMachine.SecuritySettings = &hcsschema.SecuritySettings{ EnableTpm: false, Isolation: &hcsschema.IsolationSettings{ IsolationType: "SecureNestedPaging", HclEnabled: &enableHCL, + LaunchData: hostData, }, } From c9a2cb9f9cd516a26f90795cdd0d9fb2accfac7c Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 27 Aug 2025 11:24:28 -0400 Subject: [PATCH 4/6] Attach EFI VHD in read-only mode by default EFI VHDs should always be attached as read-only by default to block UVMs from writing to it and corrupting its contents. A new annotation is added to allow attaching EFI VHDs in writable mode when debugging boot failures and such. When this annotation is included a copy of the EFI VHD is made next to the scratch VHD. This is based on the assumption that generally the scratch of the UVM would be stored in its own snapshot directory so adding another VHD in there shouldn't be a problem. It should get cleaned up when the snapshot is removed. This commit also adds the code to always grant VM group access to the VHDs and guest state files to avoid access denied failures. Signed-off-by: Amit Barve --- cmd/containerd-shim-runhcs-v1/pod.go | 16 ++++++++ internal/oci/uvm.go | 3 ++ internal/oci/uvm_test.go | 57 +++++++++++++++++++++++++--- internal/uvm/create_wcow.go | 22 ++++++++--- pkg/annotations/annotations.go | 4 ++ 5 files changed, 91 insertions(+), 11 deletions(-) diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index ccd9fbe2eb..b41e4a1b15 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" @@ -58,6 +59,21 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs // we can enable this. return fmt.Errorf("hyperv isolation is not supported with block CIM layers yet") } + + // writable EFI VHD is a valid config for both confidential and regular hyperv + // isolated WCOW. Override the default value here if required. + if wopts.WritableEFI { + // Make a copy of EFI VHD, we can't risk the UVM accidentally modifying + // the original VHD. make copy next to the scratch VHD, this assumes that + // the scratch is located in the separate directory dedicated for this + // UVM. + writableEFIVHDPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.BootFiles.BlockCIMFiles.EFIVHDPath)) + if err := copyfile.CopyFile(ctx, wopts.BootFiles.BlockCIMFiles.EFIVHDPath, writableEFIVHDPath, false); err != nil { + return fmt.Errorf("failed to copy EFI VHD at %s: %w", writableEFIVHDPath, err) + } + wopts.BootFiles.BlockCIMFiles.EFIVHDPath = writableEFIVHDPath + } + return nil } diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index d1c1e75734..03afc95e50 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -376,6 +376,9 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( wopts.AdditionalRegistryKeys = append(wopts.AdditionalRegistryKeys, parseAdditionalRegistryValues(ctx, s.Annotations)...) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) + // Writable EFI is valid for both confidential and regular Hyper-V isolated WCOW. + wopts.WritableEFI = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWWritableEFI, wopts.WritableEFI) + // Handle WCOW security policy settings if err := handleWCOWSecurityPolicy(ctx, s.Annotations, wopts); err != nil { return nil, err diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 188507ae75..4a4ceed5c2 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -139,6 +139,10 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_Defaults(t *testing.T) { if !wopts.SecurityPolicyEnabled { t.Fatal("SecurityPolicyEnabled should be true when WCOWSecurityPolicy is set") } + // Writable EFI should default to false unless explicitly enabled + if wopts.WritableEFI { + t.Fatal("WritableEFI should default to false when not specified") + } if wopts.MemorySizeInMB != 2048 { t.Fatalf("expected MemorySizeInMB to default to 2048, got %d", wopts.MemorySizeInMB) } @@ -159,6 +163,47 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_Defaults(t *testing.T) { } } +func Test_SpecToUVMCreateOptions_WCOW_Confidential_WritableEFI_Enabled(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.WCOWWritableEFI: "true", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if !wopts.WritableEFI { + t.Fatal("WritableEFI should be true when WCOWWritableEFI annotation is set to true") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_NonConfidential_WritableEFI_Enabled(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + // No WCOWSecurityPolicy means non-confidential path + annotations.WCOWWritableEFI: "true", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + // WritableEFI should be respected for non-confidential as well + if !wopts.WritableEFI { + t.Fatal("WritableEFI should be true for non-confidential when WCOWWritableEFI is set") + } +} + func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnLowMemory(t *testing.T) { s := &specs.Spec{ Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, @@ -191,13 +236,13 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_Overrides(t *testing.T) { s := &specs.Spec{ Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, Annotations: map[string]string{ - annotations.WCOWSecurityPolicy: "test-policy", - annotations.MemorySizeInMB: "4096", - annotations.AllowOvercommit: "false", + annotations.WCOWSecurityPolicy: "test-policy", + annotations.MemorySizeInMB: "4096", + annotations.AllowOvercommit: "false", annotations.WCOWSecurityPolicyEnforcer: "rego", - annotations.WCOWDisableSecureBoot: "true", - annotations.WCOWGuestStateFile: "C:\\custom\\cwcow.vmgs", - annotations.WCOWIsolationType: "VirtualizationBasedSecurity", + annotations.WCOWDisableSecureBoot: "true", + annotations.WCOWGuestStateFile: "C:\\custom\\cwcow.vmgs", + annotations.WCOWIsolationType: "VirtualizationBasedSecurity", }, } diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 8585dba100..77c77f5e02 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -41,6 +41,7 @@ type ConfidentialWCOWOptions struct { IsolationType string DisableSecureBoot bool FirmwareParameters string + WritableEFI bool } // OptionsWCOW are the set of options passed to CreateWCOW() to create a utility vm. @@ -384,26 +385,37 @@ func prepareSecurityConfigDoc(ctx context.Context, uvm *UtilityVM, opts *Options Minor: 0, } + // TODO(ambarve): only scratch VHD is unique per VM, EFI & Boot CIM VHDs are + // shared across UVMs, so we don't need to assign VM group access to them every + // time. It should have been done once while deploying the package. + if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.EFIVHDPath); err != nil { + return nil, errors.Wrap(err, "failed to grant vm access to EFI VHD") + } + if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.BootCIMVHDPath); err != nil { - return nil, errors.Wrap(err, "failed to grant vm access to boot CIM VHD") + return nil, errors.Wrap(err, "failed to grant vm access to Boot CIM VHD") } - if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.EFIVHDPath); err != nil { - return nil, errors.Wrap(err, "failed to grant vm access to EFI VHD") + if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.GuestStateFilePath); err != nil { + return nil, errors.Wrap(err, "failed to grant vm access to guest state file") } if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.BlockCIMFiles.ScratchVHDPath); err != nil { return nil, errors.Wrap(err, "failed to grant vm access to scratch VHD") } + // boot depends on scratch being attached at LUN 0, it MUST ALWAYS remain at LUN 0 doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["0"] = hcsschema.Attachment{ Path: opts.BootFiles.BlockCIMFiles.ScratchVHDPath, Type_: "VirtualDisk", } + doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["1"] = hcsschema.Attachment{ - Path: opts.BootFiles.BlockCIMFiles.EFIVHDPath, - Type_: "VirtualDisk", + Path: opts.BootFiles.BlockCIMFiles.EFIVHDPath, + Type_: "VirtualDisk", + ReadOnly: !opts.WritableEFI, } + doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["2"] = hcsschema.Attachment{ Path: opts.BootFiles.BlockCIMFiles.BootCIMVHDPath, Type_: "VirtualDisk", diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index c4ebd4f2b8..abc0a2772d 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -223,6 +223,10 @@ const ( // Allows disabling secure boot for testing and debugging scenarios, secure boot doesn't apply to confidential LCOW so // this is a WCOW only config WCOWDisableSecureBoot = "io.microsoft.virtualmachine.wcow.no_secure_boot" + + // Attaches the EFI/boot VHD in the writable mode (instead of the default read-only mode). This is usually required + // when debugging boot to capture bootstat traces. + WCOWWritableEFI = "io.microsoft.virtualmachine.wcow.writable_efi" ) // WCOW host process container annotations. From e48451f4ae412c29f0cd0ebb82e25165919d0667 Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 27 Aug 2025 11:24:29 -0400 Subject: [PATCH 5/6] format container scratch in superfloppy mode Signed-off-by: Amit Barve --- internal/fsformatter/formatter_driver.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/fsformatter/formatter_driver.go b/internal/fsformatter/formatter_driver.go index 14f743cf0b..7317e4c779 100644 --- a/internal/fsformatter/formatter_driver.go +++ b/internal/fsformatter/formatter_driver.go @@ -60,12 +60,17 @@ func (filesystemType kernelFormatVolumeFilesystemTypes) String() string { type kernelFormatVolumeFormatInputBufferFlags uint32 -const kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000) +const ( + kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000) + kernelFormatVolumeFormatInputBufferFlagSuperFloppy = kernelFormatVolumeFormatInputBufferFlags(0x00000001) +) func (flag kernelFormatVolumeFormatInputBufferFlags) String() string { switch flag { case kernelFormatVolumeFormatInputBufferFlagNone: return "kernelFormatVolumeFormatInputBufferFlagNone" + case kernelFormatVolumeFormatInputBufferFlagSuperFloppy: + return "kernelFormatVolumeFormatInputBufferFlagSuperFloppy" default: return "Unknown" } @@ -211,7 +216,7 @@ func KmFmtCreateFormatInputBuffer(diskPath string) *KernelFormatVolumeFormatInpu inputBuffer := (*KernelFormatVolumeFormatInputBuffer)(unsafe.Pointer(&buf[0])) inputBuffer.Size = uint64(bufferSize) - inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagNone + inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagSuperFloppy inputBuffer.FsParameters.FileSystemType = kernelFormatVolumeFilesystemTypeRefs inputBuffer.FsParameters.VolumeLabelLength = 0 From 11fd4bdb173ca7e94acb2d5bf2e52fa2ce05d3c5 Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Fri, 29 Aug 2025 10:53:10 -0400 Subject: [PATCH 6/6] Add containerID to CombineLayers and MountBlockCIM request types containerID is required for all gcs requests in confidential mode. Most of the requests already include it but CombineLayers and MountBlockCIM request types don't. This commit adds that to these request types. Signed-off-by: Amit Barve --- internal/gcs-sidecar/handlers.go | 11 +++++- internal/gcs-sidecar/uvm.go | 2 +- internal/layers/wcow_mount.go | 8 ++-- internal/protocol/guestresource/resources.go | 9 +++-- internal/uvm/cimfs.go | 16 +++++--- internal/uvm/combine_layers.go | 41 ++++++++++++++------ internal/uvm/security_policy.go | 11 ++++++ 7 files changed, 71 insertions(+), 27 deletions(-) diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go index e321d4e092..689e02bb66 100644 --- a/internal/gcs-sidecar/handlers.go +++ b/internal/gcs-sidecar/handlers.go @@ -338,7 +338,11 @@ func (b *Bridge) modifySettings(req *request) (err error) { case guestresource.ResourceTypeWCOWBlockCims: // This is request to mount the merged cim at given volumeGUID - wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWBlockCIMMounts) + if modifyGuestSettingsRequest.RequestType == guestrequest.RequestTypeRemove { + return fmt.Errorf("not implemented") + } + + wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWBlockCIMMounts) log.G(ctx).Tracef("WCOWBlockCIMMounts { %v}", wcowBlockCimMounts) // The block device takes some time to show up. Wait for a few seconds. @@ -386,6 +390,11 @@ func (b *Bridge) modifySettings(req *request) (err error) { return nil case guestresource.ResourceTypeCWCOWCombinedLayers: + + if modifyGuestSettingsRequest.RequestType == guestrequest.RequestTypeRemove { + return fmt.Errorf("not implemented") + } + settings := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWCombinedLayers) containerID := settings.ContainerID log.G(ctx).Tracef("CWCOWCombinedLayers:: ContainerID: %v, ContainerRootPath: %v, Layers: %v, ScratchPath: %v", diff --git a/internal/gcs-sidecar/uvm.go b/internal/gcs-sidecar/uvm.go index 4fccdf3df6..9578e0273c 100644 --- a/internal/gcs-sidecar/uvm.go +++ b/internal/gcs-sidecar/uvm.go @@ -106,7 +106,7 @@ func unmarshalContainerModifySettings(req *request) (_ *prot.ContainerModifySett modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk case guestresource.ResourceTypeWCOWBlockCims: - wcowBlockCimMounts := &guestresource.WCOWBlockCIMMounts{} + wcowBlockCimMounts := &guestresource.CWCOWBlockCIMMounts{} if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowBlockCimMounts); err != nil { return nil, fmt.Errorf("invalid ResourceTypeWCOWBlockCims request: %w", err) } diff --git a/internal/layers/wcow_mount.go b/internal/layers/wcow_mount.go index f20217459e..c5ee9cbabb 100644 --- a/internal/layers/wcow_mount.go +++ b/internal/layers/wcow_mount.go @@ -440,7 +440,8 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm }) } - err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.WCIFS) + // containerID isn't required when using non-confidential pods (WCIFS based layers can't run confidential pods) + err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.WCIFS, "") if err != nil { return nil, nil, err } @@ -476,7 +477,7 @@ func mountHypervIsolatedBlockCIMLayers(ctx context.Context, l *wcowBlockCIMLayer "parent layers": l.parentLayers, }).Debug("mounting hyperv isolated block CIM layers") - mountedCIMs, err := vm.MountBlockCIMs(ctx, l.mergedLayer, l.parentLayers) + mountedCIMs, err := vm.MountBlockCIMs(ctx, l.mergedLayer, l.parentLayers, containerID) if err != nil { return nil, nil, fmt.Errorf("failed to mount block CIMs in UVM: %w", err) } @@ -524,8 +525,7 @@ func mountHypervIsolatedBlockCIMLayers(ctx context.Context, l *wcowBlockCIMLayer }, } - // TODO(ambarve): Do we need CWCOW specific request type here? - err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.UnionFS) + err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.UnionFS, containerID) if err != nil { return nil, nil, err } diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index 7508668980..c19d54e89a 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -114,11 +114,12 @@ type BlockCIMDevice struct { Lun int32 } -type WCOWBlockCIMMounts struct { +type CWCOWBlockCIMMounts struct { // BlockCIMs should be ordered from merged CIM followed by Layer n .. layer 1 - BlockCIMs []BlockCIMDevice `json:"BlockCIMs,omitempty"` - VolumeGUID guid.GUID `json:"VolumeGUID,omitempty"` - MountFlags uint32 `json:"MountFlags,omitempty"` + BlockCIMs []BlockCIMDevice `json:"BlockCIMs,omitempty"` + VolumeGUID guid.GUID `json:"VolumeGUID,omitempty"` + MountFlags uint32 `json:"MountFlags,omitempty"` + ContainerID string `json:"ContainerID,omitempty"` } type WCOWMappedVirtualDisk struct { diff --git a/internal/uvm/cimfs.go b/internal/uvm/cimfs.go index 92cd45b8d7..c1e8704883 100644 --- a/internal/uvm/cimfs.go +++ b/internal/uvm/cimfs.go @@ -46,7 +46,7 @@ func (umb *UVMMountedBlockCIMs) Release(ctx context.Context) error { guestReq := guestrequest.ModificationRequest{ ResourceType: guestresource.ResourceTypeWCOWBlockCims, RequestType: guestrequest.RequestTypeRemove, - Settings: &guestresource.WCOWBlockCIMMounts{ + Settings: &guestresource.CWCOWBlockCIMMounts{ VolumeGUID: umb.volumeGUID, }, } @@ -67,7 +67,7 @@ func (umb *UVMMountedBlockCIMs) Release(ctx context.Context) error { // mergedCIM can be nil, // sourceCIMs MUST be in the top to bottom order -func (uvm *UtilityVM) MountBlockCIMs(ctx context.Context, mergedCIM *cimfs.BlockCIM, sourceCIMs []*cimfs.BlockCIM) (_ *UVMMountedBlockCIMs, retErr error) { +func (uvm *UtilityVM) MountBlockCIMs(ctx context.Context, mergedCIM *cimfs.BlockCIM, sourceCIMs []*cimfs.BlockCIM, containerID string) (_ *UVMMountedBlockCIMs, retErr error) { if len(sourceCIMs) < 1 { return nil, fmt.Errorf("at least 1 source CIM is required") } @@ -94,10 +94,14 @@ func (uvm *UtilityVM) MountBlockCIMs(ctx context.Context, mergedCIM *cimfs.Block return nil, fmt.Errorf("generated cim mount GUID: %w", err) } - settings := &guestresource.WCOWBlockCIMMounts{ - BlockCIMs: []guestresource.BlockCIMDevice{}, - VolumeGUID: volumeGUID, - MountFlags: cimfs.CimMountBlockDeviceCim, + // TODO(ambarve): When inbox GCS adds support for mounting block CIMs, we should + // use the appropriate request type for confidential vs regular pods as inbox GCS + // may not understand the CWCOWBlockCIMMounts type. + settings := &guestresource.CWCOWBlockCIMMounts{ + BlockCIMs: []guestresource.BlockCIMDevice{}, + VolumeGUID: volumeGUID, + MountFlags: cimfs.CimMountBlockDeviceCim, + ContainerID: containerID, } umb := &UVMMountedBlockCIMs{ diff --git a/internal/uvm/combine_layers.go b/internal/uvm/combine_layers.go index c3b79622f4..b577f706f3 100644 --- a/internal/uvm/combine_layers.go +++ b/internal/uvm/combine_layers.go @@ -14,22 +14,41 @@ import ( // container file system. // // Note: `layerPaths` and `containerRootPath` are paths from within the UVM. -func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcsschema.Layer, containerRootPath string, filterType hcsschema.FileSystemFilterType) error { +func (uvm *UtilityVM) CombineLayersWCOW(ctx context.Context, layerPaths []hcsschema.Layer, containerRootPath string, filterType hcsschema.FileSystemFilterType, containerID string) error { if uvm.operatingSystem != "windows" { return errNotSupported } - msr := &hcsschema.ModifySettingRequest{ - GuestRequest: guestrequest.ModificationRequest{ - ResourceType: guestresource.ResourceTypeCombinedLayers, - RequestType: guestrequest.RequestTypeAdd, - Settings: guestresource.WCOWCombinedLayers{ - ContainerRootPath: containerRootPath, - Layers: layerPaths, - FilterType: filterType, + + var modifyRequest *hcsschema.ModifySettingRequest + if uvm.HasConfidentialPolicy() { + modifyRequest = &hcsschema.ModifySettingRequest{ + GuestRequest: guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeCWCOWCombinedLayers, + RequestType: guestrequest.RequestTypeAdd, + Settings: guestresource.CWCOWCombinedLayers{ + ContainerID: containerID, + CombinedLayers: guestresource.WCOWCombinedLayers{ + ContainerRootPath: containerRootPath, + Layers: layerPaths, + FilterType: filterType, + }, + }, }, - }, + } + } else { + modifyRequest = &hcsschema.ModifySettingRequest{ + GuestRequest: guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypeCombinedLayers, + RequestType: guestrequest.RequestTypeAdd, + Settings: guestresource.WCOWCombinedLayers{ + ContainerRootPath: containerRootPath, + Layers: layerPaths, + FilterType: filterType, + }, + }, + } } - return uvm.modify(ctx, msr) + return uvm.modify(ctx, modifyRequest) } // CombineLayersLCOW combines `layerPaths` and optionally `scratchPath` into an diff --git a/internal/uvm/security_policy.go b/internal/uvm/security_policy.go index 195b93d3ce..950179acc4 100644 --- a/internal/uvm/security_policy.go +++ b/internal/uvm/security_policy.go @@ -120,3 +120,14 @@ func (uvm *UtilityVM) InjectPolicyFragment(ctx context.Context, fragment *ctrdta } return uvm.modify(ctx, mod) } + +// returns if this instance of the UtilityVM is created with confidential policy +func (uvm *UtilityVM) HasConfidentialPolicy() bool { + switch opts := uvm.createOpts.(type) { + case *OptionsWCOW: + return opts.SecurityPolicyEnabled + case *OptionsLCOW: + return opts.SecurityPolicyEnabled + } + return false +}