Support for starting confidential containers

cmd/containerd-shim-runhcs-v1/pod.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/Microsoft/hcsshim/internal/copyfile"
 	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oci"
@@ -18,13 +19,64 @@ import (
 	"github.com/Microsoft/hcsshim/pkg/annotations"
 	eventstypes "github.com/containerd/containerd/api/events"
 	task "github.com/containerd/containerd/api/runtime/task/v2"
+	"github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/v2/core/runtime"
 	"github.com/containerd/errdefs"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
 
+// initializeWCOWBootFiles handles the initialization of boot files for WCOW VMs
+func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs []*types.Mount, s *specs.Spec) error {
+	var (
+		layerFolders []string
+		err          error
+	)
+
+	if s.Windows != nil {
+		layerFolders = s.Windows.LayerFolders
+	}
+
+	wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, rootfs, layerFolders)
+	if err != nil {
+		return err
+	}
+
+	if wopts.SecurityPolicyEnabled {
+		if wopts.BootFiles.BootType != uvm.BlockCIMBoot {
+			return fmt.Errorf("security policy (confidential mode) only works with block CIM based layers")
+		}
+		// we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs
+		wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath()
+		wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath()
+	} else if wopts.BootFiles.BootType == uvm.BlockCIMBoot {
+		// Supporting hyperv isolation with block CIM layers requires changes in
+		// the image pull path to prepare the EFI VHD. But more importantly, since both the
+		// container and UtilityVM files will be in the same CIM, the early boot
+		// code (bootmgr, winload etc.) will need to support reading the UVM OS
+		// files from `UtilityVM/Files` inside the CIM. Once that support is added
+		// we can enable this.
+		return fmt.Errorf("hyperv isolation is not supported with block CIM layers yet")
+	}
+
+	// writable EFI VHD is a valid config for both confidential and regular hyperv
+	// isolated WCOW. Override the default value here if required.
+	if wopts.WritableEFI {
+		// Make a copy of EFI VHD, we can't risk the UVM accidentally modifying
+		// the original VHD.  make copy next to the scratch VHD, this assumes that
+		// the scratch is located in the separate directory dedicated for this
+		// UVM.
+		writableEFIVHDPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.BootFiles.BlockCIMFiles.EFIVHDPath))
+		if err := copyfile.CopyFile(ctx, wopts.BootFiles.BlockCIMFiles.EFIVHDPath, writableEFIVHDPath, false); err != nil {
+			return fmt.Errorf("failed to copy EFI VHD at %s: %w", writableEFIVHDPath, err)
+		}
+		wopts.BootFiles.BlockCIMFiles.EFIVHDPath = writableEFIVHDPath
+	}
+
+	return nil
+}
+
 // shimPod represents the logical grouping of all tasks in a single set of
 // shared namespaces. The pod sandbox (container) is represented by the task
 // that matches the `shimPod.ID()`
@@ -123,16 +175,11 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 				return nil, err
 			}
 		case *uvm.OptionsWCOW:
-			var layerFolders []string
-			if s.Windows != nil {
-				layerFolders = s.Windows.LayerFolders
-			}
 			wopts := (opts).(*uvm.OptionsWCOW)
-			wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders)
+			err = initializeWCOWBootFiles(ctx, wopts, req.Rootfs, s)
 			if err != nil {
 				return nil, err
 			}
-
 			parent, err = uvm.CreateWCOW(ctx, wopts)
 			if err != nil {
 				return nil, err

internal/fsformatter/formatter_driver.go

@@ -60,12 +60,17 @@ func (filesystemType kernelFormatVolumeFilesystemTypes) String() string {
 
 type kernelFormatVolumeFormatInputBufferFlags uint32
 
-const kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000)
+const (
+	kernelFormatVolumeFormatInputBufferFlagNone        = kernelFormatVolumeFormatInputBufferFlags(0x00000000)
+	kernelFormatVolumeFormatInputBufferFlagSuperFloppy = kernelFormatVolumeFormatInputBufferFlags(0x00000001)
+)
 
 func (flag kernelFormatVolumeFormatInputBufferFlags) String() string {
 	switch flag {
 	case kernelFormatVolumeFormatInputBufferFlagNone:
 		return "kernelFormatVolumeFormatInputBufferFlagNone"
+	case kernelFormatVolumeFormatInputBufferFlagSuperFloppy:
+		return "kernelFormatVolumeFormatInputBufferFlagSuperFloppy"
 	default:
 		return "Unknown"
 	}
@@ -211,7 +216,7 @@ func KmFmtCreateFormatInputBuffer(diskPath string) *KernelFormatVolumeFormatInpu
 	inputBuffer := (*KernelFormatVolumeFormatInputBuffer)(unsafe.Pointer(&buf[0]))
 
 	inputBuffer.Size = uint64(bufferSize)
-	inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagNone
+	inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagSuperFloppy
 
 	inputBuffer.FsParameters.FileSystemType = kernelFormatVolumeFilesystemTypeRefs
 	inputBuffer.FsParameters.VolumeLabelLength = 0

internal/gcs-sidecar/handlers.go

@@ -338,7 +338,11 @@ func (b *Bridge) modifySettings(req *request) (err error) {
 
 		case guestresource.ResourceTypeWCOWBlockCims:
 			// This is request to mount the merged cim at given volumeGUID
-			wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWBlockCIMMounts)
+			if modifyGuestSettingsRequest.RequestType == guestrequest.RequestTypeRemove {
+				return fmt.Errorf("not implemented")
+			}
+
+			wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWBlockCIMMounts)
 			log.G(ctx).Tracef("WCOWBlockCIMMounts { %v}", wcowBlockCimMounts)
 
 			// The block device takes some time to show up. Wait for a few seconds.
@@ -386,6 +390,11 @@ func (b *Bridge) modifySettings(req *request) (err error) {
 			return nil
 
 		case guestresource.ResourceTypeCWCOWCombinedLayers:
+
+			if modifyGuestSettingsRequest.RequestType == guestrequest.RequestTypeRemove {
+				return fmt.Errorf("not implemented")
+			}
+
 			settings := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWCombinedLayers)
 			containerID := settings.ContainerID
 			log.G(ctx).Tracef("CWCOWCombinedLayers:: ContainerID: %v, ContainerRootPath: %v, Layers: %v, ScratchPath: %v",

internal/gcs-sidecar/uvm.go

@@ -106,7 +106,7 @@ func unmarshalContainerModifySettings(req *request) (_ *prot.ContainerModifySett
 			modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk
 
 		case guestresource.ResourceTypeWCOWBlockCims:
-			wcowBlockCimMounts := &guestresource.WCOWBlockCIMMounts{}
+			wcowBlockCimMounts := &guestresource.CWCOWBlockCIMMounts{}
 			if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowBlockCimMounts); err != nil {
 				return nil, fmt.Errorf("invalid ResourceTypeWCOWBlockCims request: %w", err)
 			}

internal/guest/runtime/hcsv2/uvm.go

@@ -456,9 +456,9 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 	// containing the files is exposed via UVM_SECURITY_CONTEXT_DIR env var.
 	// It may be an error to have a security policy but not expose it to the
 	// container as in that case it can never be checked as correct by a verifier.
-	if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.UVMSecurityPolicyEnv, true) {
+	if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWSecurityPolicyEnv, true) {
 		encodedPolicy := h.securityPolicyEnforcer.EncodedSecurityPolicy()
-		hostAMDCert := settings.OCISpecification.Annotations[annotations.HostAMDCertificate]
+		hostAMDCert := settings.OCISpecification.Annotations[annotations.LCOWHostAMDCertificate]
 		if len(encodedPolicy) > 0 || len(hostAMDCert) > 0 || len(h.uvmReferenceInfo) > 0 {
 			// Use os.MkdirTemp to make sure that the directory is unique.
 			securityContextDir, err := os.MkdirTemp(settings.OCISpecification.Root.Path, securitypolicy.SecurityContextDirTemplate)

internal/layers/wcow_mount.go

@@ -43,7 +43,7 @@ func MountWCOWLayers(ctx context.Context, containerID string, vm *uvm.UtilityVM,
 		if vm == nil {
 			return mountProcessIsolatedBlockCIMLayers(ctx, containerID, l)
 		}
-		return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with block cim layers")
+		return mountHypervIsolatedBlockCIMLayers(ctx, l, vm, containerID)
 	default:
 		return nil, nil, fmt.Errorf("invalid layer type %T", wl)
 	}
@@ -440,7 +440,8 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm
 		})
 	}
 
-	err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS)
+	// containerID isn't required when using non-confidential pods (WCIFS based layers can't run confidential pods)
+	err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.WCIFS, "")
 	if err != nil {
 		return nil, nil, err
 	}
@@ -454,6 +455,90 @@ func mountHypervIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers, vm
 	}, nil
 }
 
+func mountHypervIsolatedBlockCIMLayers(ctx context.Context, l *wcowBlockCIMLayers, vm *uvm.UtilityVM, containerID string) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	ctx, span := oc.StartSpan(ctx, "mountHyperVIsolatedBlockCIMLayers")
+	defer func() {
+		oc.SetSpanStatus(span, err)
+		span.End()
+	}()
+
+	rcl := &resources.ResourceCloserList{}
+	defer func() {
+		if err != nil {
+			if rErr := rcl.Release(ctx); rErr != nil {
+				log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr)
+			}
+		}
+	}()
+
+	log.G(ctx).WithFields(logrus.Fields{
+		"scratch":       l.scratchLayerPath,
+		"merged layer":  l.mergedLayer,
+		"parent layers": l.parentLayers,
+	}).Debug("mounting hyperv isolated block CIM layers")
+
+	mountedCIMs, err := vm.MountBlockCIMs(ctx, l.mergedLayer, l.parentLayers, containerID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to mount block CIMs in UVM: %w", err)
+	}
+	rcl.Add(mountedCIMs)
+
+	// mount the CIM inside UVM now
+	log.G(ctx).WithField("volume", mountedCIMs.MountedVolumePath()).Debug("mounted blockCIM layers for hyperV isolated container")
+
+	hostPath := filepath.Join(l.scratchLayerPath, "sandbox.vhdx")
+
+	scsiMount, err := vm.SCSIManager.AddVirtualDisk(ctx, hostPath, false, vm.ID(), "",
+		&scsi.MountConfig{
+			// TODO(ambarve): Add SCSI config to format the scratch in guest
+		})
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to add SCSI scratch VHD: %w", err)
+	}
+	containerScratchPathInUVM := scsiMount.GuestPath()
+	rcl.Add(scsiMount)
+
+	log.G(ctx).WithFields(logrus.Fields{
+		"hostPath": hostPath,
+		"uvmPath":  containerScratchPathInUVM,
+	}).Debug("mounted scratch VHD")
+
+	mountedCIMLayerID, err := cimlayer.LayerID(mountedCIMs.MountedVolumePath())
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get layer ID for mounted block CIM: %w", err)
+	}
+
+	ml := &MountedWCOWLayers{
+		RootFS: containerScratchPathInUVM,
+		MountedLayerPaths: []MountedWCOWLayer{
+			{
+				LayerID:     mountedCIMLayerID,
+				MountedPath: mountedCIMs.MountedVolumePath(),
+			},
+		},
+	}
+
+	hcsLayers := []hcsschema.Layer{
+		{
+			Id:   mountedCIMLayerID,
+			Path: filepath.Join(mountedCIMs.MountedVolumePath(), "Files"),
+		},
+	}
+
+	err = vm.CombineLayersWCOW(ctx, hcsLayers, ml.RootFS, hcsschema.UnionFS, containerID)
+	if err != nil {
+		return nil, nil, err
+	}
+	log.G(ctx).Debug("hcsshim::mountHyperVIsolatedBlockCIMLayers Succeeded")
+
+	return ml, &wcowIsolatedWCIFSLayerCloser{
+		uvm:                     vm,
+		guestCombinedLayersPath: ml.RootFS,
+		scratchMount:            scsiMount,
+		layerClosers:            []resources.ResourceCloser{rcl},
+	}, nil
+}
+
 // Mount the sandbox vhd to a user friendly path.
 func MountSandboxVolume(ctx context.Context, hostPath, volumeName string) (err error) {
 	log.G(ctx).WithFields(logrus.Fields{

internal/layers/wcow_parse.go

@@ -217,33 +217,7 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers,
 	}
 }
 
-// GetWCOWUVMBootFilesFromLayers prepares the UVM boot files from the rootfs or layerFolders.
-func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) {
-	var parentLayers []string
-	var scratchLayer string
-	var err error
-
-	if err = validateRootfsAndLayers(rootfs, layerFolders); err != nil {
-		return nil, err
-	}
-
-	if len(layerFolders) > 0 {
-		parentLayers = layerFolders[:len(layerFolders)-1]
-		scratchLayer = layerFolders[len(layerFolders)-1]
-	} else {
-		m := rootfs[0]
-		switch m.Type {
-		case legacyMountType:
-			parentLayers, err = getOptionAsArray(m, parentLayerPathsFlag)
-			if err != nil {
-				return nil, err
-			}
-			scratchLayer = m.Source
-		default:
-			return nil, fmt.Errorf("mount type '%s' is not supported for UVM boot", m.Type)
-		}
-	}
-
+func getVmbFSBootFiles(ctx context.Context, scratchLayer string, parentLayers []string) (*uvm.WCOWBootFiles, error) {
 	uvmFolder, err := uvmfolder.LocateUVMFolder(ctx, parentLayers)
 	if err != nil {
 		return nil, fmt.Errorf("failed to locate utility VM folder from layer folders: %w", err)
@@ -252,6 +226,11 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l
 	// In order for the UVM sandbox.vhdx not to collide with the actual
 	// nested Argon sandbox.vhdx we append the \vm folder to the last
 	// entry in the list.
+	// TODO(ambarve): This probably is a bug, we should make a separate `vm` directory
+	// only if we are creating a standalone task. If we are starting a separate pod,
+	// containerd snapshotter already creates a separate directory for the pod. We
+	// won't have to worry about the name collision in that case. Keeping this for
+	// backward compatibility/avoid breaking existing code.
 	scratchLayer = filepath.Join(scratchLayer, "vm")
 	scratchVHDPath := filepath.Join(scratchLayer, "sandbox.vhdx")
 	if err = os.MkdirAll(scratchLayer, 0777); err != nil {
@@ -264,6 +243,7 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l
 			return nil, err
 		}
 	}
+
 	return &uvm.WCOWBootFiles{
 		BootType: uvm.VmbFSBoot,
 		VmbFSFiles: &uvm.VmbFSBootFiles{
@@ -273,3 +253,38 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l
 		},
 	}, nil
 }
+
+func getBlockCIMBootFiles(ctx context.Context, wl *wcowBlockCIMLayers) (*uvm.WCOWBootFiles, error) {
+	scratchVHDPath := filepath.Join(wl.scratchLayerPath, "sandbox.vhdx")
+
+	// block CIM based layers don't support multiple layers in the UVM, pick the base layer and continue
+	efiVHDPath := filepath.Join(filepath.Dir(wl.parentLayers[0].BlockPath), "boot.vhd")
+
+	return &uvm.WCOWBootFiles{
+		BootType: uvm.BlockCIMBoot,
+		BlockCIMFiles: &uvm.BlockCIMBootFiles{
+			BootCIMVHDPath: wl.parentLayers[0].BlockPath, // This should be a block CIM with VHD footer attached.
+			EFIVHDPath:     efiVHDPath,
+			ScratchVHDPath: scratchVHDPath,
+		},
+	}, nil
+}
+
+// GetWCOWUVMBootFilesFromLayers prepares the UVM boot files from the rootfs or layerFolders.
+func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) {
+	var err error
+
+	parsedWCOWLayers, err := ParseWCOWLayers(rootfs, layerFolders)
+	if err != nil {
+		return nil, err
+	}
+
+	switch wl := parsedWCOWLayers.(type) {
+	case *wcowWCIFSLayers:
+		return getVmbFSBootFiles(ctx, wl.scratchLayerPath, wl.layerPaths)
+	case *wcowBlockCIMLayers:
+		return getBlockCIMBootFiles(ctx, wl)
+	default:
+		return nil, fmt.Errorf("unsupported layer format for UVM boot: %T", parsedWCOWLayers)
+	}
+}