Support for mounting verified merged CIMs

cmd/containerd-shim-runhcs-v1/pod.go

@@ -37,6 +37,7 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs
 	if s.Windows != nil {
 		layerFolders = s.Windows.LayerFolders
 	}
+	log.G(ctx).WithField("options", log.Format(ctx, *wopts)).Debug("initialize WCOW boot files")
 
 	wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, rootfs, layerFolders)
 	if err != nil {
@@ -50,6 +51,16 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs
 		// we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs
 		wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath()
 		wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath()
+
+		// make a copy of the vmgs file as the same vmgs can not be used by multiple pods in parallel
+		// TODO(ambarve): for C-LCOW we make a copy in the bundle directory, is it better
+		// to use the bundle directory instead of the snapshot directory?
+		vmgsCopyPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.GuestStateFilePath))
+		if err := copyfile.CopyFile(ctx, wopts.GuestStateFilePath, vmgsCopyPath, false); err != nil {
+			return fmt.Errorf("failed to make a copy of VMGS: %w", err)
+		}
+		wopts.GuestStateFilePath = vmgsCopyPath
+
 	} else if wopts.BootFiles.BootType == uvm.BlockCIMBoot {
 		// Supporting hyperv isolation with block CIM layers requires changes in
 		// the image pull path to prepare the EFI VHD. But more importantly, since both the
@@ -69,7 +80,7 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs
 		// UVM.
 		writableEFIVHDPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.BootFiles.BlockCIMFiles.EFIVHDPath))
 		if err := copyfile.CopyFile(ctx, wopts.BootFiles.BlockCIMFiles.EFIVHDPath, writableEFIVHDPath, false); err != nil {
-			return fmt.Errorf("failed to copy EFI VHD at %s: %w", writableEFIVHDPath, err)
+			return fmt.Errorf("failed to copy EFI VHD: %w", err)
 		}
 		wopts.BootFiles.BlockCIMFiles.EFIVHDPath = writableEFIVHDPath
 	}

internal/gcs-sidecar/handlers.go

@@ -677,15 +677,14 @@ func (b *Bridge) modifySettings(req *request) (err error) {
 			}
 
 			if len(layerCIMs) > 1 {
-				// Get the topmost merge CIM and invoke the MountMergedBlockCIMs
-				_, err := cimfs.MountMergedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID)
+				_, err = cimfs.MountMergedVerifiedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID, layerDigests[0])
 				if err != nil {
 					return fmt.Errorf("error mounting multilayer block cims: %w", err)
 				}
 			} else {
-				_, err := cimfs.Mount(filepath.Join(layerCIMs[0].BlockPath, layerCIMs[0].CimName), wcowBlockCimMounts.VolumeGUID, wcowBlockCimMounts.MountFlags)
+				_, err = cimfs.MountVerifiedBlockCIM(layerCIMs[0], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID, layerDigests[0])
 				if err != nil {
-					return fmt.Errorf("error mounting merged block cims: %w", err)
+					return fmt.Errorf("error mounting verified block cim: %w", err)
 				}
 			}
 

internal/oci/uvm.go

@@ -330,6 +330,7 @@ func specToUVMCreateOptionsCommon(ctx context.Context, opts *uvm.Options, s *spe
 		opts.NumaProcessorCounts)
 	opts.NumaMemoryBlocksCounts = ParseAnnotationCommaSeparatedUint64(ctx, s.Annotations, annotations.NumaCountOfMemoryBlocks,
 		opts.NumaMemoryBlocksCounts)
+	opts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, opts.ConsolePipe)
 
 	maps.Copy(opts.AdditionalHyperVConfig, parseHVSocketServiceTable(ctx, s.Annotations))
 
@@ -376,7 +377,6 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.LCOWReferenceInfoFile, lopts.UVMReferenceInfoFile)
 		lopts.KernelBootOptions = ParseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions)
 		lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService)
-		lopts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, lopts.ConsolePipe)
 		handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts)
 		handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts)
 		handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, lopts)

internal/uvm/create_wcow.go

@@ -66,7 +66,7 @@ func defaultConfidentialWCOWOSBootFilesPath() string {
 }
 
 func GetDefaultConfidentialVMGSPath() string {
-	return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.vmgs")
+	return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.snp.vmgs")
 }
 
 func GetDefaultConfidentialBootCIMPath() string {

internal/winapi/cimfs.go

@@ -59,3 +59,4 @@ type CimFsImagePath struct {
 //sys CimSealImage(blockCimPath string, hashSize *uint64, fixedHeaderSize *uint64, hash *byte) (hr error) = cimfs.CimSealImage?
 //sys CimGetVerificationInformation(blockCimPath string, isSealed *uint32, hashSize *uint64, signatureSize *uint64, fixedHeaderSize *uint64, hash *byte, signature *byte) (hr error) = cimfs.CimGetVerificationInformation?
 //sys CimMountVerifiedImage(imagePath string, fsName string, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) = cimfs.CimMountVerifiedImage?
+//sys CimMergeMountVerifiedImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) = cimfs.CimMergeMountVerifiedImage

pkg/cimfs/cim_test.go

@@ -679,33 +679,6 @@ func TestMergedLinksInMergedBlockCIMs(rootT *testing.T) {
 	}
 }
 
-func TestVerifiedSingleFileBlockCIM(t *testing.T) {
-	if !IsVerifiedCimSupported() {
-		t.Skipf("verified CIMs are not supported")
-	}
-
-	// contents to write to the CIM
-	testContents := []tuple{
-		{"foo.txt", []byte("foo1"), false},
-		{"bar.txt", []byte("bar"), false},
-	}
-
-	root := t.TempDir()
-	blockPath := filepath.Join(root, "layer.bcim")
-	tc := &testVerifiedBlockCIM{
-		BlockCIM: BlockCIM{
-			Type:      BlockCIMTypeSingleFile,
-			BlockPath: blockPath,
-			CimName:   "layer.cim",
-		}}
-	writer := openNewCIM(t, tc)
-	writeCIM(t, writer, testContents)
-
-	mountvol := mountCIM(t, tc, CimMountVerifiedCim|CimMountSingleFileCim)
-
-	compareContent(t, mountvol, testContents)
-}
-
 func TestVerifiedSingleFileBlockCIMMount(t *testing.T) {
 	if !IsVerifiedCimSupported() {
 		t.Skipf("verified CIMs are not supported")
@@ -794,3 +767,102 @@ func TestVerifiedSingleFileBlockCIMMountReadFailure(t *testing.T) {
 		t.Fatalf("expected integrity violation error")
 	}
 }
+
+func TestMergedVerifiedBlockCIMs(rootT *testing.T) {
+	if !IsVerifiedCimSupported() {
+		rootT.Skipf("verified BlockCIMs are not supported")
+	}
+
+	// A slice of 3 slices, 1 slice for contents of each CIM
+	testContents := [][]tuple{
+		{{"foo.txt", []byte("foo1"), false}},
+		{{"bar.txt", []byte("bar"), false}},
+		{{"foo.txt", []byte("foo2"), false}},
+	}
+	// create 3 separate block CIMs
+	nCIMs := len(testContents)
+
+	// test merging for both SingleFile & BlockDevice type of block CIMs
+	type testBlock struct {
+		name               string
+		blockType          BlockCIMType
+		mountFlag          uint32
+		blockPathGenerator func(t *testing.T, dir string) string
+	}
+
+	tests := []testBlock{
+		{
+			name:      "single file",
+			blockType: BlockCIMTypeSingleFile,
+			mountFlag: CimMountSingleFileCim,
+			blockPathGenerator: func(t *testing.T, dir string) string {
+				t.Helper()
+				return filepath.Join(dir, "layer.bcim")
+			},
+		},
+		{
+			name:      "block device",
+			blockType: BlockCIMTypeDevice,
+			mountFlag: CimMountBlockDeviceCim,
+			blockPathGenerator: func(t *testing.T, dir string) string {
+				t.Helper()
+				return createBlockDevice(t, dir)
+			},
+		},
+	}
+
+	for _, test := range tests {
+		rootT.Run(test.name, func(t *testing.T) {
+			sourceCIMs := make([]*BlockCIM, 0, nCIMs)
+			for i := 0; i < nCIMs; i++ {
+				root := t.TempDir()
+				blockPath := test.blockPathGenerator(t, root)
+				tc := &testVerifiedBlockCIM{
+					BlockCIM: BlockCIM{
+						Type:      test.blockType,
+						BlockPath: blockPath,
+						CimName:   "layer.cim",
+					}}
+				writer := openNewCIM(t, tc)
+				writeCIM(t, writer, testContents[i])
+				sourceCIMs = append(sourceCIMs, &tc.BlockCIM)
+			}
+
+			mergedBlockPath := test.blockPathGenerator(t, t.TempDir())
+			// prepare a merged CIM
+			mergedCIM := &BlockCIM{
+				Type:      test.blockType,
+				BlockPath: mergedBlockPath,
+				CimName:   "merged.cim",
+			}
+
+			if err := MergeBlockCIMsWithOpts(context.Background(), mergedCIM, sourceCIMs, WithDataIntegrity()); err != nil {
+				t.Fatalf("failed to merge block CIMs: %s", err)
+			}
+
+			rootHash, err := GetVerificationInfo(mergedBlockPath)
+			if err != nil {
+				t.Fatalf("failed to get verification info: %s", err)
+			}
+
+			// mount and read the contents of the cim
+			volumeGUID, err := guid.NewV4()
+			if err != nil {
+				t.Fatalf("generate cim mount GUID: %s", err)
+			}
+
+			mountvol, err := MountMergedVerifiedBlockCIMs(mergedCIM, sourceCIMs, test.mountFlag, volumeGUID, rootHash)
+			if err != nil {
+				t.Fatalf("failed to mount merged block CIMs: %s\n", err)
+			}
+			defer func() {
+				if err := Unmount(mountvol); err != nil {
+					t.Logf("CIM unmount failed: %s", err)
+				}
+			}()
+			// since we are merging, only 1 foo.txt (from the 1st CIM) should
+			// show up
+			compareContent(t, mountvol, []tuple{testContents[0][0], testContents[1][0]})
+		})
+	}
+}

pkg/cimfs/cim_writer_windows.go

@@ -346,7 +346,7 @@ func (c *CimFsWriter) Close() (err error) {
 	}
 	if c.sealOnClose {
 		if err = sealBlockCIM(filepath.Dir(c.name)); err != nil {
-			return &OpError{Cim: c.name, Op: "seal", Err: err}
+			return &OpError{Cim: filepath.Dir(c.name), Op: "seal", Err: err}
 		}
 	}
 	return nil
@@ -506,6 +506,10 @@ func MergeBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM) (err error) {
 func sealBlockCIM(blockPath string) error {
 	var hashSize, fixedHeaderSize uint64
 	hashBuf := make([]byte, cimHashSize)
+
+	// the blockPath could be a path to a block device or a file. In either case there should be no trailing backslash.
+	blockPath = strings.TrimSuffix(blockPath, "\\")
+
 	if err := winapi.CimSealImage(blockPath, &hashSize, &fixedHeaderSize, &hashBuf[0]); err != nil {
 		return fmt.Errorf("failed to seal block CIM: %w", err)
 	} else if hashSize != cimHashSize {

pkg/cimfs/mount_cim.go

@@ -151,3 +151,63 @@ func MountVerifiedBlockCIM(bCIM *BlockCIM, mountFlags uint32, volumeGUID guid.GU
 	}
 	return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil
 }
+
+// MountMergedVerifiedBlockCIMs mounts the given merged verified BlockCIM (usually created
+// with `MergeBlockCIMs`) at a volume with given GUID, with the given root hash. The
+// `sourceCIMs` MUST be identical to the `sourceCIMs` passed to `MergeBlockCIMs` when
+// creating this merged CIM. The root hash is usually returned when the CIM is sealed or
+// the root hash can be queried from a block CIM. In case of merged CIMs, the root hash of
+// the merged CIM should be passed here. Every read on the mounted volume will be verified
+// to match against the provided root hash if it doesn't, the read will fail.  The source
+// CIMs and the merged CIM MUST have been created with the verified creation flag.
+func MountMergedVerifiedBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM, mountFlags uint32, volumeGUID guid.GUID, rootHash []byte) (string, error) {
+	if !IsVerifiedCimSupported() {
+		return "", fmt.Errorf("verified CIMs aren't supported on this OS version")
+	} else if len(sourceCIMs) < 2 {
+		return "", fmt.Errorf("need at least 2 source CIMs, got %d: %w", len(sourceCIMs), os.ErrInvalid)
+	} else if len(rootHash) != cimHashSize {
+		return "", fmt.Errorf("unexpected root hash size %d, expected size is %d", len(rootHash), cimHashSize)
+	}
+
+	switch mergedCIM.Type {
+	case BlockCIMTypeDevice:
+		mountFlags |= CimMountBlockDeviceCim
+	case BlockCIMTypeSingleFile:
+		mountFlags |= CimMountSingleFileCim
+	default:
+		return "", fmt.Errorf("invalid block CIM type `%d`", mergedCIM.Type)
+	}
+
+	for _, sCIM := range sourceCIMs {
+		if sCIM.Type != mergedCIM.Type {
+			return "", fmt.Errorf("source CIM (%s) type doesn't match with merged CIM type: %w", sCIM.String(), os.ErrInvalid)
+		}
+	}
+
+	// win32 mount merged CIM API expects an array of all CIMs. 0th entry in the array
+	// should be the merged CIM. All remaining entries should be the source CIM paths
+	// in the same order that was used while creating the merged CIM.
+	allcims := append([]*BlockCIM{mergedCIM}, sourceCIMs...)
+	cimsToMerge := []winapi.CimFsImagePath{}
+	for _, bcim := range allcims {
+		// Trailing backslashes cause problems-remove those
+		imageDir, err := windows.UTF16PtrFromString(strings.TrimRight(bcim.BlockPath, `\`))
+		if err != nil {
+			return "", fmt.Errorf("convert string to utf16: %w", err)
+		}
+		cimName, err := windows.UTF16PtrFromString(bcim.CimName)
+		if err != nil {
+			return "", fmt.Errorf("convert string to utf16: %w", err)
+		}
+
+		cimsToMerge = append(cimsToMerge, winapi.CimFsImagePath{
+			ImageDir:  imageDir,
+			ImageName: cimName,
+		})
+	}
+
+	if err := winapi.CimMergeMountVerifiedImage(uint32(len(cimsToMerge)), &cimsToMerge[0], mountFlags, &volumeGUID, cimHashSize, &rootHash[0]); err != nil {
+		return "", &MountError{Cim: filepath.Join(mergedCIM.BlockPath, mergedCIM.CimName), Op: "MountMergedVerified", Err: err}
+	}
+	return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil
+}