From 36f0201e0280f20e81e9961fb15a79d648c76dfb Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 1 Oct 2025 15:11:04 -0400 Subject: [PATCH 1/2] Make copies of the VMGS for each confidential pod Even though VMGS files aren't written to, we can't share the same file across multiple pods because HCS seems to be taking an exclusive lock on the file. To handle this, we make copies of the VMGS file (similar to how we make copies of the EFI VHD) per pod. This commit also changes the name of the default VMGS file to `cwcow.snp.vmgs` to better convey that by default we run in SNP mode. If a different VMGS file is desired, `"io.microsoft.virtualmachine.wcow.gueststatefile"` annotation can be used to override that. This commit also adds the code to handle the UVM console pipe annotation, if provided, for windows pods. Signed-off-by: Amit Barve --- cmd/containerd-shim-runhcs-v1/pod.go | 13 ++++++++++++- internal/oci/uvm.go | 2 +- internal/uvm/create_wcow.go | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index b41e4a1b15..6a8f5ef62c 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -37,6 +37,7 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs if s.Windows != nil { layerFolders = s.Windows.LayerFolders } + log.G(ctx).WithField("options", log.Format(ctx, *wopts)).Debug("initialize WCOW boot files") wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, rootfs, layerFolders) if err != nil { @@ -50,6 +51,16 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs // we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath() wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath() + + // make a copy of the vmgs file as the same vmgs can not be used by multiple pods in parallel + // TODO(ambarve): for C-LCOW we make a copy in the bundle directory, is it better + // to use the bundle directory instead of the snapshot directory? + vmgsCopyPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.GuestStateFilePath)) + if err := copyfile.CopyFile(ctx, wopts.GuestStateFilePath, vmgsCopyPath, false); err != nil { + return fmt.Errorf("failed to make a copy of VMGS: %w", err) + } + wopts.GuestStateFilePath = vmgsCopyPath + } else if wopts.BootFiles.BootType == uvm.BlockCIMBoot { // Supporting hyperv isolation with block CIM layers requires changes in // the image pull path to prepare the EFI VHD. But more importantly, since both the @@ -69,7 +80,7 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs // UVM. writableEFIVHDPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.BootFiles.BlockCIMFiles.EFIVHDPath)) if err := copyfile.CopyFile(ctx, wopts.BootFiles.BlockCIMFiles.EFIVHDPath, writableEFIVHDPath, false); err != nil { - return fmt.Errorf("failed to copy EFI VHD at %s: %w", writableEFIVHDPath, err) + return fmt.Errorf("failed to copy EFI VHD: %w", err) } wopts.BootFiles.BlockCIMFiles.EFIVHDPath = writableEFIVHDPath } diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 889e31f09e..204456da46 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -330,6 +330,7 @@ func specToUVMCreateOptionsCommon(ctx context.Context, opts *uvm.Options, s *spe opts.NumaProcessorCounts) opts.NumaMemoryBlocksCounts = ParseAnnotationCommaSeparatedUint64(ctx, s.Annotations, annotations.NumaCountOfMemoryBlocks, opts.NumaMemoryBlocksCounts) + opts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, opts.ConsolePipe) maps.Copy(opts.AdditionalHyperVConfig, parseHVSocketServiceTable(ctx, s.Annotations)) @@ -376,7 +377,6 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.LCOWReferenceInfoFile, lopts.UVMReferenceInfoFile) lopts.KernelBootOptions = ParseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService) - lopts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, lopts.ConsolePipe) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, lopts) diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 6798d23fa5..da105960eb 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -66,7 +66,7 @@ func defaultConfidentialWCOWOSBootFilesPath() string { } func GetDefaultConfidentialVMGSPath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.vmgs") + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.snp.vmgs") } func GetDefaultConfidentialBootCIMPath() string { From 2b246a16375e2544d59d53fb520d9bc3886ab4de Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Wed, 1 Oct 2025 15:12:22 -0400 Subject: [PATCH 2/2] Support for mounting merged verified CIMs This commit adds the new CimFS API that mounts the merged verified CIMs. Merge mounting verified CIM requires passing the root hash of the merged CIM. All reads happening on this mounted verified CIM will then be compared with this root hash for integrity check. This commit also adds the tests for the same. (A test that simply mounts verified CIMs was removed as that is being tested as a part of other verified CIM tests.) GCS sidecar code is also updated to use the newly added verified CIM mount APIs. Signed-off-by: Amit Barve --- internal/gcs-sidecar/handlers.go | 7 +- internal/winapi/cimfs.go | 1 + internal/winapi/zsyscall_windows.go | 12 +++ pkg/cimfs/cim_test.go | 126 ++++++++++++++++++++++------ pkg/cimfs/cim_writer_windows.go | 6 +- pkg/cimfs/mount_cim.go | 60 +++++++++++++ 6 files changed, 180 insertions(+), 32 deletions(-) diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go index 7660e6d069..e3f423faf8 100644 --- a/internal/gcs-sidecar/handlers.go +++ b/internal/gcs-sidecar/handlers.go @@ -677,15 +677,14 @@ func (b *Bridge) modifySettings(req *request) (err error) { } if len(layerCIMs) > 1 { - // Get the topmost merge CIM and invoke the MountMergedBlockCIMs - _, err := cimfs.MountMergedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID) + _, err = cimfs.MountMergedVerifiedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID, layerDigests[0]) if err != nil { return fmt.Errorf("error mounting multilayer block cims: %w", err) } } else { - _, err := cimfs.Mount(filepath.Join(layerCIMs[0].BlockPath, layerCIMs[0].CimName), wcowBlockCimMounts.VolumeGUID, wcowBlockCimMounts.MountFlags) + _, err = cimfs.MountVerifiedBlockCIM(layerCIMs[0], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID, layerDigests[0]) if err != nil { - return fmt.Errorf("error mounting merged block cims: %w", err) + return fmt.Errorf("error mounting verified block cim: %w", err) } } diff --git a/internal/winapi/cimfs.go b/internal/winapi/cimfs.go index cc3d254120..56c7b442fb 100644 --- a/internal/winapi/cimfs.go +++ b/internal/winapi/cimfs.go @@ -59,3 +59,4 @@ type CimFsImagePath struct { //sys CimSealImage(blockCimPath string, hashSize *uint64, fixedHeaderSize *uint64, hash *byte) (hr error) = cimfs.CimSealImage? //sys CimGetVerificationInformation(blockCimPath string, isSealed *uint32, hashSize *uint64, signatureSize *uint64, fixedHeaderSize *uint64, hash *byte, signature *byte) (hr error) = cimfs.CimGetVerificationInformation? //sys CimMountVerifiedImage(imagePath string, fsName string, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) = cimfs.CimMountVerifiedImage? +//sys CimMergeMountVerifiedImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) = cimfs.CimMergeMountVerifiedImage diff --git a/internal/winapi/zsyscall_windows.go b/internal/winapi/zsyscall_windows.go index 761e09bbd4..97519200f4 100644 --- a/internal/winapi/zsyscall_windows.go +++ b/internal/winapi/zsyscall_windows.go @@ -73,6 +73,7 @@ var ( procCimDismountImage = modcimfs.NewProc("CimDismountImage") procCimGetVerificationInformation = modcimfs.NewProc("CimGetVerificationInformation") procCimMergeMountImage = modcimfs.NewProc("CimMergeMountImage") + procCimMergeMountVerifiedImage = modcimfs.NewProc("CimMergeMountVerifiedImage") procCimMountImage = modcimfs.NewProc("CimMountImage") procCimMountVerifiedImage = modcimfs.NewProc("CimMountVerifiedImage") procCimSealImage = modcimfs.NewProc("CimSealImage") @@ -562,6 +563,17 @@ func CimMergeMountImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, f return } +func CimMergeMountVerifiedImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) { + r0, _, _ := syscall.SyscallN(procCimMergeMountVerifiedImage.Addr(), uintptr(numCimPaths), uintptr(unsafe.Pointer(backingImagePaths)), uintptr(flags), uintptr(unsafe.Pointer(volumeID)), uintptr(hashSize), uintptr(unsafe.Pointer(hash))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + func CimMountImage(imagePath string, fsName string, flags uint32, volumeID *g) (hr error) { var _p0 *uint16 _p0, hr = syscall.UTF16PtrFromString(imagePath) diff --git a/pkg/cimfs/cim_test.go b/pkg/cimfs/cim_test.go index bd744a4452..f469798c59 100644 --- a/pkg/cimfs/cim_test.go +++ b/pkg/cimfs/cim_test.go @@ -679,33 +679,6 @@ func TestMergedLinksInMergedBlockCIMs(rootT *testing.T) { } } -func TestVerifiedSingleFileBlockCIM(t *testing.T) { - if !IsVerifiedCimSupported() { - t.Skipf("verified CIMs are not supported") - } - - // contents to write to the CIM - testContents := []tuple{ - {"foo.txt", []byte("foo1"), false}, - {"bar.txt", []byte("bar"), false}, - } - - root := t.TempDir() - blockPath := filepath.Join(root, "layer.bcim") - tc := &testVerifiedBlockCIM{ - BlockCIM: BlockCIM{ - Type: BlockCIMTypeSingleFile, - BlockPath: blockPath, - CimName: "layer.cim", - }} - writer := openNewCIM(t, tc) - writeCIM(t, writer, testContents) - - mountvol := mountCIM(t, tc, CimMountVerifiedCim|CimMountSingleFileCim) - - compareContent(t, mountvol, testContents) -} - func TestVerifiedSingleFileBlockCIMMount(t *testing.T) { if !IsVerifiedCimSupported() { t.Skipf("verified CIMs are not supported") @@ -794,3 +767,102 @@ func TestVerifiedSingleFileBlockCIMMountReadFailure(t *testing.T) { t.Fatalf("expected integrity violation error") } } + +func TestMergedVerifiedBlockCIMs(rootT *testing.T) { + if !IsVerifiedCimSupported() { + rootT.Skipf("verified BlockCIMs are not supported") + } + + // A slice of 3 slices, 1 slice for contents of each CIM + testContents := [][]tuple{ + {{"foo.txt", []byte("foo1"), false}}, + {{"bar.txt", []byte("bar"), false}}, + {{"foo.txt", []byte("foo2"), false}}, + } + // create 3 separate block CIMs + nCIMs := len(testContents) + + // test merging for both SingleFile & BlockDevice type of block CIMs + type testBlock struct { + name string + blockType BlockCIMType + mountFlag uint32 + blockPathGenerator func(t *testing.T, dir string) string + } + + tests := []testBlock{ + { + name: "single file", + blockType: BlockCIMTypeSingleFile, + mountFlag: CimMountSingleFileCim, + blockPathGenerator: func(t *testing.T, dir string) string { + t.Helper() + return filepath.Join(dir, "layer.bcim") + }, + }, + { + name: "block device", + blockType: BlockCIMTypeDevice, + mountFlag: CimMountBlockDeviceCim, + blockPathGenerator: func(t *testing.T, dir string) string { + t.Helper() + return createBlockDevice(t, dir) + }, + }, + } + + for _, test := range tests { + rootT.Run(test.name, func(t *testing.T) { + sourceCIMs := make([]*BlockCIM, 0, nCIMs) + for i := 0; i < nCIMs; i++ { + root := t.TempDir() + blockPath := test.blockPathGenerator(t, root) + tc := &testVerifiedBlockCIM{ + BlockCIM: BlockCIM{ + Type: test.blockType, + BlockPath: blockPath, + CimName: "layer.cim", + }} + writer := openNewCIM(t, tc) + writeCIM(t, writer, testContents[i]) + sourceCIMs = append(sourceCIMs, &tc.BlockCIM) + } + + mergedBlockPath := test.blockPathGenerator(t, t.TempDir()) + // prepare a merged CIM + mergedCIM := &BlockCIM{ + Type: test.blockType, + BlockPath: mergedBlockPath, + CimName: "merged.cim", + } + + if err := MergeBlockCIMsWithOpts(context.Background(), mergedCIM, sourceCIMs, WithDataIntegrity()); err != nil { + t.Fatalf("failed to merge block CIMs: %s", err) + } + + rootHash, err := GetVerificationInfo(mergedBlockPath) + if err != nil { + t.Fatalf("failed to get verification info: %s", err) + } + + // mount and read the contents of the cim + volumeGUID, err := guid.NewV4() + if err != nil { + t.Fatalf("generate cim mount GUID: %s", err) + } + + mountvol, err := MountMergedVerifiedBlockCIMs(mergedCIM, sourceCIMs, test.mountFlag, volumeGUID, rootHash) + if err != nil { + t.Fatalf("failed to mount merged block CIMs: %s\n", err) + } + defer func() { + if err := Unmount(mountvol); err != nil { + t.Logf("CIM unmount failed: %s", err) + } + }() + // since we are merging, only 1 foo.txt (from the 1st CIM) should + // show up + compareContent(t, mountvol, []tuple{testContents[0][0], testContents[1][0]}) + }) + } +} diff --git a/pkg/cimfs/cim_writer_windows.go b/pkg/cimfs/cim_writer_windows.go index 62e45841f7..dec9cf07b5 100644 --- a/pkg/cimfs/cim_writer_windows.go +++ b/pkg/cimfs/cim_writer_windows.go @@ -346,7 +346,7 @@ func (c *CimFsWriter) Close() (err error) { } if c.sealOnClose { if err = sealBlockCIM(filepath.Dir(c.name)); err != nil { - return &OpError{Cim: c.name, Op: "seal", Err: err} + return &OpError{Cim: filepath.Dir(c.name), Op: "seal", Err: err} } } return nil @@ -506,6 +506,10 @@ func MergeBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM) (err error) { func sealBlockCIM(blockPath string) error { var hashSize, fixedHeaderSize uint64 hashBuf := make([]byte, cimHashSize) + + // the blockPath could be a path to a block device or a file. In either case there should be no trailing backslash. + blockPath = strings.TrimSuffix(blockPath, "\\") + if err := winapi.CimSealImage(blockPath, &hashSize, &fixedHeaderSize, &hashBuf[0]); err != nil { return fmt.Errorf("failed to seal block CIM: %w", err) } else if hashSize != cimHashSize { diff --git a/pkg/cimfs/mount_cim.go b/pkg/cimfs/mount_cim.go index 12fa2da036..475c0957ed 100644 --- a/pkg/cimfs/mount_cim.go +++ b/pkg/cimfs/mount_cim.go @@ -151,3 +151,63 @@ func MountVerifiedBlockCIM(bCIM *BlockCIM, mountFlags uint32, volumeGUID guid.GU } return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil } + +// MountMergedVerifiedBlockCIMs mounts the given merged verified BlockCIM (usually created +// with `MergeBlockCIMs`) at a volume with given GUID, with the given root hash. The +// `sourceCIMs` MUST be identical to the `sourceCIMs` passed to `MergeBlockCIMs` when +// creating this merged CIM. The root hash is usually returned when the CIM is sealed or +// the root hash can be queried from a block CIM. In case of merged CIMs, the root hash of +// the merged CIM should be passed here. Every read on the mounted volume will be verified +// to match against the provided root hash if it doesn't, the read will fail. The source +// CIMs and the merged CIM MUST have been created with the verified creation flag. +func MountMergedVerifiedBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM, mountFlags uint32, volumeGUID guid.GUID, rootHash []byte) (string, error) { + if !IsVerifiedCimSupported() { + return "", fmt.Errorf("verified CIMs aren't supported on this OS version") + } else if len(sourceCIMs) < 2 { + return "", fmt.Errorf("need at least 2 source CIMs, got %d: %w", len(sourceCIMs), os.ErrInvalid) + } else if len(rootHash) != cimHashSize { + return "", fmt.Errorf("unexpected root hash size %d, expected size is %d", len(rootHash), cimHashSize) + } + + switch mergedCIM.Type { + case BlockCIMTypeDevice: + mountFlags |= CimMountBlockDeviceCim + case BlockCIMTypeSingleFile: + mountFlags |= CimMountSingleFileCim + default: + return "", fmt.Errorf("invalid block CIM type `%d`", mergedCIM.Type) + } + + for _, sCIM := range sourceCIMs { + if sCIM.Type != mergedCIM.Type { + return "", fmt.Errorf("source CIM (%s) type doesn't match with merged CIM type: %w", sCIM.String(), os.ErrInvalid) + } + } + + // win32 mount merged CIM API expects an array of all CIMs. 0th entry in the array + // should be the merged CIM. All remaining entries should be the source CIM paths + // in the same order that was used while creating the merged CIM. + allcims := append([]*BlockCIM{mergedCIM}, sourceCIMs...) + cimsToMerge := []winapi.CimFsImagePath{} + for _, bcim := range allcims { + // Trailing backslashes cause problems-remove those + imageDir, err := windows.UTF16PtrFromString(strings.TrimRight(bcim.BlockPath, `\`)) + if err != nil { + return "", fmt.Errorf("convert string to utf16: %w", err) + } + cimName, err := windows.UTF16PtrFromString(bcim.CimName) + if err != nil { + return "", fmt.Errorf("convert string to utf16: %w", err) + } + + cimsToMerge = append(cimsToMerge, winapi.CimFsImagePath{ + ImageDir: imageDir, + ImageName: cimName, + }) + } + + if err := winapi.CimMergeMountVerifiedImage(uint32(len(cimsToMerge)), &cimsToMerge[0], mountFlags, &volumeGUID, cimHashSize, &rootHash[0]); err != nil { + return "", &MountError{Cim: filepath.Join(mergedCIM.BlockPath, mergedCIM.CimName), Op: "MountMergedVerified", Err: err} + } + return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil +}