diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index b41e4a1b15..6a8f5ef62c 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -37,6 +37,7 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs if s.Windows != nil { layerFolders = s.Windows.LayerFolders } + log.G(ctx).WithField("options", log.Format(ctx, *wopts)).Debug("initialize WCOW boot files") wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, rootfs, layerFolders) if err != nil { @@ -50,6 +51,16 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs // we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath() wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath() + + // make a copy of the vmgs file as the same vmgs can not be used by multiple pods in parallel + // TODO(ambarve): for C-LCOW we make a copy in the bundle directory, is it better + // to use the bundle directory instead of the snapshot directory? + vmgsCopyPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.GuestStateFilePath)) + if err := copyfile.CopyFile(ctx, wopts.GuestStateFilePath, vmgsCopyPath, false); err != nil { + return fmt.Errorf("failed to make a copy of VMGS: %w", err) + } + wopts.GuestStateFilePath = vmgsCopyPath + } else if wopts.BootFiles.BootType == uvm.BlockCIMBoot { // Supporting hyperv isolation with block CIM layers requires changes in // the image pull path to prepare the EFI VHD. But more importantly, since both the @@ -69,7 +80,7 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs // UVM. writableEFIVHDPath := filepath.Join(filepath.Dir(wopts.BootFiles.BlockCIMFiles.ScratchVHDPath), filepath.Base(wopts.BootFiles.BlockCIMFiles.EFIVHDPath)) if err := copyfile.CopyFile(ctx, wopts.BootFiles.BlockCIMFiles.EFIVHDPath, writableEFIVHDPath, false); err != nil { - return fmt.Errorf("failed to copy EFI VHD at %s: %w", writableEFIVHDPath, err) + return fmt.Errorf("failed to copy EFI VHD: %w", err) } wopts.BootFiles.BlockCIMFiles.EFIVHDPath = writableEFIVHDPath } diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go index 7660e6d069..e3f423faf8 100644 --- a/internal/gcs-sidecar/handlers.go +++ b/internal/gcs-sidecar/handlers.go @@ -677,15 +677,14 @@ func (b *Bridge) modifySettings(req *request) (err error) { } if len(layerCIMs) > 1 { - // Get the topmost merge CIM and invoke the MountMergedBlockCIMs - _, err := cimfs.MountMergedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID) + _, err = cimfs.MountMergedVerifiedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID, layerDigests[0]) if err != nil { return fmt.Errorf("error mounting multilayer block cims: %w", err) } } else { - _, err := cimfs.Mount(filepath.Join(layerCIMs[0].BlockPath, layerCIMs[0].CimName), wcowBlockCimMounts.VolumeGUID, wcowBlockCimMounts.MountFlags) + _, err = cimfs.MountVerifiedBlockCIM(layerCIMs[0], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID, layerDigests[0]) if err != nil { - return fmt.Errorf("error mounting merged block cims: %w", err) + return fmt.Errorf("error mounting verified block cim: %w", err) } } diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 889e31f09e..204456da46 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -330,6 +330,7 @@ func specToUVMCreateOptionsCommon(ctx context.Context, opts *uvm.Options, s *spe opts.NumaProcessorCounts) opts.NumaMemoryBlocksCounts = ParseAnnotationCommaSeparatedUint64(ctx, s.Annotations, annotations.NumaCountOfMemoryBlocks, opts.NumaMemoryBlocksCounts) + opts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, opts.ConsolePipe) maps.Copy(opts.AdditionalHyperVConfig, parseHVSocketServiceTable(ctx, s.Annotations)) @@ -376,7 +377,6 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( lopts.UVMReferenceInfoFile = ParseAnnotationsString(s.Annotations, annotations.LCOWReferenceInfoFile, lopts.UVMReferenceInfoFile) lopts.KernelBootOptions = ParseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService) - lopts.ConsolePipe = ParseAnnotationsString(s.Annotations, iannotations.UVMConsolePipe, lopts.ConsolePipe) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, lopts) diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 6798d23fa5..da105960eb 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -66,7 +66,7 @@ func defaultConfidentialWCOWOSBootFilesPath() string { } func GetDefaultConfidentialVMGSPath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.vmgs") + return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.snp.vmgs") } func GetDefaultConfidentialBootCIMPath() string { diff --git a/internal/winapi/cimfs.go b/internal/winapi/cimfs.go index cc3d254120..56c7b442fb 100644 --- a/internal/winapi/cimfs.go +++ b/internal/winapi/cimfs.go @@ -59,3 +59,4 @@ type CimFsImagePath struct { //sys CimSealImage(blockCimPath string, hashSize *uint64, fixedHeaderSize *uint64, hash *byte) (hr error) = cimfs.CimSealImage? //sys CimGetVerificationInformation(blockCimPath string, isSealed *uint32, hashSize *uint64, signatureSize *uint64, fixedHeaderSize *uint64, hash *byte, signature *byte) (hr error) = cimfs.CimGetVerificationInformation? //sys CimMountVerifiedImage(imagePath string, fsName string, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) = cimfs.CimMountVerifiedImage? +//sys CimMergeMountVerifiedImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) = cimfs.CimMergeMountVerifiedImage diff --git a/internal/winapi/zsyscall_windows.go b/internal/winapi/zsyscall_windows.go index 761e09bbd4..97519200f4 100644 --- a/internal/winapi/zsyscall_windows.go +++ b/internal/winapi/zsyscall_windows.go @@ -73,6 +73,7 @@ var ( procCimDismountImage = modcimfs.NewProc("CimDismountImage") procCimGetVerificationInformation = modcimfs.NewProc("CimGetVerificationInformation") procCimMergeMountImage = modcimfs.NewProc("CimMergeMountImage") + procCimMergeMountVerifiedImage = modcimfs.NewProc("CimMergeMountVerifiedImage") procCimMountImage = modcimfs.NewProc("CimMountImage") procCimMountVerifiedImage = modcimfs.NewProc("CimMountVerifiedImage") procCimSealImage = modcimfs.NewProc("CimSealImage") @@ -562,6 +563,17 @@ func CimMergeMountImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, f return } +func CimMergeMountVerifiedImage(numCimPaths uint32, backingImagePaths *CimFsImagePath, flags uint32, volumeID *g, hashSize uint16, hash *byte) (hr error) { + r0, _, _ := syscall.SyscallN(procCimMergeMountVerifiedImage.Addr(), uintptr(numCimPaths), uintptr(unsafe.Pointer(backingImagePaths)), uintptr(flags), uintptr(unsafe.Pointer(volumeID)), uintptr(hashSize), uintptr(unsafe.Pointer(hash))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + func CimMountImage(imagePath string, fsName string, flags uint32, volumeID *g) (hr error) { var _p0 *uint16 _p0, hr = syscall.UTF16PtrFromString(imagePath) diff --git a/pkg/cimfs/cim_test.go b/pkg/cimfs/cim_test.go index bd744a4452..f469798c59 100644 --- a/pkg/cimfs/cim_test.go +++ b/pkg/cimfs/cim_test.go @@ -679,33 +679,6 @@ func TestMergedLinksInMergedBlockCIMs(rootT *testing.T) { } } -func TestVerifiedSingleFileBlockCIM(t *testing.T) { - if !IsVerifiedCimSupported() { - t.Skipf("verified CIMs are not supported") - } - - // contents to write to the CIM - testContents := []tuple{ - {"foo.txt", []byte("foo1"), false}, - {"bar.txt", []byte("bar"), false}, - } - - root := t.TempDir() - blockPath := filepath.Join(root, "layer.bcim") - tc := &testVerifiedBlockCIM{ - BlockCIM: BlockCIM{ - Type: BlockCIMTypeSingleFile, - BlockPath: blockPath, - CimName: "layer.cim", - }} - writer := openNewCIM(t, tc) - writeCIM(t, writer, testContents) - - mountvol := mountCIM(t, tc, CimMountVerifiedCim|CimMountSingleFileCim) - - compareContent(t, mountvol, testContents) -} - func TestVerifiedSingleFileBlockCIMMount(t *testing.T) { if !IsVerifiedCimSupported() { t.Skipf("verified CIMs are not supported") @@ -794,3 +767,102 @@ func TestVerifiedSingleFileBlockCIMMountReadFailure(t *testing.T) { t.Fatalf("expected integrity violation error") } } + +func TestMergedVerifiedBlockCIMs(rootT *testing.T) { + if !IsVerifiedCimSupported() { + rootT.Skipf("verified BlockCIMs are not supported") + } + + // A slice of 3 slices, 1 slice for contents of each CIM + testContents := [][]tuple{ + {{"foo.txt", []byte("foo1"), false}}, + {{"bar.txt", []byte("bar"), false}}, + {{"foo.txt", []byte("foo2"), false}}, + } + // create 3 separate block CIMs + nCIMs := len(testContents) + + // test merging for both SingleFile & BlockDevice type of block CIMs + type testBlock struct { + name string + blockType BlockCIMType + mountFlag uint32 + blockPathGenerator func(t *testing.T, dir string) string + } + + tests := []testBlock{ + { + name: "single file", + blockType: BlockCIMTypeSingleFile, + mountFlag: CimMountSingleFileCim, + blockPathGenerator: func(t *testing.T, dir string) string { + t.Helper() + return filepath.Join(dir, "layer.bcim") + }, + }, + { + name: "block device", + blockType: BlockCIMTypeDevice, + mountFlag: CimMountBlockDeviceCim, + blockPathGenerator: func(t *testing.T, dir string) string { + t.Helper() + return createBlockDevice(t, dir) + }, + }, + } + + for _, test := range tests { + rootT.Run(test.name, func(t *testing.T) { + sourceCIMs := make([]*BlockCIM, 0, nCIMs) + for i := 0; i < nCIMs; i++ { + root := t.TempDir() + blockPath := test.blockPathGenerator(t, root) + tc := &testVerifiedBlockCIM{ + BlockCIM: BlockCIM{ + Type: test.blockType, + BlockPath: blockPath, + CimName: "layer.cim", + }} + writer := openNewCIM(t, tc) + writeCIM(t, writer, testContents[i]) + sourceCIMs = append(sourceCIMs, &tc.BlockCIM) + } + + mergedBlockPath := test.blockPathGenerator(t, t.TempDir()) + // prepare a merged CIM + mergedCIM := &BlockCIM{ + Type: test.blockType, + BlockPath: mergedBlockPath, + CimName: "merged.cim", + } + + if err := MergeBlockCIMsWithOpts(context.Background(), mergedCIM, sourceCIMs, WithDataIntegrity()); err != nil { + t.Fatalf("failed to merge block CIMs: %s", err) + } + + rootHash, err := GetVerificationInfo(mergedBlockPath) + if err != nil { + t.Fatalf("failed to get verification info: %s", err) + } + + // mount and read the contents of the cim + volumeGUID, err := guid.NewV4() + if err != nil { + t.Fatalf("generate cim mount GUID: %s", err) + } + + mountvol, err := MountMergedVerifiedBlockCIMs(mergedCIM, sourceCIMs, test.mountFlag, volumeGUID, rootHash) + if err != nil { + t.Fatalf("failed to mount merged block CIMs: %s\n", err) + } + defer func() { + if err := Unmount(mountvol); err != nil { + t.Logf("CIM unmount failed: %s", err) + } + }() + // since we are merging, only 1 foo.txt (from the 1st CIM) should + // show up + compareContent(t, mountvol, []tuple{testContents[0][0], testContents[1][0]}) + }) + } +} diff --git a/pkg/cimfs/cim_writer_windows.go b/pkg/cimfs/cim_writer_windows.go index 62e45841f7..dec9cf07b5 100644 --- a/pkg/cimfs/cim_writer_windows.go +++ b/pkg/cimfs/cim_writer_windows.go @@ -346,7 +346,7 @@ func (c *CimFsWriter) Close() (err error) { } if c.sealOnClose { if err = sealBlockCIM(filepath.Dir(c.name)); err != nil { - return &OpError{Cim: c.name, Op: "seal", Err: err} + return &OpError{Cim: filepath.Dir(c.name), Op: "seal", Err: err} } } return nil @@ -506,6 +506,10 @@ func MergeBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM) (err error) { func sealBlockCIM(blockPath string) error { var hashSize, fixedHeaderSize uint64 hashBuf := make([]byte, cimHashSize) + + // the blockPath could be a path to a block device or a file. In either case there should be no trailing backslash. + blockPath = strings.TrimSuffix(blockPath, "\\") + if err := winapi.CimSealImage(blockPath, &hashSize, &fixedHeaderSize, &hashBuf[0]); err != nil { return fmt.Errorf("failed to seal block CIM: %w", err) } else if hashSize != cimHashSize { diff --git a/pkg/cimfs/mount_cim.go b/pkg/cimfs/mount_cim.go index 12fa2da036..475c0957ed 100644 --- a/pkg/cimfs/mount_cim.go +++ b/pkg/cimfs/mount_cim.go @@ -151,3 +151,63 @@ func MountVerifiedBlockCIM(bCIM *BlockCIM, mountFlags uint32, volumeGUID guid.GU } return fmt.Sprintf("\\\\?\\Volume{%s}\\", volumeGUID.String()), nil } + +// MountMergedVerifiedBlockCIMs mounts the given merged verified BlockCIM (usually created +// with `MergeBlockCIMs`) at a volume with given GUID, with the given root hash. The +// `sourceCIMs` MUST be identical to the `sourceCIMs` passed to `MergeBlockCIMs` when +// creating this merged CIM. The root hash is usually returned when the CIM is sealed or +// the root hash can be queried from a block CIM. In case of merged CIMs, the root hash of +// the merged CIM should be passed here. Every read on the mounted volume will be verified +// to match against the provided root hash if it doesn't, the read will fail. The source +// CIMs and the merged CIM MUST have been created with the verified creation flag. +func MountMergedVerifiedBlockCIMs(mergedCIM *BlockCIM, sourceCIMs []*BlockCIM, mountFlags uint32, volumeGUID guid.GUID, rootHash []byte) (string, error) { + if !IsVerifiedCimSupported() { + return "", fmt.Errorf("verified CIMs aren't supported on this OS version") + } else if len(sourceCIMs) < 2 { + return "", fmt.Errorf("need at least 2 source CIMs, got %d: %w", len(sourceCIMs), os.ErrInvalid) + } else if len(rootHash) != cimHashSize { + return "", fmt.Errorf("unexpected root hash size %d, expected size is %d", len(rootHash), cimHashSize) + } + + switch mergedCIM.Type { + case BlockCIMTypeDevice: + mountFlags |= CimMountBlockDeviceCim + case BlockCIMTypeSingleFile: + mountFlags |= CimMountSingleFileCim + default: + return "", fmt.Errorf("invalid block CIM type `%d`", mergedCIM.Type) + } + + for _, sCIM := range sourceCIMs { + if sCIM.Type != mergedCIM.Type { + return "", fmt.Errorf("source CIM (%s) type doesn't match with merged CIM type: %w", sCIM.String(), os.ErrInvalid) + } + } + + // win32 mount merged CIM API expects an array of all CIMs. 0th entry in the array + // should be the merged CIM. All remaining entries should be the source CIM paths + // in the same order that was used while creating the merged CIM. + allcims := append([]*BlockCIM{mergedCIM}, sourceCIMs...) + cimsToMerge := []winapi.CimFsImagePath{} + for _, bcim := range allcims { + // Trailing backslashes cause problems-remove those + imageDir, err := windows.UTF16PtrFromString(strings.TrimRight(bcim.BlockPath, `\`)) + if err != nil { + return "", fmt.Errorf("convert string to utf16: %w", err) + } + cimName, err := windows.UTF16PtrFromString(bcim.CimName) + if err != nil { + return "", fmt.Errorf("convert string to utf16: %w", err) + } + + cimsToMerge = append(cimsToMerge, winapi.CimFsImagePath{ + ImageDir: imageDir, + ImageName: cimName, + }) + } + + if err := winapi.CimMergeMountVerifiedImage(uint32(len(cimsToMerge)), &cimsToMerge[0], mountFlags, &volumeGUID, cimHashSize, &rootHash[0]); err != nil { + return "", &MountError{Cim: filepath.Join(mergedCIM.BlockPath, mergedCIM.CimName), Op: "MountMergedVerified", Err: err} + } + return fmt.Sprintf(VolumePathFormat, volumeGUID.String()), nil +}