From 6a2eb0d72bdf0cc6b7e30528033880d4bd2c3755 Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 11:26:20 +0100 Subject: [PATCH 1/9] Change input HCS json for C-WCOW so that it can boot Signed-off-by: Takuro Sato --- internal/uvm/create_wcow.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index aedd960776..2b0bb626d6 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -221,9 +221,6 @@ func prepareCommonConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWC ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.VirtualMachineMemory{ SizeInMB: memorySizeInMB, - AllowOvercommit: opts.AllowOvercommit, - // EnableHotHint is not compatible with physical. - EnableHotHint: opts.AllowOvercommit, EnableDeferredCommit: opts.EnableDeferredCommit, LowMMIOGapInMB: opts.LowMMIOGapInMB, HighMMIOBaseInMB: opts.HighMMIOBaseInMB, @@ -382,8 +379,15 @@ func prepareSecurityConfigDoc(ctx context.Context, uvm *UtilityVM, opts *Options } } - memoryBacking := hcsschema.MemoryBackingType_PHYSICAL - doc.VirtualMachine.ComputeTopology.Memory.Backing = &memoryBacking + if opts.IsolationType != "SecureNestedPaging" { + // With these options SNP UVM doesn't boot. + memoryBacking := hcsschema.MemoryBackingType_PHYSICAL + doc.VirtualMachine.ComputeTopology.Memory.Backing = &memoryBacking + doc.VirtualMachine.ComputeTopology.Memory.AllowOvercommit = opts.AllowOvercommit + // EnableHotHint is not compatible with physical. + doc.VirtualMachine.ComputeTopology.Memory.EnableHotHint = opts.AllowOvercommit + } + doc.SchemaVersion = schemaversion.SchemaV25() doc.VirtualMachine.Version = &hcsschema.Version{ Major: 11, From b53e344f763ad93f71bfc303ff76e9738066ae41 Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 11:27:29 +0100 Subject: [PATCH 2/9] Allow SNP not only SecureNestedPaging for WCOWIsolationType annotation Signed-off-by: Takuro Sato --- internal/oci/uvm.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index ece82e3d9b..10d51acb8e 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -253,7 +253,21 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u wopts.SecurityPolicyEnforcer = ParseAnnotationsString(a, annotations.WCOWSecurityPolicyEnforcer, wopts.SecurityPolicyEnforcer) wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false) wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) - wopts.IsolationType = ParseAnnotationsString(a, annotations.WCOWIsolationType, "") + + isolationType := ParseAnnotationsString(a, annotations.WCOWIsolationType, "") + if isolationType != "" { + switch isolationType { + case "SecureNestedPaging", "SNP": + wopts.IsolationType = "SecureNestedPaging" + case "VirtualizationBasedSecurity", "VBS": + wopts.IsolationType = "VirtualizationBasedSecurity" + case "GuestStateOnly": + wopts.IsolationType = "GuestStateOnly" + default: + return fmt.Errorf("invalid WCOW isolation type %q", isolationType) + } + } + return nil } @@ -386,7 +400,6 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( wopts.NoDirectMap = ParseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) wopts.NoInheritHostTimezone = ParseAnnotationsBool(ctx, s.Annotations, annotations.NoInheritHostTimezone, wopts.NoInheritHostTimezone) wopts.AdditionalRegistryKeys = append(wopts.AdditionalRegistryKeys, parseAdditionalRegistryValues(ctx, s.Annotations)...) - handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) // Writable EFI is valid for both confidential and regular Hyper-V isolated WCOW. wopts.WritableEFI = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWWritableEFI, wopts.WritableEFI) From 00d1c6ed732a7dface2262661857b8b1781eca42 Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 11:36:26 +0100 Subject: [PATCH 3/9] Fix lint error Signed-off-by: Takuro Sato --- internal/uvm/create_wcow.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 2b0bb626d6..aba0780d2b 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -220,7 +220,7 @@ func prepareCommonConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWC RegistryChanges: ®istryChanges, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.VirtualMachineMemory{ - SizeInMB: memorySizeInMB, + SizeInMB: memorySizeInMB, EnableDeferredCommit: opts.EnableDeferredCommit, LowMMIOGapInMB: opts.LowMMIOGapInMB, HighMMIOBaseInMB: opts.HighMMIOBaseInMB, From 416844c4d55409c7c3e2d6a12aea0181212df154 Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 13:53:50 +0100 Subject: [PATCH 4/9] Make "SecureNestedPaging" default for WCOWIsolationType Signed-off-by: Takuro Sato Co-authored-by: Amit Barve --- internal/oci/uvm.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 10d51acb8e..8c25bf7679 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -254,18 +254,16 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false) wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) - isolationType := ParseAnnotationsString(a, annotations.WCOWIsolationType, "") - if isolationType != "" { - switch isolationType { - case "SecureNestedPaging", "SNP": - wopts.IsolationType = "SecureNestedPaging" - case "VirtualizationBasedSecurity", "VBS": - wopts.IsolationType = "VirtualizationBasedSecurity" - case "GuestStateOnly": - wopts.IsolationType = "GuestStateOnly" - default: - return fmt.Errorf("invalid WCOW isolation type %q", isolationType) - } + isolationType := ParseAnnotationsString(a, annotations.WCOWIsolationType, "SecureNestedPaging") + switch isolationType { + case "SecureNestedPaging", "SNP": // Allow VBS & SNP shorthands + wopts.IsolationType = "SecureNestedPaging" + case "VirtualizationBasedSecurity", "VBS": + wopts.IsolationType = "VirtualizationBasedSecurity" + case "GuestStateOnly": + wopts.IsolationType = "GuestStateOnly" + default: + return fmt.Errorf("invalid WCOW isolation type %q", isolationType) } return nil From d86b617d90932e79d077fbaf6fff60afb0474f1d Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 16:20:58 +0100 Subject: [PATCH 5/9] Change based on discussion Signed-off-by: Takuro Sato --- internal/oci/uvm.go | 54 +++++++++++++++++++++++++++++----- internal/uvm/create_wcow.go | 11 ++----- pkg/annotations/annotations.go | 7 ++--- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 8c25bf7679..46c6e29d78 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -196,7 +196,7 @@ func handleLCOWSecurityPolicy(ctx context.Context, a map[string]string, lopts *u lopts.SecurityPolicy = ParseAnnotationsString(a, annotations.LCOWSecurityPolicy, lopts.SecurityPolicy) // allow actual isolated boot etc to be ignored if we have no hardware. Required for dev // this is not a security issue as the attestation will fail without a genuine report - noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.LCOWNoSecurityHardware, false) + noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false) // if there is a security policy (and SNP) we currently boot in a way that doesn't support any boot options // this might change if the building of the vmgs file were to be done on demand but that is likely @@ -245,23 +245,63 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u return fmt.Errorf("minimum 2048MB of memory is required for confidential pods, got: %d", wopts.MemorySizeInMB) } - wopts.AllowOvercommit = ParseAnnotationsBool(ctx, a, annotations.AllowOvercommit, false) - if wopts.AllowOvercommit { - return fmt.Errorf("allow overcommit MUST be false for confidential pods") - } - wopts.SecurityPolicyEnforcer = ParseAnnotationsString(a, annotations.WCOWSecurityPolicyEnforcer, wopts.SecurityPolicyEnforcer) wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false) wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) + wopts.IsolationType = "SecureNestedPaging" + if noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false); noSecurityHardware { + wopts.IsolationType = "VirtualizationBasedSecurity" + } + if err := handleWCOWIsolationType(ctx, a, wopts); err != nil { + return err + } + + return nil +} - isolationType := ParseAnnotationsString(a, annotations.WCOWIsolationType, "SecureNestedPaging") +// Isolation controls the behavior of the boot and actual isolation type of the VM. +// +// SecureNestedPaging: +// +// UVM is a SNP encrypted VM. +// HCL (from VMGS file) enforces the hash of the bootmngr. +// HCL sets an UEFI variable which is the expected hash of the BCD. +// Bootmngr enforces the hash of the BCD matches the UEFI variable. +// BCD contains the root hash of the OS disk CIM. +// Bootmngr also requires the root hash of the OS disk CIM to be in the BCD so it can find the correct OS disk. +// +// VirtualizationBasedSecurity: +// +// UVM is a VBS isolated VM. +// HCL (from VMGS file) enforces the hash of the bootmngr. +// HCL sets an UEFI variable which is the expected hash of the BCD. +// Bootmngr enforces the hash of the BCD matches the UEFI variable. +// BCD contains the root hash of the OS disk CIM. +// Bootmngr also requires the root hash of the OS disk CIM to be in the BCD so it can find the correct OS disk. +// This is useful for development purposes. +// +// GuestStateOnly: +// +// Boots with the platform HCL, does not enforce CWOW rules. +// Bootmngr cannot enforce the BCD hash as it is not set by HCL. +// BCD must contain the root hash of the OS disk CIM so that the Bootmngr can find the correct OS disk. +// This is useful for development purposes. +// This is also useful if you need kernel debugger. +// +// Note we do not support the regular WCOW case here. +// Regular WCOW uses normal Hyper-V isolation i.e. without HCL. +func handleWCOWIsolationType(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) error { + isolationType := ParseAnnotationsString(a, annotations.WCOWIsolationType, wopts.IsolationType) switch isolationType { case "SecureNestedPaging", "SNP": // Allow VBS & SNP shorthands wopts.IsolationType = "SecureNestedPaging" + wopts.AllowOvercommit = false case "VirtualizationBasedSecurity", "VBS": wopts.IsolationType = "VirtualizationBasedSecurity" + wopts.AllowOvercommit = false case "GuestStateOnly": wopts.IsolationType = "GuestStateOnly" + wopts.AllowOvercommit = false default: return fmt.Errorf("invalid WCOW isolation type %q", isolationType) } diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index aba0780d2b..aa4465fabf 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -221,6 +221,8 @@ func prepareCommonConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWC ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.VirtualMachineMemory{ SizeInMB: memorySizeInMB, + AllowOvercommit: opts.AllowOvercommit, + EnableHotHint: opts.AllowOvercommit, EnableDeferredCommit: opts.EnableDeferredCommit, LowMMIOGapInMB: opts.LowMMIOGapInMB, HighMMIOBaseInMB: opts.HighMMIOBaseInMB, @@ -379,15 +381,6 @@ func prepareSecurityConfigDoc(ctx context.Context, uvm *UtilityVM, opts *Options } } - if opts.IsolationType != "SecureNestedPaging" { - // With these options SNP UVM doesn't boot. - memoryBacking := hcsschema.MemoryBackingType_PHYSICAL - doc.VirtualMachine.ComputeTopology.Memory.Backing = &memoryBacking - doc.VirtualMachine.ComputeTopology.Memory.AllowOvercommit = opts.AllowOvercommit - // EnableHotHint is not compatible with physical. - doc.VirtualMachine.ComputeTopology.Memory.EnableHotHint = opts.AllowOvercommit - } - doc.SchemaVersion = schemaversion.SchemaV25() doc.VirtualMachine.Version = &hcsschema.Version{ Major: 11, diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index b00522344b..9e857340d2 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -145,10 +145,9 @@ const ( // Deprecated: use [LCOWHostAMDCertificate] instead. HostAMDCertificate = LCOWHostAMDCertificate - // LCOWNoSecurityHardware allows us, when it is set to true, to do testing and development without requiring SNP hardware. - LCOWNoSecurityHardware = "io.microsoft.virtualmachine.lcow.no_security_hardware" - // Deprecated: use [LCOWNoSecurityHardware] instead. - NoSecurityHardware = LCOWNoSecurityHardware + // NoSecurityHardware allows us, when it is set to true, to do testing and development without requiring SNP hardware. + // + NoSecurityHardware = "io.microsoft.virtualmachine.no_security_hardware" // LCOWSecurityPolicy is used to specify a security policy for opengcs to enforce. LCOWSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" From bc23a4ebc436482f527db378bf85b869a1ff673d Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 16:23:52 +0100 Subject: [PATCH 6/9] Change the default for NoSecurityHardware GuestStateOnly Signed-off-by: Takuro Sato --- internal/oci/uvm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 46c6e29d78..96c67ae665 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -250,7 +250,7 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) wopts.IsolationType = "SecureNestedPaging" if noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false); noSecurityHardware { - wopts.IsolationType = "VirtualizationBasedSecurity" + wopts.IsolationType = "GuestStateOnly" } if err := handleWCOWIsolationType(ctx, a, wopts); err != nil { return err From 624e8eb92f2923c3b1f927fc1b989e7906130001 Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 17:29:38 +0100 Subject: [PATCH 7/9] Revert accidental change Signed-off-by: Takuro Sato --- internal/oci/uvm.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 96c67ae665..344d0082c3 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -438,6 +438,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( wopts.NoDirectMap = ParseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) wopts.NoInheritHostTimezone = ParseAnnotationsBool(ctx, s.Annotations, annotations.NoInheritHostTimezone, wopts.NoInheritHostTimezone) wopts.AdditionalRegistryKeys = append(wopts.AdditionalRegistryKeys, parseAdditionalRegistryValues(ctx, s.Annotations)...) + handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) // Writable EFI is valid for both confidential and regular Hyper-V isolated WCOW. wopts.WritableEFI = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWWritableEFI, wopts.WritableEFI) From 2b9810015c313dd8f5b294d2ceb05e15ce87a71d Mon Sep 17 00:00:00 2001 From: Takuro Sato Date: Fri, 12 Sep 2025 17:40:57 +0100 Subject: [PATCH 8/9] Remove some comments Signed-off-by: Takuro Sato --- internal/oci/uvm.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 344d0082c3..889e31f09e 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -259,37 +259,6 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u return nil } -// Isolation controls the behavior of the boot and actual isolation type of the VM. -// -// SecureNestedPaging: -// -// UVM is a SNP encrypted VM. -// HCL (from VMGS file) enforces the hash of the bootmngr. -// HCL sets an UEFI variable which is the expected hash of the BCD. -// Bootmngr enforces the hash of the BCD matches the UEFI variable. -// BCD contains the root hash of the OS disk CIM. -// Bootmngr also requires the root hash of the OS disk CIM to be in the BCD so it can find the correct OS disk. -// -// VirtualizationBasedSecurity: -// -// UVM is a VBS isolated VM. -// HCL (from VMGS file) enforces the hash of the bootmngr. -// HCL sets an UEFI variable which is the expected hash of the BCD. -// Bootmngr enforces the hash of the BCD matches the UEFI variable. -// BCD contains the root hash of the OS disk CIM. -// Bootmngr also requires the root hash of the OS disk CIM to be in the BCD so it can find the correct OS disk. -// This is useful for development purposes. -// -// GuestStateOnly: -// -// Boots with the platform HCL, does not enforce CWOW rules. -// Bootmngr cannot enforce the BCD hash as it is not set by HCL. -// BCD must contain the root hash of the OS disk CIM so that the Bootmngr can find the correct OS disk. -// This is useful for development purposes. -// This is also useful if you need kernel debugger. -// -// Note we do not support the regular WCOW case here. -// Regular WCOW uses normal Hyper-V isolation i.e. without HCL. func handleWCOWIsolationType(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) error { isolationType := ParseAnnotationsString(a, annotations.WCOWIsolationType, wopts.IsolationType) switch isolationType { From d3290c5670e7cff609e191dcdc7151dba3f4a1cd Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Fri, 12 Sep 2025 14:16:33 -0400 Subject: [PATCH 9/9] Remove invalid test This PR changes the code to always have the default isolation type of "SecureNestedPaging". It also updates the code so that it doesn't error out if allow over commit is set to true in annotations, instead that value is silently overridden to false. This commit updates the tests to follow the same logic. Signed-off-by: Amit Barve --- internal/oci/uvm_test.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 4a4ceed5c2..f8405c64c6 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -158,8 +158,8 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_Defaults(t *testing.T) { if wopts.GuestStateFilePath != uvm.GetDefaultConfidentialVMGSPath() { t.Fatalf("expected GuestStateFilePath to default to %q, got %q", uvm.GetDefaultConfidentialVMGSPath(), wopts.GuestStateFilePath) } - if wopts.IsolationType != "" { - t.Fatalf("expected empty IsolationType by default, got %q", wopts.IsolationType) + if wopts.IsolationType != "SecureNestedPaging" { + t.Fatalf("expected SecureNestedPaging IsolationType by default, got %q", wopts.IsolationType) } } @@ -218,20 +218,6 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnLowMemory(t *testing.T } } -func Test_SpecToUVMCreateOptions_WCOW_Confidential_ErrorOnAllowOvercommit(t *testing.T) { - s := &specs.Spec{ - Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, - Annotations: map[string]string{ - annotations.WCOWSecurityPolicy: "test-policy", - annotations.AllowOvercommit: "true", - }, - } - - if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { - t.Fatal("expected error for confidential pods when AllowOvercommit=true, got nil") - } -} - func Test_SpecToUVMCreateOptions_WCOW_Confidential_Overrides(t *testing.T) { s := &specs.Spec{ Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}},