diff --git a/.gitignore b/.gitignore index 7a6d5a4ec..4030f74ff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ .cache _out/ + +# Test binaries +*.test diff --git a/Makefile b/Makefile index 5028549fa..d786325dc 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,21 @@ test-e2e-autoscaler: ## Run openshift specific e2e test including autoscaler test-e2e-periodic-autoscaler: ## Run openshift specific periodic e2e test including autoscaler hack/ci-integration.sh $(GINKGO_ARGS) --label-filter='periodic&&autoscaler' -p +.PHONY: test-mapi +test-mapi: ## Run MAPI authoritative testing (MAPI auth, CAPI mirrors) + TEST_BACKEND_TYPE=MAPI TEST_AUTHORITATIVE_API=MAPI hack/ci-integration.sh $(GINKGO_ARGS) --label-filter='unified||mapi' + +.PHONY: test-capi +test-capi: ## Run pure CAPI testing (no MAPI involvement) + TEST_BACKEND_TYPE=CAPI TEST_AUTHORITATIVE_API=CAPI hack/ci-integration.sh $(GINKGO_ARGS) --label-filter='unified' + +.PHONY: test-mapi-with-capi-auth +test-mapi-with-capi-auth: ## Run tests with MAPI backend and CAPI authority (conversion layer) + TEST_BACKEND_TYPE=MAPI TEST_AUTHORITATIVE_API=CAPI hack/ci-integration.sh $(GINKGO_ARGS) --label-filter='unified' + +.PHONY: test-all +test-all: test-mapi test-capi test-mapi-with-capi-auth ## Run all unified framework test scenarios + .PHONY: help help: @grep -E '^[a-zA-Z/0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/pkg/e2e_test.go b/pkg/e2e_test.go index 2a6792786..197f741c1 100644 --- a/pkg/e2e_test.go +++ b/pkg/e2e_test.go @@ -9,6 +9,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" osconfigv1 "github.com/openshift/api/config/v1" machinev1 "github.com/openshift/api/machine/v1beta1" @@ -27,6 +28,7 @@ import ( _ "github.com/openshift/cluster-api-actuator-pkg/pkg/mapi" _ "github.com/openshift/cluster-api-actuator-pkg/pkg/operators" _ "github.com/openshift/cluster-api-actuator-pkg/pkg/providers" + _ "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/e2e" ) func init() { @@ -75,6 +77,9 @@ var _ = BeforeSuite(func() { client, err := framework.LoadClient() Expect(err).ToNot(HaveOccurred()) + // Set komega client for all tests + komega.SetClient(client) + ctx := framework.GetContext() platform, err := framework.GetPlatform(ctx, client) diff --git a/pkg/framework/ginkgo-labels.go b/pkg/framework/ginkgo-labels.go index 2d355af0d..0acce9dff 100644 --- a/pkg/framework/ginkgo-labels.go +++ b/pkg/framework/ginkgo-labels.go @@ -38,4 +38,7 @@ var ( // LabelConnectedOnly indicates that the test can run in a connection cluster only. LabelConnectedOnly = ginkgo.Label("connected-only") + + // LabelUnified applies to tests using the unified MAPI/CAPI testing framework. + LabelUnified = ginkgo.Label("unified") ) diff --git a/pkg/framework/utils.go b/pkg/framework/utils.go index cd3fbe522..bdc9321f8 100644 --- a/pkg/framework/utils.go +++ b/pkg/framework/utils.go @@ -113,3 +113,17 @@ func GetControlPlaneHostAndPort(ctx context.Context, cl client.Client) (string, return apiURL.Hostname(), int32(port), nil } + +// MergeLabels merges multiple label mappings into a single map. +// Later maps override earlier ones for conflicting keys. +func MergeLabels(labelMaps ...map[string]string) map[string]string { + result := make(map[string]string) + + for _, labels := range labelMaps { + for k, v := range labels { + result[k] = v + } + } + + return result +} diff --git a/pkg/unified/backends/backend_interface.go b/pkg/unified/backends/backend_interface.go new file mode 100644 index 000000000..5c2ff61bc --- /dev/null +++ b/pkg/unified/backends/backend_interface.go @@ -0,0 +1,75 @@ +package backends + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + corev1 "k8s.io/api/core/v1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/config" +) + +// MachineBackend defines the abstract interface for machine backends. +type MachineBackend interface { + GetBackendType() config.BackendType + GetAuthoritativeAPI() config.BackendType + CreateMachineTemplate(ctx context.Context, client runtimeclient.Client, platform configv1.PlatformType, params BackendMachineTemplateParams) (interface{}, error) + DeleteMachineTemplate(ctx context.Context, client runtimeclient.Client, template interface{}) error + CreateMachineSet(ctx context.Context, client runtimeclient.Client, params BackendMachineSetParams) (interface{}, error) + DeleteMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error + WaitForMachineSetDeleted(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error + WaitForMachinesRunning(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error + GetMachineSetStatus(ctx context.Context, client runtimeclient.Client, machineSet interface{}) (*MachineSetStatus, error) + GetNodesFromMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) ([]corev1.Node, error) +} + +// BackendMachineSetParams defines common parameters for creating machine sets. +type BackendMachineSetParams struct { + Name string + Replicas int32 + Labels map[string]string + Annotations map[string]string + Template interface{} + FailureDomain string + // AuthoritativeAPI specifies which API should be authoritative for this MachineSet + AuthoritativeAPI config.BackendType +} + +// BackendMachineTemplateParams defines common parameters for creating machine templates. +type BackendMachineTemplateParams struct { + Name string + Platform configv1.PlatformType + Spec interface{} +} + +// MachineSetStatus defines common structure for machine set status. +type MachineSetStatus struct { + Replicas int32 + AvailableReplicas int32 + ReadyReplicas int32 + AuthoritativeAPI string +} + +// NewBackend creates appropriate backend instance based on configuration. +func NewBackend(backendType config.BackendType, authoritativeAPI config.BackendType) (MachineBackend, error) { + switch backendType { + case config.BackendTypeMAPI: + return NewMAPIBackend(authoritativeAPI), nil + case config.BackendTypeCAPI: + return NewCAPIBackend(authoritativeAPI), nil + default: + return nil, fmt.Errorf("unsupported backend type: %s", backendType) + } +} + +// NewMAPIBackend creates a new MAPI backend instance. +func NewMAPIBackend(authoritativeAPI config.BackendType) MachineBackend { + return &mapiBackend{backendType: config.BackendTypeMAPI, authoritativeAPI: authoritativeAPI} +} + +// NewCAPIBackend creates a new CAPI backend instance. +func NewCAPIBackend(authoritativeAPI config.BackendType) MachineBackend { + return &capiBackend{backendType: config.BackendTypeCAPI, authoritativeAPI: authoritativeAPI} +} diff --git a/pkg/unified/backends/capi_backend.go b/pkg/unified/backends/capi_backend.go new file mode 100644 index 000000000..c85049815 --- /dev/null +++ b/pkg/unified/backends/capi_backend.go @@ -0,0 +1,317 @@ +package backends + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + machinev1 "github.com/openshift/api/machine/v1beta1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + gcpv1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + yaml "sigs.k8s.io/yaml" + + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/config" +) + +// capiBackend implements the MachineBackend interface for CAPI backend. +type capiBackend struct { + backendType config.BackendType + authoritativeAPI config.BackendType +} + +func (c *capiBackend) GetBackendType() config.BackendType { return c.backendType } +func (c *capiBackend) GetAuthoritativeAPI() config.BackendType { return c.authoritativeAPI } + +func (c *capiBackend) CreateMachineSet(ctx context.Context, client runtimeclient.Client, params BackendMachineSetParams) (interface{}, error) { + GinkgoHelper() + + infra, err := framework.GetInfrastructure(ctx, client) + Expect(err).NotTo(HaveOccurred(), "Should get infrastructure global object") + Expect(infra.Status.InfrastructureName).ShouldNot(BeEmpty(), "Should have infrastructure name on Infrastructure.Status") + + clusterName := infra.Status.InfrastructureName + userDataSecret := "worker-user-data" + machineSet := &clusterv1beta1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: params.Name, + Namespace: framework.ClusterAPINamespace, + Labels: params.Labels, + Annotations: params.Annotations, + }, + Spec: clusterv1beta1.MachineSetSpec{ + ClusterName: clusterName, + Replicas: ¶ms.Replicas, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{ + "cluster.x-k8s.io/set-name": params.Name, + "cluster.x-k8s.io/cluster-name": clusterName, + }}, + + Template: clusterv1beta1.MachineTemplateSpec{ + ObjectMeta: clusterv1beta1.ObjectMeta{Labels: framework.MergeLabels(params.Labels, map[string]string{ + "cluster.x-k8s.io/set-name": params.Name, + "cluster.x-k8s.io/cluster-name": clusterName, + framework.WorkerNodeRoleLabel: "", + })}, + Spec: clusterv1beta1.MachineSpec{ + Bootstrap: clusterv1beta1.Bootstrap{ + DataSecretName: &userDataSecret, + }, + ClusterName: clusterName, + }, + }, + }, + } + + // Set InfrastructureRef based on params.Template + if params.Template != nil { + c.setInfrastructureRef(&machineSet.Spec.Template.Spec, params.Template) + } + + Eventually(func() error { + return client.Create(ctx, machineSet) + }, framework.WaitMedium, framework.RetryMedium).Should(Succeed(), "Should create CAPI MachineSet %s", machineSet.Name) + + return machineSet, nil +} + +func (c *capiBackend) DeleteMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + GinkgoHelper() + + ms, ok := machineSet.(*clusterv1beta1.MachineSet) + Expect(ok).To(BeTrue(), "Should be CAPI MachineSet, got %T", machineSet) + + framework.DeleteCAPIMachineSets(ctx, client, ms) + + return nil +} + +func (c *capiBackend) WaitForMachineSetDeleted(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + GinkgoHelper() + + ms, ok := machineSet.(*clusterv1beta1.MachineSet) + Expect(ok).To(BeTrue(), "Should be CAPI MachineSet, got %T", machineSet) + + framework.WaitForCAPIMachineSetsDeleted(ctx, client, ms) + + return nil +} + +func (c *capiBackend) WaitForMachinesRunning(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + GinkgoHelper() + + ms, ok := machineSet.(*clusterv1beta1.MachineSet) + Expect(ok).To(BeTrue(), "Should be CAPI MachineSet, got %T", machineSet) + + framework.WaitForCAPIMachinesRunning(ctx, client, ms.Name) + + return nil +} + +func (c *capiBackend) GetMachineSetStatus(ctx context.Context, client runtimeclient.Client, machineSet interface{}) (*MachineSetStatus, error) { + GinkgoHelper() + + ms, ok := machineSet.(*clusterv1beta1.MachineSet) + Expect(ok).To(BeTrue(), "Should be CAPI MachineSet, got %T", machineSet) + + status := &MachineSetStatus{ + Replicas: 0, + AvailableReplicas: ms.Status.AvailableReplicas, + ReadyReplicas: ms.Status.ReadyReplicas, + } + if ms.Spec.Replicas != nil { + status.Replicas = *ms.Spec.Replicas + } + + return status, nil +} + +func (c *capiBackend) GetNodesFromMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) ([]corev1.Node, error) { + GinkgoHelper() + + _, ok := machineSet.(*clusterv1beta1.MachineSet) + Expect(ok).To(BeTrue(), "Should be CAPI MachineSet, got %T", machineSet) + + // TODO: Implement node query when needed. Currently not required by any test scenarios. + return []corev1.Node{}, nil +} + +func (c *capiBackend) CreateMachineTemplate(ctx context.Context, client runtimeclient.Client, platform configv1.PlatformType, params BackendMachineTemplateParams) (interface{}, error) { + GinkgoHelper() + + switch platform { + case configv1.AWSPlatformType: + return c.createAWSMachineTemplate(ctx, client, params) + case configv1.AzurePlatformType: + return nil, fmt.Errorf("azure machine template creation not yet implemented") + case configv1.GCPPlatformType: + return nil, fmt.Errorf("gcp machine template creation not yet implemented") + default: + return nil, fmt.Errorf("unsupported platform: %s", platform) + } +} + +func (c *capiBackend) DeleteMachineTemplate(ctx context.Context, client runtimeclient.Client, template interface{}) error { + GinkgoHelper() + + obj, ok := template.(runtimeclient.Object) + Expect(ok).To(BeTrue(), "Should be runtimeclient.Object, got %T", template) + + Eventually(func() error { + return client.Delete(ctx, obj) + }, framework.WaitShort, framework.RetryShort).Should(SatisfyAny( + Succeed(), + WithTransform(apierrors.IsNotFound, BeTrue()), + ), "Should delete MachineTemplate of type %T %s/%s successfully or MachineTemplate should not be found", + template, obj.GetNamespace(), obj.GetName()) + + return nil +} + +// createAWSMachineTemplate creates AWS machine template. +func (c *capiBackend) createAWSMachineTemplate(ctx context.Context, client runtimeclient.Client, params BackendMachineTemplateParams) (interface{}, error) { + GinkgoHelper() + + awsMachineSpec := c.getDefaultAWSCAPIMachineSpec(ctx, client) + + awsMachineTemplate := &awsv1.AWSMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: params.Name, + Namespace: framework.ClusterAPINamespace, + }, + Spec: awsv1.AWSMachineTemplateSpec{ + Template: awsv1.AWSMachineTemplateResource{ + Spec: *awsMachineSpec, + }, + }, + } + + // Apply custom configuration from params.Spec if provided + if params.Spec != nil { + // Apply the configuration directly to the template before creating it + templateConfig, ok := params.Spec.(*config.MachineTemplateConfig) + Expect(ok).To(BeTrue(), "Should be *config.MachineTemplateConfig, got %T", params.Spec) + + configErr := config.ConfigureMachineTemplate(awsMachineTemplate, templateConfig) + Expect(configErr).NotTo(HaveOccurred(), "Should apply custom configuration to template before creation") + } + + Eventually(func() error { + return client.Create(ctx, awsMachineTemplate) + }, framework.WaitMedium, framework.RetryMedium).Should(Succeed(), "Should create AWS machine template %s", awsMachineTemplate.Name) + + return awsMachineTemplate, nil +} + +// getDefaultAWSCAPIMachineSpec gets default AWS CAPI machine specification. +// Uses the first AWSMachineTemplate if exists, otherwise creates spec from worker MachineSet. +func (c *capiBackend) getDefaultAWSCAPIMachineSpec(ctx context.Context, client runtimeclient.Client) *awsv1.AWSMachineSpec { + GinkgoHelper() + // Find existing AWS machine templates + awsTemplateList := &awsv1.AWSMachineTemplateList{} + + Eventually(komega.List(awsTemplateList, runtimeclient.InNamespace(framework.ClusterAPINamespace)), framework.WaitMedium, framework.RetryMedium).Should(Succeed(), "Should list AWS machine templates") + + if len(awsTemplateList.Items) == 0 { + // If no existing CAPI templates found, create spec from worker MachineSet AMI + GinkgoWriter.Println("No CAPI AWSMachineTemplate found, creating spec from worker MachineSet") + return c.createDefaultAWSCAPIMachineSpec(ctx, client) + } + + // Use the first template's specification as default + return &awsTemplateList.Items[0].Spec.Template.Spec +} + +// createDefaultAWSCAPIMachineSpec creates default AWS CAPI machine spec. +func (c *capiBackend) createDefaultAWSCAPIMachineSpec(ctx context.Context, cl runtimeclient.Client) *awsv1.AWSMachineSpec { + GinkgoHelper() + // Get worker MachineSet to extract AMI and other config + workers, err := framework.GetWorkerMachineSets(ctx, cl) + Expect(err).ToNot(HaveOccurred(), "Should list worker MachineSets") + Expect(workers).NotTo(BeEmpty(), "Should find worker MachineSets to determine default machine configuration") + + // Extract AMI and configuration from first worker MachineSet + workerMS := workers[0] + Expect(workers[0].Spec.Template.Spec.ProviderSpec.Value).NotTo(BeNil(), "Should have ProviderSpec in worker MachineSet") + + var providerSpec machinev1.AWSMachineProviderConfig + + err = yaml.Unmarshal(workerMS.Spec.Template.Spec.ProviderSpec.Value.Raw, &providerSpec) + Expect(err).NotTo(HaveOccurred(), "Should unmarshal provider spec") + + // Build CAPI spec from worker MachineSet config + capiSpec := &awsv1.AWSMachineSpec{ + InstanceType: providerSpec.InstanceType, + AMI: awsv1.AMIReference{ + ID: providerSpec.AMI.ID, + }, + Ignition: &awsv1.Ignition{ + Version: "3.4", + StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData, + }, + Subnet: &awsv1.AWSResourceReference{ + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: []string{"*worker*"}, + }, + }, + }, + AdditionalSecurityGroups: []awsv1.AWSResourceReference{ + { + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: []string{"*worker*"}, + }, + }, + }, + }, + } + + GinkgoWriter.Printf("Created CAPI spec from worker MachineSet %s: AMI=%s, InstanceType=%s\n", + workerMS.Name, *providerSpec.AMI.ID, providerSpec.InstanceType) + + return capiSpec +} + +// setInfrastructureRef sets InfrastructureRef based on template type. +func (c *capiBackend) setInfrastructureRef(machineSpec *clusterv1beta1.MachineSpec, template interface{}) { + GinkgoHelper() + + switch t := template.(type) { + case *awsv1.AWSMachineTemplate: + machineSpec.InfrastructureRef = corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + Kind: "AWSMachineTemplate", + Name: t.Name, + Namespace: t.Namespace, + } + case *azurev1.AzureMachineTemplate: + machineSpec.InfrastructureRef = corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "AzureMachineTemplate", + Name: t.Name, + Namespace: t.Namespace, + } + case *gcpv1.GCPMachineTemplate: + machineSpec.InfrastructureRef = corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "GCPMachineTemplate", + Name: t.Name, + Namespace: t.Namespace, + } + default: + // This should never happen as template types are validated during creation + Expect(false).To(BeTrue(), "Should have supported template type for infrastructure ref, got %T", template) + } +} diff --git a/pkg/unified/backends/mapi_backend.go b/pkg/unified/backends/mapi_backend.go new file mode 100644 index 000000000..a19c137a5 --- /dev/null +++ b/pkg/unified/backends/mapi_backend.go @@ -0,0 +1,329 @@ +package backends + +import ( + "context" + "encoding/json" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + machinev1 "github.com/openshift/api/machine/v1beta1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + yaml "sigs.k8s.io/yaml" + + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/config" +) + +// mapiBackend implements the MachineBackend interface for MAPI backend. +type mapiBackend struct { + backendType config.BackendType + authoritativeAPI config.BackendType +} + +func (m *mapiBackend) GetBackendType() config.BackendType { return m.backendType } +func (m *mapiBackend) GetAuthoritativeAPI() config.BackendType { return m.authoritativeAPI } +func (m *mapiBackend) CreateMachineSet(ctx context.Context, client runtimeclient.Client, params BackendMachineSetParams) (interface{}, error) { + GinkgoHelper() + + infra, err := framework.GetInfrastructure(ctx, client) + Expect(err).NotTo(HaveOccurred(), "Should get infrastructure global object") + Expect(infra.Status.InfrastructureName).ShouldNot(BeEmpty(), "Should have infrastructure name on Infrastructure.Status") + + clusterName := infra.Status.InfrastructureName + machineSet := &machinev1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: params.Name, + Namespace: framework.MachineAPINamespace, + Labels: params.Labels, + Annotations: params.Annotations, + }, + Spec: machinev1.MachineSetSpec{ + Replicas: ¶ms.Replicas, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{ + "machine.openshift.io/cluster-api-cluster": clusterName, + "machine.openshift.io/cluster-api-machineset": params.Name, + }}, + Template: machinev1.MachineTemplateSpec{ + ObjectMeta: machinev1.ObjectMeta{Labels: framework.MergeLabels(params.Labels, map[string]string{ + "machine.openshift.io/cluster-api-cluster": clusterName, + "machine.openshift.io/cluster-api-machine-role": "worker", + "machine.openshift.io/cluster-api-machine-type": "worker", + "machine.openshift.io/cluster-api-machineset": params.Name, + })}, + Spec: machinev1.MachineSpec{}, + }, + }, + } + + // Set AuthoritativeAPI based on backend configuration + if m.authoritativeAPI == config.BackendTypeCAPI { + machineSet.Spec.Template.Spec.AuthoritativeAPI = machinev1.MachineAuthorityClusterAPI + } else { + machineSet.Spec.Template.Spec.AuthoritativeAPI = machinev1.MachineAuthorityMachineAPI + } + + // Set ProviderSpec based on params.Template + if params.Template != nil { + providerSpec, err := m.convertTemplateToProviderSpec(ctx, params.Template) + Expect(err).NotTo(HaveOccurred(), "Should convert template to provider spec") + + machineSet.Spec.Template.Spec.ProviderSpec = *providerSpec + } + + Eventually(func() error { + return client.Create(ctx, machineSet) + }, framework.WaitMedium, framework.RetryMedium).Should(Succeed(), "Should create MAPI MachineSet %s", machineSet.Name) + + return machineSet, nil +} + +func (m *mapiBackend) DeleteMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + GinkgoHelper() + + ms, ok := machineSet.(*machinev1.MachineSet) + Expect(ok).To(BeTrue(), "Should be MAPI MachineSet, got %T", machineSet) + + Eventually(func() error { + return client.Delete(ctx, ms) + }, framework.WaitShort, framework.RetryShort).Should(SatisfyAny( + Succeed(), + WithTransform(apierrors.IsNotFound, BeTrue()), + ), "Should delete MachineSet %s/%s successfully or MachineSet should not be found", + ms.Namespace, ms.Name) + + return nil +} + +func (m *mapiBackend) WaitForMachineSetDeleted(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + GinkgoHelper() + + ms, ok := machineSet.(*machinev1.MachineSet) + Expect(ok).To(BeTrue(), "Should be MAPI MachineSet, got %T", machineSet) + + framework.WaitForMachineSetsDeleted(ctx, client, ms) + + return nil +} + +func (m *mapiBackend) WaitForMachinesRunning(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + GinkgoHelper() + + ms, ok := machineSet.(*machinev1.MachineSet) + Expect(ok).To(BeTrue(), "Should be MAPI MachineSet, got %T", machineSet) + + framework.WaitForMachineSet(ctx, client, ms.Name) + + return nil +} + +func (m *mapiBackend) GetMachineSetStatus(ctx context.Context, client runtimeclient.Client, machineSet interface{}) (*MachineSetStatus, error) { + GinkgoHelper() + + ms, ok := machineSet.(*machinev1.MachineSet) + Expect(ok).To(BeTrue(), "Should be MAPI MachineSet, got %T", machineSet) + + status := &MachineSetStatus{ + Replicas: 0, + AvailableReplicas: ms.Status.AvailableReplicas, + ReadyReplicas: ms.Status.ReadyReplicas, + AuthoritativeAPI: "MachineAPI", + } + + if ms.Spec.Replicas != nil { + status.Replicas = *ms.Spec.Replicas + } + + if ms.Status.AuthoritativeAPI != "" { + status.AuthoritativeAPI = string(ms.Status.AuthoritativeAPI) + } + + return status, nil +} + +func (m *mapiBackend) GetNodesFromMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) ([]corev1.Node, error) { + GinkgoHelper() + + _, ok := machineSet.(*machinev1.MachineSet) + Expect(ok).To(BeTrue(), "Should be MAPI MachineSet, got %T", machineSet) + + // TODO: Implement node query when needed. Currently not required by any test scenarios. + return []corev1.Node{}, nil +} + +func (m *mapiBackend) CreateMachineTemplate(ctx context.Context, client runtimeclient.Client, platform configv1.PlatformType, params BackendMachineTemplateParams) (interface{}, error) { + GinkgoHelper() + + switch platform { + case configv1.AWSPlatformType: + return m.createAWSMachineTemplate(ctx, client, params) + case configv1.AzurePlatformType: + return nil, fmt.Errorf("azure machine template creation not yet implemented") + case configv1.GCPPlatformType: + return nil, fmt.Errorf("gcp machine template creation not yet implemented") + default: + return nil, fmt.Errorf("unsupported platform: %s", platform) + } +} + +func (m *mapiBackend) DeleteMachineTemplate(ctx context.Context, client runtimeclient.Client, template interface{}) error { + return nil +} + +// createAWSMachineTemplate creates AWS machine template, returns MAPI ProviderSpec. +func (m *mapiBackend) createAWSMachineTemplate(ctx context.Context, _ runtimeclient.Client, params BackendMachineTemplateParams) (interface{}, error) { + GinkgoHelper() + + // Get the default AWS MAPI ProviderSpec + mapiProviderSpec := m.getDefaultAWSMAPIProviderSpec(ctx) + + // For MAPI backend, we directly return serialized ProviderSpec + // This is because MAPI backend uses ProviderSpec, not independent template resources + // Serialize ProviderSpec to RawExtension + providerSpecBytes, err := json.Marshal(mapiProviderSpec) + Expect(err).NotTo(HaveOccurred(), "Should marshal provider spec") + + providerSpecRaw := &runtime.RawExtension{ + Raw: providerSpecBytes, + } + + // Apply custom configuration from params.Spec if provided + if params.Spec != nil { + // Apply the configuration directly to the ProviderSpec + templateConfig, ok := params.Spec.(*config.MachineTemplateConfig) + Expect(ok).To(BeTrue(), "Should be *config.MachineTemplateConfig, got %T", params.Spec) + + configErr := config.ConfigureMachineTemplate(providerSpecRaw, templateConfig) + Expect(configErr).NotTo(HaveOccurred(), "Should apply custom configuration to MAPI ProviderSpec") + } + + return providerSpecRaw, nil +} + +// getDefaultAWSMAPIProviderSpec gets default AWS MAPI ProviderSpec. +func (m *mapiBackend) getDefaultAWSMAPIProviderSpec(ctx context.Context) *machinev1.AWSMachineProviderConfig { + GinkgoHelper() + + machineSetList := &machinev1.MachineSetList{} + + // List existing MAPI MachineSets + Eventually(komega.List(machineSetList, runtimeclient.InNamespace(framework.MachineAPINamespace)), framework.WaitMedium, framework.RetryMedium).WithContext(ctx).Should(Succeed(), "Should list MAPI machinesets") + Expect(machineSetList.Items).NotTo(BeEmpty(), "Should have MAPI machinesets") + + // Use the first MachineSet's ProviderSpec as template + machineSet := &machineSetList.Items[0] + Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).NotTo(BeNil(), "Should have ProviderSpec in MAPI MachineSet") + + providerSpec := &machinev1.AWSMachineProviderConfig{} + err := yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec) + Expect(err).NotTo(HaveOccurred(), "Should unmarshal MAPI provider spec") + + return providerSpec +} + +// convertTemplateToProviderSpec converts CAPI template to MAPI ProviderSpec. +func (m *mapiBackend) convertTemplateToProviderSpec(ctx context.Context, template interface{}) (*machinev1.ProviderSpec, error) { + GinkgoHelper() + + switch t := template.(type) { + case *awsv1.AWSMachineTemplate: + return m.convertAWSTemplateToProviderSpec(ctx, t) + case *runtime.RawExtension: + // If already RawExtension, use directly + return &machinev1.ProviderSpec{Value: t}, nil + default: + return nil, fmt.Errorf("unsupported template type: %T", template) + } +} + +// convertAWSTemplateToProviderSpec converts AWS CAPI template to MAPI ProviderSpec. +func (m *mapiBackend) convertAWSTemplateToProviderSpec(ctx context.Context, awsTemplate *awsv1.AWSMachineTemplate) (*machinev1.ProviderSpec, error) { + GinkgoHelper() + + // Get default AWS MAPI ProviderSpec as base + defaultProviderSpec := m.getDefaultAWSMAPIProviderSpec(ctx) + + // Update MAPI ProviderSpec using CAPI template configuration + awsSpec := awsTemplate.Spec.Template.Spec + + // Update instance type + if awsSpec.InstanceType != "" { + defaultProviderSpec.InstanceType = awsSpec.InstanceType + } + + // Update AMI information + if awsSpec.AMI.ID != nil { + defaultProviderSpec.AMI.ID = awsSpec.AMI.ID + } + + // Update IAM instance profile + if awsSpec.IAMInstanceProfile != "" { + defaultProviderSpec.IAMInstanceProfile = &machinev1.AWSResourceReference{ + ID: &awsSpec.IAMInstanceProfile, + } + } + + // Update subnet configuration + if awsSpec.Subnet != nil { + if awsSpec.Subnet.ID != nil { + defaultProviderSpec.Subnet = machinev1.AWSResourceReference{ + ID: awsSpec.Subnet.ID, + } + } else if len(awsSpec.Subnet.Filters) > 0 { + filters := make([]machinev1.Filter, len(awsSpec.Subnet.Filters)) + for i, filter := range awsSpec.Subnet.Filters { + filters[i] = machinev1.Filter{ + Name: filter.Name, + Values: filter.Values, + } + } + + defaultProviderSpec.Subnet = machinev1.AWSResourceReference{ + Filters: filters, + } + } + } + + // Update security group configuration + if len(awsSpec.AdditionalSecurityGroups) > 0 { + securityGroups := make([]machinev1.AWSResourceReference, len(awsSpec.AdditionalSecurityGroups)) + + for i, sg := range awsSpec.AdditionalSecurityGroups { + if sg.ID != nil { + securityGroups[i] = machinev1.AWSResourceReference{ + ID: sg.ID, + } + } else if len(sg.Filters) > 0 { + filters := make([]machinev1.Filter, len(sg.Filters)) + for j, filter := range sg.Filters { + filters[j] = machinev1.Filter{ + Name: filter.Name, + Values: filter.Values, + } + } + + securityGroups[i] = machinev1.AWSResourceReference{ + Filters: filters, + } + } + } + + defaultProviderSpec.SecurityGroups = securityGroups + } + + providerSpecBytes, err := json.Marshal(defaultProviderSpec) + Expect(err).NotTo(HaveOccurred(), "Should marshal provider spec") + + return &machinev1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: providerSpecBytes, + }, + }, nil +} diff --git a/pkg/unified/config/template_config.go b/pkg/unified/config/template_config.go new file mode 100644 index 000000000..8161db4cb --- /dev/null +++ b/pkg/unified/config/template_config.go @@ -0,0 +1,285 @@ +package config + +import ( + "encoding/json" + "fmt" + + machinev1 "github.com/openshift/api/machine/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" +) + +// MachineTemplateConfig defines common configuration for machine templates. +type MachineTemplateConfig struct { + // AWS specific configurations + AWS *AWSMachineConfig `json:"aws,omitempty"` + // Azure specific configurations + Azure *AzureMachineConfig `json:"azure,omitempty"` + // GCP specific configurations + GCP *GCPMachineConfig `json:"gcp,omitempty"` +} + +// AWSMachineConfig AWS platform machine configuration. +type AWSMachineConfig struct { + InstanceType *string `json:"instanceType,omitempty"` + SpotMarketOptions *SpotMarketConfig `json:"spotMarketOptions,omitempty"` + PlacementGroup *PlacementGroupConfig `json:"placementGroup,omitempty"` + KMSKey *KMSKeyConfig `json:"kmsKey,omitempty"` + AdditionalTags map[string]string `json:"additionalTags,omitempty"` + Tenancy *string `json:"tenancy,omitempty"` + NetworkInterfaceType *string `json:"networkInterfaceType,omitempty"` + NonRootVolumes []VolumeConfig `json:"nonRootVolumes,omitempty"` +} + +// SpotMarketConfig spot instance configuration. +type SpotMarketConfig struct { + MaxPrice *string `json:"maxPrice,omitempty"` +} + +// PlacementGroupConfig placement group configuration. +type PlacementGroupConfig struct { + Name string `json:"name"` +} + +// KMSKeyConfig KMS key configuration. +type KMSKeyConfig struct { + KeyID string `json:"keyId"` +} + +// VolumeConfig storage volume configuration. +type VolumeConfig struct { + DeviceName string `json:"deviceName"` + Size int64 `json:"size"` + Type string `json:"type"` +} + +// AzureMachineConfig Azure platform machine configuration (reserved for future extension). +type AzureMachineConfig struct { + // Future: Azure specific configurations +} + +// GCPMachineConfig GCP platform machine configuration (reserved for future extension). +type GCPMachineConfig struct { + // Future: GCP specific configurations +} + +// ConfigureMachineTemplate configures machine template using configuration object. +func ConfigureMachineTemplate(template interface{}, config *MachineTemplateConfig) error { + if config == nil { + return nil + } + + switch t := template.(type) { + case *awsv1.AWSMachineTemplate: + return configureAWSCAPITemplate(t, config.AWS) + case *runtime.RawExtension: + return configureAWSMAPIProviderSpec(t, config.AWS) + default: + return fmt.Errorf("unsupported template type: %T", template) + } +} + +// configureAWSCAPITemplate configures CAPI AWS template. +func configureAWSCAPITemplate(template *awsv1.AWSMachineTemplate, config *AWSMachineConfig) error { + if config == nil { + return nil + } + + spec := &template.Spec.Template.Spec + + // Configure instance type + if config.InstanceType != nil { + spec.InstanceType = *config.InstanceType + } + + // Configure spot instance + if config.SpotMarketOptions != nil { + spec.SpotMarketOptions = &awsv1.SpotMarketOptions{} + if config.SpotMarketOptions.MaxPrice != nil { + spec.SpotMarketOptions.MaxPrice = config.SpotMarketOptions.MaxPrice + } + } + + // Configure placement group + if config.PlacementGroup != nil { + spec.PlacementGroupName = config.PlacementGroup.Name + } + + // Configure tenancy type + if config.Tenancy != nil { + spec.Tenancy = *config.Tenancy + } + + // Configure network interface type + if config.NetworkInterfaceType != nil { + if *config.NetworkInterfaceType == "efa" { + spec.NetworkInterfaceType = awsv1.NetworkInterfaceTypeEFAWithENAInterface + } + } + + // Configure additional tags + if len(config.AdditionalTags) > 0 { + if spec.AdditionalTags == nil { + spec.AdditionalTags = make(map[string]string) + } + + for k, v := range config.AdditionalTags { + spec.AdditionalTags[k] = v + } + } + + // Configure non-root volumes + if len(config.NonRootVolumes) > 0 { + volumes := make([]awsv1.Volume, len(config.NonRootVolumes)) + for i, v := range config.NonRootVolumes { + volumes[i] = awsv1.Volume{ + DeviceName: v.DeviceName, + Size: v.Size, + Type: awsv1.VolumeType(v.Type), + } + } + + spec.NonRootVolumes = volumes + } + + // Configure KMS encryption for root volume + if config.KMSKey != nil { + if spec.RootVolume == nil { + // Initialize RootVolume if it doesn't exist + // We need at least Size to create a volume, use a sensible default + spec.RootVolume = &awsv1.Volume{ + Size: 120, // Default root volume size + } + } + + spec.RootVolume.EncryptionKey = config.KMSKey.KeyID + // KMS encryption requires Encrypted to be true + encrypted := true + spec.RootVolume.Encrypted = &encrypted + } + + return nil +} + +// configureAWSMAPIProviderSpec configures MAPI AWS ProviderSpec. +func configureAWSMAPIProviderSpec(providerSpec *runtime.RawExtension, config *AWSMachineConfig) error { + if config == nil { + return nil + } + + var spec machinev1.AWSMachineProviderConfig + + err := json.Unmarshal(providerSpec.Raw, &spec) + if err != nil { + return fmt.Errorf("failed to unmarshal providerspec: %w", err) + } + + // Configure instance type + if config.InstanceType != nil { + spec.InstanceType = *config.InstanceType + } + + // Configure spot instance + if config.SpotMarketOptions != nil { + spec.SpotMarketOptions = &machinev1.SpotMarketOptions{} + if config.SpotMarketOptions.MaxPrice != nil { + spec.SpotMarketOptions.MaxPrice = config.SpotMarketOptions.MaxPrice + } + } + + // Configure placement group + if config.PlacementGroup != nil { + spec.PlacementGroupName = config.PlacementGroup.Name + } + + // Configure tenancy type + if config.Tenancy != nil { + spec.Placement.Tenancy = machinev1.InstanceTenancy(*config.Tenancy) + } + + // Configure network interface type + if config.NetworkInterfaceType != nil { + if *config.NetworkInterfaceType == "efa" { + spec.NetworkInterfaceType = machinev1.AWSEFANetworkInterfaceType + } + } + + // Configure additional tags + if len(config.AdditionalTags) > 0 { + if spec.Tags == nil { + spec.Tags = make([]machinev1.TagSpecification, 0) + } + // Note: MAPI tag structure differs from CAPI, adaptation required here + for k, v := range config.AdditionalTags { + spec.Tags = append(spec.Tags, machinev1.TagSpecification{ + Name: k, + Value: v, + }) + } + } + + // Configure non-root volumes + if len(config.NonRootVolumes) > 0 { + for _, v := range config.NonRootVolumes { + blockDevice := machinev1.BlockDeviceMappingSpec{ + DeviceName: &v.DeviceName, + EBS: &machinev1.EBSBlockDeviceSpec{ + VolumeSize: &v.Size, + VolumeType: &v.Type, + }, + } + spec.BlockDevices = append(spec.BlockDevices, blockDevice) + } + } + + // Configure KMS encryption for root volume + if config.KMSKey != nil { + // Find or create root volume (BlockDevice without DeviceName or with empty DeviceName) + var rootVolume *machinev1.BlockDeviceMappingSpec + + for i := range spec.BlockDevices { + if spec.BlockDevices[i].DeviceName == nil || *spec.BlockDevices[i].DeviceName == "" { + rootVolume = &spec.BlockDevices[i] + break + } + } + + // If no root volume exists, create one + if rootVolume == nil { + rootSize := int64(120) // Default root volume size + blockDevice := machinev1.BlockDeviceMappingSpec{ + EBS: &machinev1.EBSBlockDeviceSpec{ + VolumeSize: &rootSize, + }, + } + spec.BlockDevices = append(spec.BlockDevices, blockDevice) + rootVolume = &spec.BlockDevices[len(spec.BlockDevices)-1] + } + + // Ensure EBS is initialized + if rootVolume.EBS == nil { + rootSize := int64(120) + rootVolume.EBS = &machinev1.EBSBlockDeviceSpec{ + VolumeSize: &rootSize, + } + } + + // Ensure KMSKey is initialized + if rootVolume.EBS.KMSKey.ID == nil { + rootVolume.EBS.KMSKey = machinev1.AWSResourceReference{} + } + + // Apply KMS encryption + rootVolume.EBS.KMSKey.ID = &config.KMSKey.KeyID + encrypted := true + rootVolume.EBS.Encrypted = &encrypted + } + + // Re-serialize + providerSpec.Raw, err = json.Marshal(spec) + if err != nil { + return fmt.Errorf("failed to marshal providerspec: %w", err) + } + + return nil +} diff --git a/pkg/unified/config/test_config.go b/pkg/unified/config/test_config.go new file mode 100644 index 000000000..33c115ab6 --- /dev/null +++ b/pkg/unified/config/test_config.go @@ -0,0 +1,73 @@ +package config + +import ( + "fmt" + "os" + "strings" +) + +// BackendType represents the type of backend. +type BackendType string + +const ( + // BackendTypeMAPI represents MAPI backend. + BackendTypeMAPI BackendType = "MAPI" + // BackendTypeCAPI represents CAPI backend. + BackendTypeCAPI BackendType = "CAPI" +) + +// TestConfig defines test configuration. +type TestConfig struct { + // Backend type: MAPI or CAPI + BackendType BackendType + + // Authoritative API type: MAPI or CAPI + AuthoritativeAPI BackendType +} + +// LoadTestConfig loads test configuration from environment variables. +// Returns error if environment variables contain invalid values. +func LoadTestConfig() (*TestConfig, error) { + config := &TestConfig{ + BackendType: BackendTypeMAPI, // Default to MAPI + AuthoritativeAPI: BackendTypeMAPI, // Default to MAPI + } + + // Read and validate TEST_BACKEND_TYPE + if backendType := strings.TrimSpace(os.Getenv("TEST_BACKEND_TYPE")); backendType != "" { + backendTypeUpper := strings.ToUpper(backendType) + switch backendTypeUpper { + case "MAPI": + config.BackendType = BackendTypeMAPI + case "CAPI": + config.BackendType = BackendTypeCAPI + default: + return nil, fmt.Errorf("invalid TEST_BACKEND_TYPE value %q: must be 'MAPI' or 'CAPI'", backendType) + } + } + + // Read and validate TEST_AUTHORITATIVE_API + if authAPI := strings.TrimSpace(os.Getenv("TEST_AUTHORITATIVE_API")); authAPI != "" { + authAPIUpper := strings.ToUpper(authAPI) + switch authAPIUpper { + case "MAPI": + config.AuthoritativeAPI = BackendTypeMAPI + case "CAPI": + config.AuthoritativeAPI = BackendTypeCAPI + default: + return nil, fmt.Errorf("invalid TEST_AUTHORITATIVE_API value %q: must be 'MAPI' or 'CAPI'", authAPI) + } + } + + return config, nil +} + +// GetTestConfigOrDie loads test config and panics on error. +func GetTestConfigOrDie() *TestConfig { + cfg, err := LoadTestConfig() + if err != nil { + panic(fmt.Sprintf("Failed to load test config: %v", err)) + } + + return cfg +} diff --git a/pkg/unified/e2e/aws_machineset.go b/pkg/unified/e2e/aws_machineset.go new file mode 100644 index 000000000..18de471af --- /dev/null +++ b/pkg/unified/e2e/aws_machineset.go @@ -0,0 +1,152 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + testframework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/backends" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/config" + "k8s.io/utils/ptr" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var testConfig = config.GetTestConfigOrDie() + +var _ = Describe(fmt.Sprintf("MachineSet creation on AWS [Backend=%s, Authority=%s]", + testConfig.BackendType, testConfig.AuthoritativeAPI), + testframework.LabelDisruptive, testframework.LabelUnified, Ordered, func() { + var framework *unified.UnifiedFramework + + var cl runtimeclient.Client + + var ctx context.Context + + var platform configv1.PlatformType + + var helper *TestHelper + + BeforeAll(func() { + var err error + + framework = unified.NewUnifiedFramework() + By(fmt.Sprintf("Verifying backend=%s, authority=%s", + framework.GetBackendType(), framework.GetAuthoritativeAPI())) + + cl, err = testframework.LoadClient() + Expect(err).NotTo(HaveOccurred(), "Should load client") + komega.SetClient(cl) + + ctx = testframework.GetContext() + platform, err = testframework.GetPlatform(ctx, cl) + Expect(err).NotTo(HaveOccurred(), "Should get platform") + + helper = NewTestHelper(ctx, framework, cl, platform, nil) + helper.SkipIfNotPlatform(configv1.AWSPlatformType) + }) + + Context("when creating a new MachineSet", func() { + It("should create and wait for machines to run", func() { + By("Creating a MachineTemplate") + + template := helper.CreateTemplate(generateName("machineset-template-")) + DeferCleanup(helper.DeleteTemplate, template) + + By("Creating a MachineSet from the template") + + machineSet := helper.CreateMachineSet(generateName("machineset-"), template, nil) + DeferCleanup(helper.DeleteMachineSet, machineSet) + + By("Waiting for Machines to become Running") + Expect(framework.WaitForMachinesRunning(ctx, cl, machineSet)).To(Succeed(), "Should have machines running") + }) + }) + + Context("when using spot instances", func() { + It("should create machines with spot market options", func() { + By("Creating a MachineTemplate with spot instance configuration") + + spotConfig := &config.MachineTemplateConfig{ + AWS: &config.AWSMachineConfig{ + SpotMarketOptions: &config.SpotMarketConfig{ + MaxPrice: nil, // Use default price. + }, + Tenancy: ptr.To("default"), + }, + } + + template, err := framework.CreateMachineTemplate(ctx, cl, platform, backends.BackendMachineTemplateParams{ + Name: generateName("spot-template-"), + Platform: platform, + Spec: spotConfig, + }) + Expect(err).NotTo(HaveOccurred(), "Should create Machine Template with spot configuration") + DeferCleanup(helper.DeleteTemplate, template) + + By("Creating a MachineSet from the spot template") + + machineSet := helper.CreateMachineSet(generateName("spot-machineset-"), template, nil) + DeferCleanup(helper.DeleteMachineSet, machineSet) + + By("Waiting for Machines to become Running or skipping on capacity error") + helper.WaitForMachinesRunningOrSkipOnCapacityError(machineSet) + + By("Verifying spot instance configuration is correctly applied") + helper.VerifyMachineSetContainsString(machineSet, "spot") + }) + }) + + Context("when using EFA network interface", func() { + var region string + + BeforeEach(func() { + By("Checking if region supports EFA") + + region = helper.GetRegion() + + if region != "us-east-2" && region != "us-west-2" { + Skip(fmt.Sprintf("EFA test is only supported in us-east-2 and us-west-2, current region: %s", region)) + } + }) + + It("should create machines with EFA configuration", func() { + By("Creating a MachineTemplate with EFA network interface and c5n.9xlarge instance type") + + efaConfig := &config.MachineTemplateConfig{ + AWS: &config.AWSMachineConfig{ + InstanceType: ptr.To("c5n.9xlarge"), + NetworkInterfaceType: ptr.To("efa"), + }, + } + + template, err := framework.CreateMachineTemplate(ctx, cl, platform, backends.BackendMachineTemplateParams{ + Name: generateName("efa-template-"), + Platform: platform, + Spec: efaConfig, + }) + Expect(err).NotTo(HaveOccurred(), "Should create Machine Template with EFA network interface configuration") + DeferCleanup(helper.DeleteTemplate, template) + + By("Creating a MachineSet from the EFA template") + + machineSet := helper.CreateMachineSet(generateName("efa-machineset-"), template, nil) + DeferCleanup(helper.DeleteMachineSet, machineSet) + + By("Waiting for Machines to become Running or skipping on capacity error") + helper.WaitForMachinesRunningOrSkipOnCapacityError(machineSet) + + By("Verifying EFA network interface configuration is correctly applied") + // MAPI uses "EFA", CAPI uses "efa" + if framework.GetBackendType() == config.BackendTypeMAPI { + helper.VerifyMachineSetContainsString(machineSet, "EFA") + } else { + helper.VerifyMachineSetContainsString(machineSet, "efa") + } + }) + }) + }) diff --git a/pkg/unified/e2e/test_helpers.go b/pkg/unified/e2e/test_helpers.go new file mode 100644 index 000000000..149d26174 --- /dev/null +++ b/pkg/unified/e2e/test_helpers.go @@ -0,0 +1,233 @@ +package e2e + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + machinev1 "github.com/openshift/api/machine/v1beta1" + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/backends" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/config" + corev1 "k8s.io/api/core/v1" + utilrand "k8s.io/apimachinery/pkg/util/rand" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // Infrastructure reference kinds. + awsMachineKind = "AWSMachine" + awsMachineTemplateKind = "AWSMachineTemplate" + + // Kubernetes topology labels. + topologyRegionLabel = "topology.kubernetes.io/region" +) + +// generateName returns a unique resource name by appending a random suffix to +// the given prefix. This avoids name collisions between Ordered test contexts +// that run sequentially on the same cluster. +func generateName(prefix string) string { + return prefix + utilrand.String(5) +} + +// TestHelper provides common helper functions for unified framework tests. +type TestHelper struct { + framework *unified.UnifiedFramework + client runtimeclient.Client + ctx context.Context + platform configv1.PlatformType + machineSpec interface{} +} + +// NewTestHelper creates a new test helper instance. +func NewTestHelper(ctx context.Context, framework *unified.UnifiedFramework, client runtimeclient.Client, platform configv1.PlatformType, machineSpec interface{}) *TestHelper { + return &TestHelper{ + framework: framework, + client: client, + ctx: ctx, + platform: platform, + machineSpec: machineSpec, + } +} + +// CreateTemplate creates and validates a machine template. +func (helper *TestHelper) CreateTemplate(name string) interface{} { + GinkgoHelper() + + template, err := helper.framework.CreateMachineTemplate(helper.ctx, helper.client, helper.platform, backends.BackendMachineTemplateParams{ + Name: name, + Platform: helper.platform, + Spec: helper.machineSpec, + }) + Expect(err).NotTo(HaveOccurred(), "Should create Machine Template") + + return template +} + +// CreateMachineSet creates and validates a machine set. +func (helper *TestHelper) CreateMachineSet(name string, template interface{}, labels map[string]string) interface{} { + GinkgoHelper() + + machineSet, err := helper.framework.CreateMachineSet(helper.ctx, helper.client, backends.BackendMachineSetParams{ + Name: name, + Replicas: 1, + Labels: labels, + Annotations: map[string]string{"e2e": name}, + Template: template, + FailureDomain: "auto", + AuthoritativeAPI: helper.framework.GetAuthoritativeAPI(), // Use the framework's authoritative API setting + }) + Expect(err).NotTo(HaveOccurred(), "Should create MachineSet") + + return machineSet +} + +// ValidateStatus validates machine set status. +func (helper *TestHelper) ValidateStatus(machineSet interface{}) { + GinkgoHelper() + + status, err := helper.framework.GetMachineSetStatus(helper.ctx, helper.client, machineSet) + Expect(err).NotTo(HaveOccurred(), "Should get MachineSet status") + Expect(status.Replicas).To(BeNumerically(">=", 0), "Should have valid replica count") +} + +// WaitForMachinesRunningOrSkipOnCapacityError waits for machines to become running, +// but skips the test if InsufficientInstanceCapacity error is detected. +func (helper *TestHelper) WaitForMachinesRunningOrSkipOnCapacityError(machineSet interface{}) { + GinkgoHelper() + + var err error + + switch helper.framework.GetBackendType() { + case config.BackendTypeMAPI: + ms, ok := machineSet.(*machinev1.MachineSet) + Expect(ok).To(BeTrue(), "Should be MAPI MachineSet, got %T", machineSet) + // WaitForSpotMachineSet re-fetches Machines each poll cycle and inspects + // per-machine ProviderStatus for capacity conditions. + err = framework.WaitForSpotMachineSet(helper.ctx, helper.client, ms.Name) + case config.BackendTypeCAPI: + ms, ok := machineSet.(*clusterv1beta1.MachineSet) + Expect(ok).To(BeTrue(), "Should be CAPI MachineSet, got %T", machineSet) + // WaitForCAPIMachinesRunningWithRetry re-fetches InfraMachines each poll + // cycle and searches their status for the given error keys. + err = framework.WaitForCAPIMachinesRunningWithRetry(helper.ctx, helper.client, ms.Name, + []string{"InsufficientInstanceCapacity"}) + default: + Expect(false).To(BeTrue(), "Should have supported backend type, got %s", helper.framework.GetBackendType()) + } + + if errors.Is(err, framework.ErrMachineNotProvisionedInsufficientCloudCapacity) { + Skip("Skipping test: insufficient cloud provider capacity in the requested Availability Zone") + } + + Expect(err).NotTo(HaveOccurred(), "Should have machines running") +} + +// DeleteTemplate safely deletes a machine template. +func (helper *TestHelper) DeleteTemplate(template interface{}) { + GinkgoHelper() + + err := helper.framework.DeleteMachineTemplate(helper.ctx, helper.client, template) + Expect(err).NotTo(HaveOccurred(), "Should delete machine template") +} + +// DeleteMachineSet safely deletes a machine set. +func (helper *TestHelper) DeleteMachineSet(machineSet interface{}) { + GinkgoHelper() + + err := helper.framework.DeleteMachineSet(helper.ctx, helper.client, machineSet) + Expect(err).NotTo(HaveOccurred(), "Should delete MachineSet") +} + +// SkipIfNotPlatform skips the test if the platform does not match the required platform. +func (helper *TestHelper) SkipIfNotPlatform(requiredPlatform configv1.PlatformType) { + GinkgoHelper() + + if helper.platform != requiredPlatform { + Skip(fmt.Sprintf("These features are only supported on %s platform", requiredPlatform)) + } +} + +// VerifyMachineSetContainsString verifies that MachineSet contains specific strings (fuzzy matching). +func (helper *TestHelper) VerifyMachineSetContainsString(machineSet interface{}, searchStrings ...string) { + GinkgoHelper() + + switch machineSetType := machineSet.(type) { + case *machinev1.MachineSet: + helper.verifyMAPIMachineSetContainsString(machineSetType, searchStrings...) + case *clusterv1beta1.MachineSet: + helper.verifyCAPIMachineSetContainsString(machineSetType, searchStrings...) + default: + Expect(false).To(BeTrue(), "Should have supported MachineSet type, got %T", machineSet) + } +} + +// verifyMAPIMachineSetContainsString verifies MAPI MachineSet contains specific strings. +func (helper *TestHelper) verifyMAPIMachineSetContainsString(machineSet *machinev1.MachineSet, searchStrings ...string) { + GinkgoHelper() + + Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).NotTo(BeNil(), "Should have ProviderSpec in MAPI MachineSet") + + // Convert entire ProviderSpec to string for searching. + configBytes := machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw + configString := string(configBytes) + + for _, searchString := range searchStrings { + Expect(configString).To(ContainSubstring(searchString), + fmt.Sprintf("Should contain string '%s' in MAPI MachineSet ProviderSpec", searchString)) + } +} + +// verifyCAPIMachineSetContainsString verifies CAPI MachineSet contains specific strings. +func (helper *TestHelper) verifyCAPIMachineSetContainsString(machineSet *clusterv1beta1.MachineSet, searchStrings ...string) { + GinkgoHelper() + + // For CAPI, we need to check the associated MachineTemplate (AWSMachineTemplate). + infraRef := machineSet.Spec.Template.Spec.InfrastructureRef + Expect(infraRef.Kind).To(Equal(awsMachineTemplateKind), "Should have %s infrastructure reference, got %s", awsMachineTemplateKind, infraRef.Kind) + + // Get the AWSMachineTemplate. + awsTemplate := &awsv1.AWSMachineTemplate{} + err := helper.client.Get(helper.ctx, runtimeclient.ObjectKey{ + Name: infraRef.Name, + Namespace: infraRef.Namespace, + }, awsTemplate) + Expect(err).NotTo(HaveOccurred(), "Should get AWSMachineTemplate %s/%s", infraRef.Namespace, infraRef.Name) + + machineSpecBytes, err := json.Marshal(awsTemplate.Spec.Template.Spec) + Expect(err).NotTo(HaveOccurred(), "Should marshal AWSMachineTemplate Spec.Template.Spec") + + configString := string(machineSpecBytes) + + for _, searchString := range searchStrings { + Expect(configString).To(ContainSubstring(searchString), + fmt.Sprintf("Should contain string '%s' in CAPI AWSMachineTemplate Spec.Template.Spec", searchString)) + } +} + +// GetRegion retrieves the region from node labels for any cloud platform. +func (helper *TestHelper) GetRegion() string { + GinkgoHelper() + + // Get the first node to read the region label + nodeList := &corev1.NodeList{} + err := helper.client.List(helper.ctx, nodeList, runtimeclient.Limit(1)) + Expect(err).NotTo(HaveOccurred(), "Should list nodes") + + Expect(nodeList.Items).NotTo(BeEmpty(), "Should have at least one node in the cluster") + + // Get region from the topology label + node := nodeList.Items[0] + region, ok := node.Labels[topologyRegionLabel] + Expect(ok).To(BeTrue(), "Should have %s label on node %s", topologyRegionLabel, node.Name) + Expect(region).NotTo(BeEmpty(), "Should have non-empty region label on node %s", node.Name) + + return region +} diff --git a/pkg/unified/framework.go b/pkg/unified/framework.go new file mode 100644 index 000000000..f34d3f56a --- /dev/null +++ b/pkg/unified/framework.go @@ -0,0 +1,89 @@ +package unified + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + corev1 "k8s.io/api/core/v1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/backends" + "github.com/openshift/cluster-api-actuator-pkg/pkg/unified/config" +) + +// UnifiedFramework provides a unified testing framework interface. +type UnifiedFramework struct { + config *config.TestConfig + backend backends.MachineBackend +} + +// NewUnifiedFramework creates a new unified testing framework. +func NewUnifiedFramework() *UnifiedFramework { + GinkgoHelper() + + testConfig, err := config.LoadTestConfig() + Expect(err).NotTo(HaveOccurred(), "Should load test config from environment variables") + + backend, err := backends.NewBackend( + testConfig.BackendType, + testConfig.AuthoritativeAPI, + ) + Expect(err).NotTo(HaveOccurred(), "Should create backend") + + return &UnifiedFramework{ + config: testConfig, + backend: backend, + } +} + +// GetBackendType returns the backend type. +func (framework *UnifiedFramework) GetBackendType() config.BackendType { + return framework.config.BackendType +} + +// GetAuthoritativeAPI returns the authoritative API type. +func (framework *UnifiedFramework) GetAuthoritativeAPI() config.BackendType { + return framework.config.AuthoritativeAPI +} + +// CreateMachineSet creates a machine set. +func (framework *UnifiedFramework) CreateMachineSet(ctx context.Context, client runtimeclient.Client, params backends.BackendMachineSetParams) (interface{}, error) { + return framework.backend.CreateMachineSet(ctx, client, params) +} + +// DeleteMachineSet deletes a machine set. +func (framework *UnifiedFramework) DeleteMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + return framework.backend.DeleteMachineSet(ctx, client, machineSet) +} + +// WaitForMachineSetDeleted waits for machine set deletion. +func (framework *UnifiedFramework) WaitForMachineSetDeleted(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + return framework.backend.WaitForMachineSetDeleted(ctx, client, machineSet) +} + +// WaitForMachinesRunning waits for all machines belonging to the machine set to enter the "Running" phase. +func (framework *UnifiedFramework) WaitForMachinesRunning(ctx context.Context, client runtimeclient.Client, machineSet interface{}) error { + return framework.backend.WaitForMachinesRunning(ctx, client, machineSet) +} + +// GetMachineSetStatus returns the machine set status. +func (framework *UnifiedFramework) GetMachineSetStatus(ctx context.Context, client runtimeclient.Client, machineSet interface{}) (*backends.MachineSetStatus, error) { + return framework.backend.GetMachineSetStatus(ctx, client, machineSet) +} + +// GetNodesFromMachineSet returns nodes from a machine set. +func (framework *UnifiedFramework) GetNodesFromMachineSet(ctx context.Context, client runtimeclient.Client, machineSet interface{}) ([]corev1.Node, error) { + return framework.backend.GetNodesFromMachineSet(ctx, client, machineSet) +} + +// CreateMachineTemplate creates a machine template. +func (framework *UnifiedFramework) CreateMachineTemplate(ctx context.Context, client runtimeclient.Client, platform configv1.PlatformType, params backends.BackendMachineTemplateParams) (interface{}, error) { + return framework.backend.CreateMachineTemplate(ctx, client, platform, params) +} + +// DeleteMachineTemplate deletes a machine template. +func (framework *UnifiedFramework) DeleteMachineTemplate(ctx context.Context, client runtimeclient.Client, template interface{}) error { + return framework.backend.DeleteMachineTemplate(ctx, client, template) +} diff --git a/vendor/k8s.io/apimachinery/pkg/util/rand/rand.go b/vendor/k8s.io/apimachinery/pkg/util/rand/rand.go new file mode 100644 index 000000000..82a473bb1 --- /dev/null +++ b/vendor/k8s.io/apimachinery/pkg/util/rand/rand.go @@ -0,0 +1,127 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package rand provides utilities related to randomization. +package rand + +import ( + "math/rand" + "sync" + "time" +) + +var rng = struct { + sync.Mutex + rand *rand.Rand +}{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), +} + +// Int returns a non-negative pseudo-random int. +func Int() int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Int() +} + +// Intn generates an integer in range [0,max). +// By design this should panic if input is invalid, <= 0. +func Intn(max int) int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Intn(max) +} + +// IntnRange generates an integer in range [min,max). +// By design this should panic if input is invalid, <= 0. +func IntnRange(min, max int) int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Intn(max-min) + min +} + +// IntnRange generates an int64 integer in range [min,max). +// By design this should panic if input is invalid, <= 0. +func Int63nRange(min, max int64) int64 { + rng.Lock() + defer rng.Unlock() + return rng.rand.Int63n(max-min) + min +} + +// Seed seeds the rng with the provided seed. +func Seed(seed int64) { + rng.Lock() + defer rng.Unlock() + + rng.rand = rand.New(rand.NewSource(seed)) +} + +// Perm returns, as a slice of n ints, a pseudo-random permutation of the integers [0,n) +// from the default Source. +func Perm(n int) []int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Perm(n) +} + +const ( + // We omit vowels from the set of available characters to reduce the chances + // of "bad words" being formed. + alphanums = "bcdfghjklmnpqrstvwxz2456789" + // No. of bits required to index into alphanums string. + alphanumsIdxBits = 5 + // Mask used to extract last alphanumsIdxBits of an int. + alphanumsIdxMask = 1<>= alphanumsIdxBits + remaining-- + } + return string(b) +} + +// SafeEncodeString encodes s using the same characters as rand.String. This reduces the chances of bad words and +// ensures that strings generated from hash functions appear consistent throughout the API. +func SafeEncodeString(s string) string { + r := make([]byte, len(s)) + for i, b := range []rune(s) { + r[i] = alphanums[(int(b) % len(alphanums))] + } + return string(r) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index fac490511..b02cbb63c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1894,6 +1894,7 @@ k8s.io/apimachinery/pkg/util/managedfields/internal k8s.io/apimachinery/pkg/util/mergepatch k8s.io/apimachinery/pkg/util/naming k8s.io/apimachinery/pkg/util/net +k8s.io/apimachinery/pkg/util/rand k8s.io/apimachinery/pkg/util/runtime k8s.io/apimachinery/pkg/util/sets k8s.io/apimachinery/pkg/util/strategicpatch