diff --git a/test/extended/node/README.md b/test/extended/node/README.md index 9a8feaa6ffb1..3dc4caa4d0d9 100644 --- a/test/extended/node/README.md +++ b/test/extended/node/README.md @@ -12,6 +12,7 @@ This directory contains OpenShift end-to-end tests for node-related features. - **node_e2e/image_registry_config.go** - Container registry config change (OCP-44820) - Verifies search registry update triggers MCO rollout and lands on nodes \[Disruptive\] - **node_e2e/netns_cleanup.go** - Network namespace cleanup - Verifies kubelet/CRI-O properly deletes network namespace when a pod is deleted \[OTP\] - **node_e2e/pdb_drain.go** - PodDisruptionBudget drain blocking (OCP-67564) - Tests that node drain is blocked when PDB has minAvailable=100% with empty selector \[Disruptive\] \[Lifecycle:informing\] +- **additional_storage_e2e.go** - Additional Storage configuration and functional tests - Verifies ContainerRuntimeConfig with additionalImageStores, additionalLayerStores, and additionalArtifactStores. Tests configuration in storage.conf and CRI-O config, image store pre-population and registry fallback [Disruptive] ### Suite: openshift/conformance/parallel diff --git a/test/extended/node/additional_storage_e2e.go b/test/extended/node/additional_storage_e2e.go new file mode 100644 index 000000000000..b79a368d7b52 --- /dev/null +++ b/test/extended/node/additional_storage_e2e.go @@ -0,0 +1,233 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + machineconfigv1 "github.com/openshift/api/machineconfiguration/v1" + mcclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" + exutil "github.com/openshift/origin/test/extended/util" +) + +// Additional Storage E2E Tests - trigger MCO reconciliation (MCP rollouts) +// and run in the disruptive-longrunning suite. +var _ = g.Describe("[Skipped:Disconnected][apigroup:config.openshift.io][apigroup:machineconfiguration.openshift.io][Jira:Node/CRI-O][sig-node][Feature:AdditionalStorageSupport][OCPFeatureGate:AdditionalStorageConfig][Serial][Disruptive][Suite:openshift/disruptive-longrunning] Additional Storage E2E Tests", func() { + defer g.GinkgoRecover() + + var oc = exutil.NewCLI("additional-storage-e2e") + + g.BeforeEach(func(ctx context.Context) { + SkipOnMicroShift(oc) + EnsureNodesReady(ctx, oc) + + enabled, reason := IsAdditionalStorageConfigEnabled(ctx, oc) + if !enabled { + g.Skip(reason) + } + }) + + // This test verifies configuration for all three storage types (image, layer, artifact) + // and performs functional testing for image stores (pre-population and fallback). + // Artifact stores are tested with filesystem operations, and layer stores are verified + // via storage.conf :ref suffix configuration. + g.It("should verify configuration of all storage types and functionally test image stores", func(ctx context.Context) { + mcClient, err := mcclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + + // SETUP: Create single-node MachineConfigPool for faster rollouts + g.By("Getting worker node for test") + testNode := GetFirstReadyWorkerNode(oc) + + mcpName := "combined-func-test" + var mcpConfig *CustomMCPConfig + + g.By("Creating single-node MachineConfigPool for test isolation") + mcpConfig, err = CreateCustomMCPForNode(ctx, oc, mcClient, mcpName, testNode) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Custom MCP %s created, targeting node %s", mcpName, testNode) + + testNamespace := oc.Namespace() + + // Phase 1: Setup directories and prepare for testing + g.By("Phase 1: Creating storage directories and pre-populating test image") + imageStorePath := "/var/lib/combined-imagestore" + artifactStorePath := "/var/lib/combined-artifactstore" + layerStorePath := "/var/lib/stargz-store/store" + allDirs := []string{imageStorePath, artifactStorePath, layerStorePath} + + g.DeferCleanup(cleanupDirectoriesOnNode, oc, testNode, allDirs) + g.DeferCleanup(func() { + cleanupCtx := context.Background() + if err := CleanupContainerRuntimeConfig(cleanupCtx, mcClient, "combined-func-test", mcpName); err != nil { + framework.Logf("Warning: failed to cleanup ContainerRuntimeConfig: %v", err) + } + if mcpConfig != nil { + if err := CleanupCustomMCP(cleanupCtx, mcpConfig); err != nil { + framework.Logf("Warning: failed to cleanup custom MCP: %v", err) + } + } + }) + + err = createDirectoriesOnNode(oc, testNode, allDirs) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Pre-populate test image in image store + testImage := "registry.k8s.io/e2e-test-images/agnhost:2.59" + framework.Logf("Pre-populating image %s to %s on node %s", testImage, imageStorePath, testNode) + + _, err = ExecOnNodeWithChroot(ctx, oc, testNode, "podman", "--root", imageStorePath, "pull", testImage) + o.Expect(err).NotTo(o.HaveOccurred()) + + lsOutput, err := ExecOnNodeWithChroot(ctx, oc, testNode, "podman", "--root", imageStorePath, "images", "--format", "{{.Repository}}:{{.Tag}}") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(lsOutput).To(o.ContainSubstring(testImage)) + framework.Logf("Image pre-populated successfully: %s", lsOutput) + + // Phase 2: Create ContainerRuntimeConfig with all three storage types + g.By("Phase 2: Creating ContainerRuntimeConfig with all three storage types") + ctrcfg := &machineconfigv1.ContainerRuntimeConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "combined-func-test", + }, + Spec: machineconfigv1.ContainerRuntimeConfigSpec{ + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "machineconfiguration.openshift.io/pool": mcpName, + }, + }, + ContainerRuntimeConfig: &machineconfigv1.ContainerRuntimeConfiguration{ + AdditionalLayerStores: []machineconfigv1.AdditionalLayerStore{ + {Path: machineconfigv1.StorePath(layerStorePath)}, + }, + AdditionalImageStores: []machineconfigv1.AdditionalImageStore{ + {Path: machineconfigv1.StorePath(imageStorePath)}, + }, + AdditionalArtifactStores: []machineconfigv1.AdditionalArtifactStore{ + {Path: machineconfigv1.StorePath(artifactStorePath)}, + }, + }, + }, + } + + g.By("Applying ContainerRuntimeConfig and waiting for MCP rollout") + err = ApplyContainerRuntimeConfigAndWaitForMCP(ctx, mcClient, ctrcfg, mcpName, 15*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred()) + + // CONFIGURATION VERIFICATION - verify configs without usage + + // Phase 3: Verify storage configuration + g.By("Phase 3: Verifying storage.conf and CRI-O configuration") + storageConfOutput, err := ExecOnNodeWithChroot(ctx, oc, testNode, "cat", "/etc/containers/storage.conf") + o.Expect(err).NotTo(o.HaveOccurred()) + + // Verify image stores + o.Expect(storageConfOutput).To(o.ContainSubstring(imageStorePath)) + framework.Logf("Image stores verified in storage.conf") + + // Verify layer stores with :ref suffix (MCO automatically appends :ref) + expectedPathWithRef := layerStorePath + ":ref" + o.Expect(storageConfOutput).To(o.ContainSubstring("additionallayerstores"), + "storage.conf should contain additionallayerstores on node %s", testNode) + o.Expect(storageConfOutput).To(o.ContainSubstring(expectedPathWithRef), + "storage.conf should contain %s with :ref suffix on node %s", expectedPathWithRef, testNode) + framework.Logf("Layer stores verified in storage.conf with path %s (MCO added :ref)", expectedPathWithRef) + + g.By("Verifying CRI-O config contains artifact stores") + crioOutput, err := ExecOnNodeWithChroot(ctx, oc, testNode, "cat", "/etc/crio/crio.conf.d/01-ctrcfg-additionalArtifactStores") + o.Expect(err).NotTo(o.HaveOccurred()) + + expectedArtifactConfig := fmt.Sprintf(`additional_artifact_stores = ["%s"]`, artifactStorePath) + o.Expect(crioOutput).To(o.ContainSubstring(expectedArtifactConfig)) + framework.Logf("Artifact stores verified in CRI-O config") + + g.By("Verifying CRI-O is active with new configuration") + crioStatus, err := ExecOnNodeWithChroot(ctx, oc, testNode, "systemctl", "is-active", "crio") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.TrimSpace(crioStatus)).To(o.Equal("active")) + framework.Logf("CRI-O is active") + + // FUNCTIONAL VERIFICATION - actually use the stores + + // Phase 4: Test image store functionality - prepopulation and fallback + g.By("Phase 4a: Functionally testing additionalImageStores - verify pre-populated image accessible") + + // Ensure image is NOT in primary CRI-O store to guarantee additional store is used + framework.Logf("Removing %s from primary CRI-O store to ensure additional store is tested", testImage) + _, _ = ExecOnNodeWithChroot(ctx, oc, testNode, "crictl", "rmi", testImage) + + // Test prepopulated image by creating a pod + // Use ImagePullPolicy: Never to force using only local stores (proves additional store is used) + g.By("Phase 4b: Creating pod using prepopulated image") + testPod1 := createTestPod("imagestore-prepop-pod", testNamespace, testImage, testNode, corev1.PullNever) + startTime1 := time.Now() + _ = createPodAndWaitForRunning(ctx, oc, testPod1) + pod1Time := time.Since(startTime1) + framework.Logf("Pod using prepopulated image started in %v", pod1Time) + + // Test fallback to registry + g.By("Phase 5: Testing fallback to registry when image removed from additional store") + err = oc.AdminKubeClient().CoreV1().Pods(testNamespace).Delete(ctx, testPod1.Name, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := oc.AdminKubeClient().CoreV1().Pods(testNamespace).Get(ctx, testPod1.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Remove image from additional store + _, err = ExecOnNodeWithChroot(ctx, oc, testNode, "podman", "--root", imageStorePath, "rmi", testImage) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Removed image from additional store to test fallback") + + // Create second pod - should fall back to registry + // Use ImagePullPolicy: Always to force registry pull (proves fallback works) + testPod2 := createTestPod("imagestore-fallback-pod", testNamespace, testImage, testNode, corev1.PullAlways) + startTime2 := time.Now() + _ = createPodAndWaitForRunning(ctx, oc, testPod2) + g.DeferCleanup(deletePodAndWait, ctx, oc, testNamespace, testPod2.Name) + pod2Time := time.Since(startTime2) + framework.Logf("Pod using registry fallback started in %v", pod2Time) + + // Verify pod pulled from registry + var foundSuccessfullyPulled bool + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + events, err := oc.AdminKubeClient().CoreV1().Events(testNamespace).List(ctx, metav1.ListOptions{ + FieldSelector: fmt.Sprintf("involvedObject.name=%s", testPod2.Name), + }) + if err != nil { + return false, err + } + for _, event := range events.Items { + if event.Reason == "Pulled" && strings.Contains(event.Message, "Successfully pulled") { + foundSuccessfullyPulled = true + framework.Logf("SUCCESS: Image pulled from registry - %s", event.Message) + return true, nil + } + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(foundSuccessfullyPulled).To(o.BeTrue()) + framework.Logf("Phase 5 PASSED: Fallback to registry works when image not in additional store") + + framework.Logf("Test PASSED: Configuration verified for all storage types and image stores functionally tested") + }) +}) diff --git a/test/extended/node/image_volume.go b/test/extended/node/image_volume.go index 6a1a7163a127..cf2cb5ee6287 100644 --- a/test/extended/node/image_volume.go +++ b/test/extended/node/image_volume.go @@ -149,20 +149,6 @@ func describeImageVolumeTests(config imageVolumeTestConfig) bool { }) } -func createPodAndWaitForRunning(ctx context.Context, oc *exutil.CLI, pod *v1.Pod) *v1.Pod { - g.By("Creating a pod") - _, err := oc.AdminKubeClient().CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - - g.By("Waiting for pod to be running") - err = e2epod.WaitForPodRunningInNamespace(ctx, oc.AdminKubeClient(), pod) - o.Expect(err).NotTo(o.HaveOccurred()) - - created, err := oc.AdminKubeClient().CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - return created -} - func verifyPathsExist(f *framework.Framework, pod *v1.Pod, mountPoint string, paths []string) { args := []string{"ls"} for _, p := range paths { diff --git a/test/extended/node/node_kc_helpers.go b/test/extended/node/node_kc_cc_helpers.go similarity index 65% rename from test/extended/node/node_kc_helpers.go rename to test/extended/node/node_kc_cc_helpers.go index c137c5ddcee2..a1327a192faa 100644 --- a/test/extended/node/node_kc_helpers.go +++ b/test/extended/node/node_kc_cc_helpers.go @@ -105,3 +105,51 @@ func WaitForMCPUpdating(ctx context.Context, mcClient *machineconfigclient.Clien time.Sleep(10 * time.Second) } } + +// CreateContainerRuntimeConfig creates a ContainerRuntimeConfig resource. +func CreateContainerRuntimeConfig(ctx context.Context, mcClient *machineconfigclient.Clientset, ctrcfg *machineconfigv1.ContainerRuntimeConfig) (*machineconfigv1.ContainerRuntimeConfig, error) { + framework.Logf("Creating ContainerRuntimeConfig %s", ctrcfg.Name) + created, err := mcClient.MachineconfigurationV1().ContainerRuntimeConfigs().Create(ctx, ctrcfg, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return created, nil +} + +// CleanupContainerRuntimeConfig deletes a ContainerRuntimeConfig. If mcpName is non-empty, waits for that MCP to stabilize. +func CleanupContainerRuntimeConfig(ctx context.Context, mcClient *machineconfigclient.Clientset, ctrcfgName, mcpName string) error { + framework.Logf("Cleaning up ContainerRuntimeConfig %s", ctrcfgName) + + deleteErr := mcClient.MachineconfigurationV1().ContainerRuntimeConfigs().Delete(ctx, ctrcfgName, metav1.DeleteOptions{}) + if deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + + if mcpName != "" && (deleteErr == nil || apierrors.IsNotFound(deleteErr)) { + framework.Logf("Waiting for MCP %s to become ready after ContainerRuntimeConfig deletion", mcpName) + waitErr := WaitForMCP(ctx, mcClient, mcpName, 15*time.Minute) + if waitErr != nil && !apierrors.IsNotFound(waitErr) { + return waitErr + } + } + + framework.Logf("ContainerRuntimeConfig %s cleaned up successfully", ctrcfgName) + return nil +} + +// ApplyContainerRuntimeConfigAndWaitForMCP creates a ContainerRuntimeConfig and waits for the MCP rollout to complete. +func ApplyContainerRuntimeConfigAndWaitForMCP(ctx context.Context, mcClient *machineconfigclient.Clientset, ctrcfg *machineconfigv1.ContainerRuntimeConfig, mcpName string, rolloutTimeout time.Duration) error { + _, err := CreateContainerRuntimeConfig(ctx, mcClient, ctrcfg) + if err != nil { + return err + } + + framework.Logf("Waiting for MCP %s to start updating", mcpName) + err = WaitForMCPUpdating(ctx, mcClient, mcpName, 5*time.Minute) + if err != nil { + return err + } + + framework.Logf("Waiting for MCP %s to complete rollout", mcpName) + return WaitForMCP(ctx, mcClient, mcpName, rolloutTimeout) +} diff --git a/test/extended/node/node_utils.go b/test/extended/node/node_utils.go index 7ce343742b81..273cde6aa36e 100644 --- a/test/extended/node/node_utils.go +++ b/test/extended/node/node_utils.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/utils/ptr" machineconfigv1 "github.com/openshift/api/machineconfiguration/v1" machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" @@ -993,3 +995,111 @@ func EnsureNodesReady(ctx context.Context, oc *exutil.CLI) { o.Expect(notReadyNodes).To(o.BeEmpty(), "Cannot start test: nodes not Ready: %v. Cluster may be recovering from previous test.", notReadyNodes) } + +// ============================================================================ +// Helper functions for Additional Storage E2E tests +// ============================================================================ + +func createTestPod(name, namespace, image, nodeName string, pullPolicy corev1.PullPolicy) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + Tolerations: []corev1.Toleration{ + { + Operator: corev1.TolerationOpExists, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To[int64](1000), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + Containers: []corev1.Container{ + { + Name: "test", + Image: image, + Command: []string{"/agnhost", "pause"}, + ImagePullPolicy: pullPolicy, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } +} + +func createPodAndWaitForRunning(ctx context.Context, oc *exutil.CLI, pod *corev1.Pod) *corev1.Pod { + _, err := oc.AdminKubeClient().CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = e2epod.WaitForPodRunningInNamespace(ctx, oc.AdminKubeClient(), pod) + o.Expect(err).NotTo(o.HaveOccurred()) + + created, err := oc.AdminKubeClient().CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + return created +} + +func createDirectoriesOnNode(oc *exutil.CLI, nodeName string, dirs []string) error { + args := append([]string{"mkdir", "-p"}, dirs...) + _, err := ExecOnNodeWithChroot(context.Background(), oc, nodeName, args...) + if err != nil { + return fmt.Errorf("failed to create directories %v on node %s: %v", dirs, nodeName, err) + } + framework.Logf("Node %s: directories created: %v", nodeName, dirs) + return nil +} + +func cleanupDirectoriesOnNode(oc *exutil.CLI, nodeName string, dirs []string) { + args := append([]string{"rm", "-rf"}, dirs...) + _, err := ExecOnNodeWithChroot(context.Background(), oc, nodeName, args...) + if err != nil { + framework.Logf("Warning: failed to cleanup directories %v on node %s: %v", dirs, nodeName, err) + } +} + +// deletePodAndWait deletes a pod and waits for it to be gone +func deletePodAndWait(ctx context.Context, oc *exutil.CLI, namespace, podName string) { + err := oc.AdminKubeClient().CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Logf("Warning: failed to delete pod %s/%s: %v", namespace, podName, err) + return + } + if apierrors.IsNotFound(err) { + framework.Logf("Pod %s/%s already deleted", namespace, podName) + return + } + framework.Logf("Deleting pod %s/%s", namespace, podName) + + // Wait for pod to be deleted + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := oc.AdminKubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + }) + if err != nil { + framework.Logf("Warning: timeout waiting for pod %s/%s deletion: %v", namespace, podName, err) + } else { + framework.Logf("Pod %s/%s deleted successfully", namespace, podName) + } +}