[WIP] NE-2789: Creating 5 origin tests to graduate featuregate #31392
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,289 @@ | ||
| package router | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "strings" | ||
| "time" | ||
|
|
||
| g "github.com/onsi/ginkgo/v2" | ||
| o "github.com/onsi/gomega" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/labels" | ||
| "k8s.io/apimachinery/pkg/types" | ||
| "k8s.io/apimachinery/pkg/util/wait" | ||
| "k8s.io/apiserver/pkg/storage/names" | ||
| "k8s.io/utils/ptr" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| operatorv1 "github.com/openshift/api/operator/v1" | ||
|
|
||
| "github.com/openshift/origin/test/extended/router/shard" | ||
| exutil "github.com/openshift/origin/test/extended/util" | ||
| e2e "k8s.io/kubernetes/test/e2e/framework" | ||
| ) | ||
|
|
||
| var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift.io][OCPFeatureGate:IngressControllerMultipleHAProxyVersions]", func() { | ||
| defer g.GinkgoRecover() | ||
|
|
||
| // testsTimeout defines the maximum amount of time to wait for test operations to complete. | ||
| const testsTimeout = 5 * time.Minute | ||
|
|
||
| // defaultHAProxyVersion is the default HAProxy version for the current release. | ||
| var defaultHAProxyVersion operatorv1.HAProxyVersion | ||
|
|
||
| //alternateHAProxyVersion is the other accepted HAProxyVersion accepted in the current release | ||
| var alternateHAProxyVersion operatorv1.HAProxyVersion | ||
|
|
||
| // controllers is used to create new ingress controllers, and stores their reference so they can be removed after the test runs | ||
| var controllers ingressControllers | ||
|
|
||
| ctx := context.Background() | ||
| oc := exutil.NewCLI("router-select-haproxy-version").AsAdmin() | ||
| operatorClient := oc.AdminOperatorClient() | ||
|
|
||
| g.AfterEach(func() { | ||
| if g.CurrentSpecReport().Failed() { | ||
| for _, ic := range controllers.items { | ||
| exutil.DumpPodLogsStartingWithInNamespace(ic.controller.Name, ic.controller.Namespace, oc) | ||
| } | ||
| } | ||
| var errs []error | ||
| for _, ic := range controllers.items { | ||
| err := operatorClient.OperatorV1().IngressControllers(ic.controller.Namespace).Delete(ctx, ic.controller.Name, *metav1.NewDeleteOptions(1)) | ||
| errs = append(errs, client.IgnoreNotFound(err)) | ||
| } | ||
| o.Expect(errors.Join(errs...)).NotTo(o.HaveOccurred()) | ||
| controllers.items = nil | ||
| }) | ||
|
|
||
| g.BeforeEach(func() { | ||
| defaultIC, err := operatorClient.OperatorV1().IngressControllers("openshift-ingress-operator").Get(ctx, "default", metav1.GetOptions{}) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
| o.Expect(defaultIC.Status.EffectiveHAProxyVersion).NotTo(o.BeEmpty()) | ||
| defaultHAProxyVersion = defaultIC.Status.EffectiveHAProxyVersion | ||
|
|
||
| if defaultHAProxyVersion == operatorv1.HAProxyVersion28 { | ||
| alternateHAProxyVersion = operatorv1.HAProxyVersion32 | ||
| } else { | ||
| alternateHAProxyVersion = operatorv1.HAProxyVersion28 | ||
| } | ||
| }) | ||
|
rhamini3 marked this conversation as resolved.
|
||
|
|
||
| g.Describe("The HAProxy router with version selection", func() { | ||
|
|
||
| // Ensure that the haproxyVersion field in the IngressController API does not accept unknown versions | ||
| g.It("should reject invalid HAProxy versions", func() { | ||
| versions := []operatorv1.HAProxyVersion{ | ||
| " ", // Empty becomes unset, but space is invalid | ||
| "2.6", // one LTS before the oldest supported version, so always invalid | ||
| "v" + defaultHAProxyVersion, // v prefix is invalid | ||
| defaultHAProxyVersion + ".0", // .z suffix is invalid, only x.y is supported | ||
| " " + defaultHAProxyVersion, // leading space is invalid | ||
| defaultHAProxyVersion + " ", // trailing space is invalid | ||
| } | ||
| for _, version := range versions { | ||
| _, err := controllers.createIngressController(ctx, oc, testsTimeout, func(ic *operatorv1.IngressController) { | ||
| ic.Spec.HAProxyVersion = version | ||
| }) | ||
| o.Expect(err).To(o.Not(o.Succeed())) | ||
| o.Expect(err.Error()).To(o.ContainSubstring(`Unsupported value: "%s": supported values: `, version)) | ||
| } | ||
| }) | ||
|
|
||
| // Ensure that the ingress controller reverts back to the default version after unsetting the field with null | ||
| g.It("should revert to default HAProxy version when field is cleared", func() { | ||
| ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, func(ic *operatorv1.IngressController) { | ||
| ic.Spec.HAProxyVersion = alternateHAProxyVersion | ||
| }) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
|
|
||
| g.By("confirm the HAProxy version matches the controller version") | ||
| err = waitForHAProxyVersion(ctx, oc, ingress.Name, alternateHAProxyVersion) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
|
|
||
| g.By("Unset the version on the custom ingress controller") | ||
| patch := []byte(`{"spec":{"haproxyVersion":null}}`) | ||
| _, err = operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Patch(ctx, ingress.Name, types.MergePatchType, patch, metav1.PatchOptions{}) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
|
|
||
| g.By("Confirm that the HAProxy version shows default version") | ||
| err = waitForHAProxyVersion(ctx, oc, ingress.Name, defaultHAProxyVersion) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
| }) | ||
|
|
||
| // Ensure that the running HAProxy version matches the version configured in the IngressController API | ||
| g.It("should configure the same HAProxy version defined in the API", func() { | ||
| versions := []operatorv1.HAProxyVersion{ | ||
| defaultHAProxyVersion, | ||
| alternateHAProxyVersion, | ||
| } | ||
| for _, version := range versions { | ||
| ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, func(ic *operatorv1.IngressController) { | ||
| ic.Spec.HAProxyVersion = version | ||
| }) | ||
| o.Expect(err).To(o.Succeed()) | ||
| errPoll := wait.PollUntilContextTimeout(ctx, 2*time.Second, testsTimeout, true, func(ctx context.Context) (bool, error) { | ||
| ic, err := operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Get(ctx, ingress.Name, metav1.GetOptions{}) | ||
| if err != nil { | ||
| e2e.Logf("Failed to get the IngressController %s", ingress.Name) | ||
| return false, nil | ||
| } | ||
| if ic.Status.EffectiveHAProxyVersion == version { | ||
| e2e.Logf("EffectiveHAProxyVersion shows the expected version: %q", version) | ||
| return true, nil | ||
| } | ||
| e2e.Logf("EffectiveHAProxyVersion: %q does not match the expected version %q", ic.Status.EffectiveHAProxyVersion, version) | ||
| return false, nil | ||
| }) | ||
| o.Expect(errPoll).NotTo(o.HaveOccurred(), "Timed out waiting for EffectiveHAProxyVersion") | ||
| e2e.Logf("IngressController: %s matches the expected HAProxyVersion: %s", ingress.Name, string(version)) | ||
| } | ||
| }) | ||
|
|
||
| // Ensure that if the version is unset, its value in the IngressController API remains undeclared, and the running HAProxy matches the default version | ||
| g.It("should configure the default HAProxy if the version is unset", func() { | ||
| // create a custom ingress controller with an unset HAProxyVersion | ||
| ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, nil) | ||
| o.Expect(err).To(o.Succeed()) | ||
|
|
||
| //confirm that the ingresscontroller is unset | ||
| o.Expect(ingress.Spec.HAProxyVersion).To(o.BeEmpty()) | ||
|
|
||
| var effectiveVersion operatorv1.HAProxyVersion | ||
| err = wait.PollUntilContextTimeout(ctx, 2*time.Second, testsTimeout, true, func(ctx context.Context) (bool, error) { | ||
| ic, err := operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Get(ctx, ingress.Name, metav1.GetOptions{}) | ||
| if err != nil { | ||
| e2e.Logf("Failed to get the IngressController %s", ingress.Name) | ||
| return false, nil | ||
| } | ||
|
rhamini3 marked this conversation as resolved.
|
||
| if ic.Status.EffectiveHAProxyVersion == "" { | ||
| e2e.Logf("IngressController %s: EffectiveHAProxyVersion not yet set, waiting...", ingress.Name) | ||
| return false, nil | ||
| } | ||
| effectiveVersion = ic.Status.EffectiveHAProxyVersion | ||
| return true, nil | ||
| }) | ||
|
Comment on lines
+156
to
+168
Member
There was a problem hiding this comment. This poll seems to be pretty much the one on another test, but here it is polling the status. In case the other one polls status as well, wondering if it worth to extract both to a |
||
| o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for EffectiveHAProxyVersion") | ||
| o.Expect(effectiveVersion).To(o.Equal(defaultHAProxyVersion)) | ||
| e2e.Logf("IngressController: %s has the expected HAProxyVersion: %s", ingress.Name, string(defaultHAProxyVersion)) | ||
| err = waitForHAProxyVersion(ctx, oc, ingress.Name, defaultHAProxyVersion) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
| }) | ||
|
|
||
| // Ensure that changing the HAProxy version of a custom IngressController does not affect the running HAProxy of the default router | ||
| g.It("should maintain default router unchanged if updating the version on a custom IngressController", func() { | ||
| g.By("Grab the default ingresscontroller HAProxyVersion field") | ||
| ingressDefault, err := operatorClient.OperatorV1().IngressControllers("openshift-ingress-operator").Get(context.Background(), "default", metav1.GetOptions{}) | ||
|
rhamini3 marked this conversation as resolved.
|
||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
| ingressVersion := ingressDefault.Status.EffectiveHAProxyVersion | ||
|
|
||
| g.By("Create a custom controller and patch it to an older version") | ||
| ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, nil) | ||
| o.Expect(err).To(o.Succeed()) | ||
|
|
||
| patch := []byte(fmt.Sprintf(`{"spec":{"haproxyVersion":"%s"}}`, alternateHAProxyVersion)) | ||
| _, err = operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Patch(ctx, ingress.Name, types.MergePatchType, patch, metav1.PatchOptions{}) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
|
|
||
| g.By("Confirm the HaProxy version matches") | ||
| err = waitForHAProxyVersion(ctx, oc, ingress.Name, alternateHAProxyVersion) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
|
|
||
| g.By("Confirm the default router is still running the default HAProxy version") | ||
| err = waitForHAProxyVersion(ctx, oc, ingressDefault.Name, ingressVersion) | ||
| o.Expect(err).NotTo(o.HaveOccurred()) | ||
| }) | ||
|
|
||
| }) | ||
| }) | ||
|
|
||
| type ingressControllers struct { | ||
| items []*ingressController | ||
| } | ||
|
|
||
| type ingressController struct { | ||
| controller types.NamespacedName | ||
| } | ||
|
|
||
| func (i *ingressControllers) createIngressController(ctx context.Context, oc *exutil.CLI, readyTimeout time.Duration, custom func(ic *operatorv1.IngressController)) (*operatorv1.IngressController, error) { | ||
| operatorClient := oc.AdminOperatorClient() | ||
|
|
||
| // ingress controller need to be created in operator's namespace, ... | ||
| nsOperator := "openshift-ingress-operator" | ||
| controllerName := names.SimpleNameGenerator.GenerateName("e2e-haproxy-version-") | ||
|
|
||
| routeSelectorSet := labels.Set{"select": names.SimpleNameGenerator.GenerateName("haproxy-cfgmgr-")} | ||
|
|
||
| ic := operatorv1.IngressController{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Namespace: nsOperator, | ||
| Name: controllerName, | ||
| }, | ||
| Spec: operatorv1.IngressControllerSpec{ | ||
| Replicas: ptr.To[int32](1), | ||
| Domain: controllerName + ".router.local", | ||
| EndpointPublishingStrategy: &operatorv1.EndpointPublishingStrategy{ | ||
| Type: operatorv1.PrivateStrategyType, | ||
| Private: &operatorv1.PrivateStrategy{}, | ||
| }, | ||
| NamespaceSelector: metav1.SetAsLabelSelector(labels.Set{corev1.LabelMetadataName: oc.Namespace()}), | ||
| RouteSelector: metav1.SetAsLabelSelector(routeSelectorSet), | ||
| HAProxyVersion: "", | ||
| }, | ||
| } | ||
| if custom != nil { | ||
| custom(&ic) | ||
| } | ||
| ingress, err := operatorClient.OperatorV1().IngressControllers(nsOperator).Create(ctx, &ic, metav1.CreateOptions{}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| controller := types.NamespacedName{ | ||
| Namespace: nsOperator, | ||
| Name: controllerName, | ||
| } | ||
| ictr := ingressController{ | ||
| controller: controller, | ||
| } | ||
| i.items = append(i.items, &ictr) | ||
|
|
||
| ingressControllerReady := []operatorv1.OperatorCondition{ | ||
| {Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue}, | ||
| {Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse}, | ||
| {Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse}, | ||
| {Type: operatorv1.OperatorStatusTypeProgressing, Status: operatorv1.ConditionFalse}, | ||
| } | ||
|
|
||
| // wait for the controller to be available | ||
| err = shard.WaitForIngressControllerCondition(oc, readyTimeout, controller, ingressControllerReady...) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return ingress, nil | ||
| } | ||
|
|
||
| // poll the router pods HAProxy Container to check that the version is correctly asserted | ||
| func waitForHAProxyVersion(ctx context.Context, oc *exutil.CLI, ingressName string, desiredVersion operatorv1.HAProxyVersion) error { | ||
| if desiredVersion == "" { | ||
| return fmt.Errorf("desiredVersion must not be empty") | ||
| } | ||
| err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, false, func(ctx context.Context) (bool, error) { | ||
| haproxy, err := oc.Run("exec").Args("-n", "openshift-ingress", "-c", "haproxy", "deploy/router-"+ingressName, "--", "bash", "-c", "echo 'show version' | socat - /var/lib/haproxy/run/haproxy.sock").Output() | ||
| if err != nil { | ||
| e2e.Logf("Failed to extract the HAProxy Version from IngressController %s", ingressName) | ||
| return false, nil | ||
| } | ||
|
Comment on lines
+277
to
+280
Member
There was a problem hiding this comment. It worth logging here the reason of the failure as well. |
||
| if !strings.HasPrefix(haproxy, string(desiredVersion)) { | ||
| e2e.Logf("HAProxy version mismatch: got %q, waiting for %q", haproxy, desiredVersion) | ||
| return false, nil | ||
| } | ||
| e2e.Logf("IngressController: %s HAProxy has the correct version: %s", ingressName, haproxy) | ||
| return true, nil | ||
| }) | ||
| return err | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.