From b786cfe0dcb41755b7a8f9c0d226bd1f7c0ab0a2 Mon Sep 17 00:00:00 2001 From: John Kyros Date: Tue, 27 Jan 2026 00:24:06 -0600 Subject: [PATCH 1/2] Bump cluster autoscaler types for cordon flag Signed-off-by: John Kyros --- go.mod | 2 +- go.sum | 4 +- .../cluster-api/core/v1beta1/cluster.go | 16 ++++++++ .../cluster-api/core/v1beta1/machine.go | 24 +++++++++++ .../cluster-api/core/v1beta1/machineset.go | 40 +++++++++++++------ .../autoscaling/v1/clusterautoscaler_types.go | 22 ++++++++++ .../autoscaling/v1/zz_generated.deepcopy.go | 30 ++++++++++++++ vendor/modules.txt | 4 +- 8 files changed, 125 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 074e9ccc0..3aaaa4887 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/onsi/gomega v1.38.2 github.com/openshift/api v0.0.0-20251111193948-50e2ece149d7 github.com/openshift/cluster-api-actuator-pkg/testutils v0.0.0 - github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20250702183526-4eb64d553940 + github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20251016022208-dc342af67c74 github.com/openshift/library-go v0.0.0-20251112091634-ab97ebb73f0f github.com/openshift/machine-api-operator v0.2.1-0.20251121134325-1d78f2ebcae5 github.com/tidwall/gjson v1.18.0 diff --git a/go.sum b/go.sum index af0db38c4..00a0a5372 100644 --- a/go.sum +++ b/go.sum @@ -414,8 +414,8 @@ github.com/openshift/api v0.0.0-20251111193948-50e2ece149d7 h1:MemawsK6SpxEaE5y0 github.com/openshift/api v0.0.0-20251111193948-50e2ece149d7/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 h1:9JBeIXmnHlpXTQPi7LPmu1jdxznBhAE7bb1K+3D8gxY= github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235/go.mod h1:L49W6pfrZkfOE5iC1PqEkuLkXG4W0BX4w8b+L2Bv7fM= -github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20250702183526-4eb64d553940 h1:NCCRJ7JfGLpRu4IQSV7Qw9VoAdIgF1BiwospisP06a8= -github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20250702183526-4eb64d553940/go.mod h1:zCklcJwbnaNx46KvR38Rh86uZdow5gvub4ATcNDopTM= +github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20251016022208-dc342af67c74 h1:UKHeXUui6RbfRMA4XKB7j29R9aXECMFV2hDtA/DiDh8= +github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20251016022208-dc342af67c74/go.mod h1:EphbrauvJujuyjYoh8SUBeP+V0TtOXKfi0TsW+YrLPo= github.com/openshift/cluster-control-plane-machine-set-operator v0.0.0-20251029084908-344babe6a957 h1:eVnkMTFnirnoUOlAUT3Hy8WriIi1JoSrilWym3Dl8Q4= github.com/openshift/cluster-control-plane-machine-set-operator v0.0.0-20251029084908-344babe6a957/go.mod h1:TBlORAAtNZ/Tl86pO7GjNXKsH/g0QAW5GnvYstdOhYI= github.com/openshift/library-go v0.0.0-20251112091634-ab97ebb73f0f h1:r1pLosA7z3+t+lzW29FU54sg4/pAWu+lsKD0L5Gx3wg= diff --git a/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/cluster.go b/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/cluster.go index e8f83d4d2..e5bd8d4b3 100644 --- a/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/cluster.go +++ b/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/cluster.go @@ -43,6 +43,7 @@ type ClusterBuilder struct { ownerReferences []metav1.OwnerReference // Spec fields. + availabilityGates []clusterv1beta1.ClusterAvailabilityGate clusterNetwork *clusterv1beta1.ClusterNetwork controlPlaneEndpoint clusterv1beta1.APIEndpoint controlPlaneRef *corev1.ObjectReference @@ -59,6 +60,7 @@ type ClusterBuilder struct { infrastructureReady bool observedGeneration int64 phase string + v1Beta2 *clusterv1beta1.ClusterV1Beta2Status } // Build builds a new cluster based on the configuration provided. @@ -75,6 +77,7 @@ func (c ClusterBuilder) Build() *clusterv1beta1.Cluster { OwnerReferences: c.ownerReferences, }, Spec: clusterv1beta1.ClusterSpec{ + AvailabilityGates: c.availabilityGates, ClusterNetwork: c.clusterNetwork, ControlPlaneEndpoint: c.controlPlaneEndpoint, ControlPlaneRef: c.controlPlaneRef, @@ -91,6 +94,7 @@ func (c ClusterBuilder) Build() *clusterv1beta1.Cluster { InfrastructureReady: c.infrastructureReady, ObservedGeneration: c.observedGeneration, Phase: c.phase, + V1Beta2: c.v1Beta2, }, } @@ -149,6 +153,12 @@ func (c ClusterBuilder) WithOwnerReferences(ownerRefs []metav1.OwnerReference) C // Spec fields. +// WithAvailabilityGates sets the availability gates for the cluster builder. +func (c ClusterBuilder) WithAvailabilityGates(gates []clusterv1beta1.ClusterAvailabilityGate) ClusterBuilder { + c.availabilityGates = gates + return c +} + // WithClusterNetwork sets the cluster network for the cluster builder. func (c ClusterBuilder) WithClusterNetwork(network *clusterv1beta1.ClusterNetwork) ClusterBuilder { c.clusterNetwork = network @@ -234,3 +244,9 @@ func (c ClusterBuilder) WithPhase(phase string) ClusterBuilder { c.phase = phase return c } + +// WithV1Beta2Status sets the v1beta2 status for the cluster builder. +func (c ClusterBuilder) WithV1Beta2Status(v1Beta2 *clusterv1beta1.ClusterV1Beta2Status) ClusterBuilder { + c.v1Beta2 = v1Beta2 + return c +} diff --git a/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machine.go b/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machine.go index e68c2f8ac..6aaeccd5c 100644 --- a/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machine.go +++ b/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machine.go @@ -51,6 +51,7 @@ type MachineBuilder struct { nodeDrainTimeout *metav1.Duration nodeVolumeDetachTimeout *metav1.Duration providerID *string + readinessGates []clusterv1beta1.MachineReadinessGate version *string // Status fields. @@ -58,6 +59,7 @@ type MachineBuilder struct { bootstrapReady bool certificatesExpiryDate *metav1.Time conditions clusterv1beta1.Conditions + deletion *clusterv1beta1.MachineDeletionStatus failureMessage *string failureReason *capierrors.MachineStatusError infrastructureReady bool @@ -66,6 +68,7 @@ type MachineBuilder struct { nodeRef *corev1.ObjectReference observedGeneration int64 phase clusterv1beta1.MachinePhase + v1Beta2 *clusterv1beta1.MachineV1Beta2Status } // Build builds a new Machine based on the configuration provided. @@ -90,6 +93,7 @@ func (m MachineBuilder) Build() *clusterv1beta1.Machine { NodeDrainTimeout: m.nodeDrainTimeout, NodeVolumeDetachTimeout: m.nodeVolumeDetachTimeout, ProviderID: m.providerID, + ReadinessGates: m.readinessGates, Version: m.version, }, Status: clusterv1beta1.MachineStatus{ @@ -97,6 +101,7 @@ func (m MachineBuilder) Build() *clusterv1beta1.Machine { BootstrapReady: m.bootstrapReady, CertificatesExpiryDate: m.certificatesExpiryDate, Conditions: m.conditions, + Deletion: m.deletion, FailureMessage: m.failureMessage, FailureReason: m.failureReason, InfrastructureReady: m.infrastructureReady, @@ -105,6 +110,7 @@ func (m MachineBuilder) Build() *clusterv1beta1.Machine { NodeRef: m.nodeRef, ObservedGeneration: m.observedGeneration, Phase: string(m.phase), + V1Beta2: m.v1Beta2, }, } @@ -217,6 +223,12 @@ func (m MachineBuilder) WithProviderID(providerID *string) MachineBuilder { return m } +// WithReadinessGates sets the ReadinessGates for the machine builder. +func (m MachineBuilder) WithReadinessGates(gates []clusterv1beta1.MachineReadinessGate) MachineBuilder { + m.readinessGates = gates + return m +} + // WithVersion sets the Version for the machine builder. func (m MachineBuilder) WithVersion(version *string) MachineBuilder { m.version = version @@ -290,3 +302,15 @@ func (m MachineBuilder) WithPhase(phase clusterv1beta1.MachinePhase) MachineBuil m.phase = phase return m } + +// WithDeletion sets the Deletion status for the machine builder. +func (m MachineBuilder) WithDeletion(deletion *clusterv1beta1.MachineDeletionStatus) MachineBuilder { + m.deletion = deletion + return m +} + +// WithV1Beta2Status sets the v1beta2 status for the machine builder. +func (m MachineBuilder) WithV1Beta2Status(v1Beta2 *clusterv1beta1.MachineV1Beta2Status) MachineBuilder { + m.v1Beta2 = v1Beta2 + return m +} diff --git a/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machineset.go b/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machineset.go index 4b8ff3c26..80acff3c4 100644 --- a/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machineset.go +++ b/vendor/github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1/machineset.go @@ -42,12 +42,13 @@ type MachineSetBuilder struct { ownerReferences []metav1.OwnerReference // Spec fields. - clusterName string - deletePolicy string - minReadySeconds int32 - replicas *int32 - selector metav1.LabelSelector - template clusterv1beta1.MachineTemplateSpec + clusterName string + deletePolicy string + machineNamingStrategy *clusterv1beta1.MachineNamingStrategy + minReadySeconds int32 + replicas *int32 + selector metav1.LabelSelector + template clusterv1beta1.MachineTemplateSpec // Status fields. availableReplicas int32 @@ -59,6 +60,7 @@ type MachineSetBuilder struct { readyReplicas int32 statusReplicas int32 statusSelector string + v1Beta2 *clusterv1beta1.MachineSetV1Beta2Status } // Build builds a new MachineSet based on the configuration provided. @@ -75,12 +77,13 @@ func (m MachineSetBuilder) Build() *clusterv1beta1.MachineSet { OwnerReferences: m.ownerReferences, }, Spec: clusterv1beta1.MachineSetSpec{ - ClusterName: m.clusterName, - DeletePolicy: m.deletePolicy, - MinReadySeconds: m.minReadySeconds, - Replicas: m.replicas, - Selector: m.selector, - Template: m.template, + ClusterName: m.clusterName, + DeletePolicy: m.deletePolicy, + MachineNamingStrategy: m.machineNamingStrategy, + MinReadySeconds: m.minReadySeconds, + Replicas: m.replicas, + Selector: m.selector, + Template: m.template, }, Status: clusterv1beta1.MachineSetStatus{ AvailableReplicas: m.availableReplicas, @@ -92,6 +95,7 @@ func (m MachineSetBuilder) Build() *clusterv1beta1.MachineSet { ReadyReplicas: m.readyReplicas, Replicas: m.statusReplicas, Selector: m.statusSelector, + V1Beta2: m.v1Beta2, }, } @@ -162,6 +166,12 @@ func (m MachineSetBuilder) WithDeletePolicy(deletePolicy string) MachineSetBuild return m } +// WithMachineNamingStrategy sets the machineNamingStrategy for the MachineSet builder. +func (m MachineSetBuilder) WithMachineNamingStrategy(strategy *clusterv1beta1.MachineNamingStrategy) MachineSetBuilder { + m.machineNamingStrategy = strategy + return m +} + // WithMinReadySeconds sets the minReadySeconds for the MachineSet builder. func (m MachineSetBuilder) WithMinReadySeconds(minReadySeconds int32) MachineSetBuilder { m.minReadySeconds = minReadySeconds @@ -241,3 +251,9 @@ func (m MachineSetBuilder) WithStatusSelector(selector string) MachineSetBuilder m.statusSelector = selector return m } + +// WithV1Beta2Status sets the v1beta2 status for the MachineSet builder. +func (m MachineSetBuilder) WithV1Beta2Status(v1Beta2 *clusterv1beta1.MachineSetV1Beta2Status) MachineSetBuilder { + m.v1Beta2 = v1Beta2 + return m +} diff --git a/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/clusterautoscaler_types.go b/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/clusterautoscaler_types.go index 9e968225b..b7cb860bb 100644 --- a/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/clusterautoscaler_types.go +++ b/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/clusterautoscaler_types.go @@ -19,6 +19,16 @@ const ( RandomExpander ExpanderString = "Random" ) +// CordonNodeBeforeTerminatingMode represents the mode for cordoning nodes before terminating. +// +kubebuilder:validation:Enum=Enabled;Disabled +type CordonNodeBeforeTerminatingMode string + +// These constants define the valid values for CordonNodeBeforeTerminatingMode +const ( + CordonNodeBeforeTerminatingModeEnabled CordonNodeBeforeTerminatingMode = "Enabled" + CordonNodeBeforeTerminatingModeDisabled CordonNodeBeforeTerminatingMode = "Disabled" +) + // ClusterAutoscalerSpec defines the desired state of ClusterAutoscaler type ClusterAutoscalerSpec struct { // Constraints of autoscaling resources @@ -27,6 +37,9 @@ type ClusterAutoscalerSpec struct { // Configuration of scale down operation ScaleDown *ScaleDownConfig `json:"scaleDown,omitempty"` + // Configuration of scale up operation + ScaleUp *ScaleUpConfig `json:"scaleUp,omitempty"` + // Gives pods graceful termination time before scaling down MaxPodGracePeriod *int32 `json:"maxPodGracePeriod,omitempty"` @@ -182,4 +195,13 @@ type ScaleDownConfig struct { // Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down // +kubebuilder:validation:Pattern=(0.[0-9]+) UtilizationThreshold *string `json:"utilizationThreshold,omitempty"` + + // CordonNodeBeforeTerminating enables/disables cordoning nodes before terminating during scale down. + CordonNodeBeforeTerminating *CordonNodeBeforeTerminatingMode `json:"cordonNodeBeforeTerminating,omitempty"` +} + +type ScaleUpConfig struct { + // Scale up delay for new pods, if omitted defaults to 0 seconds + // +kubebuilder:validation:Pattern=([0-9]*(\.[0-9]*)?[a-z]+)+ + NewPodScaleUpDelay *string `json:"newPodScaleUpDelay,omitempty"` } diff --git a/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/zz_generated.deepcopy.go b/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/zz_generated.deepcopy.go index 4d1e88c43..0b987badb 100644 --- a/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1/zz_generated.deepcopy.go @@ -97,6 +97,11 @@ func (in *ClusterAutoscalerSpec) DeepCopyInto(out *ClusterAutoscalerSpec) { *out = new(ScaleDownConfig) (*in).DeepCopyInto(*out) } + if in.ScaleUp != nil { + in, out := &in.ScaleUp, &out.ScaleUp + *out = new(ScaleUpConfig) + (*in).DeepCopyInto(*out) + } if in.MaxPodGracePeriod != nil { in, out := &in.MaxPodGracePeriod, &out.MaxPodGracePeriod *out = new(int32) @@ -257,6 +262,11 @@ func (in *ScaleDownConfig) DeepCopyInto(out *ScaleDownConfig) { *out = new(string) **out = **in } + if in.CordonNodeBeforeTerminating != nil { + in, out := &in.CordonNodeBeforeTerminating, &out.CordonNodeBeforeTerminating + *out = new(CordonNodeBeforeTerminatingMode) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleDownConfig. @@ -268,3 +278,23 @@ func (in *ScaleDownConfig) DeepCopy() *ScaleDownConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScaleUpConfig) DeepCopyInto(out *ScaleUpConfig) { + *out = *in + if in.NewPodScaleUpDelay != nil { + in, out := &in.NewPodScaleUpDelay, &out.NewPodScaleUpDelay + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleUpConfig. +func (in *ScaleUpConfig) DeepCopy() *ScaleUpConfig { + if in == nil { + return nil + } + out := new(ScaleUpConfig) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6bbf422ed..08caf2bc5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -858,8 +858,8 @@ github.com/openshift/client-go/machine/listers/machine/v1beta1 github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/apps/v1 github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta1 github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1 -# github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20250702183526-4eb64d553940 -## explicit; go 1.23.0 +# github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20251016022208-dc342af67c74 +## explicit; go 1.24.0 github.com/openshift/cluster-autoscaler-operator/pkg/apis github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1 github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1beta1 From 7e56b39c5a095c24b49259eb1212667299ce0ba2 Mon Sep 17 00:00:00 2001 From: John Kyros Date: Wed, 19 Nov 2025 14:07:36 -0600 Subject: [PATCH 2/2] Add test for Cordon Before Drain We added configuration for cordon before drain in the cluster autoscaler operator, this just adds a test to actuate that functionality all the way through (from CR configuration to actual cordoned downscaling). Signed-off-by: John Kyros --- pkg/autoscaler/autoscaler.go | 161 +++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/pkg/autoscaler/autoscaler.go b/pkg/autoscaler/autoscaler.go index 5f70cd838..f27f322d4 100644 --- a/pkg/autoscaler/autoscaler.go +++ b/pkg/autoscaler/autoscaler.go @@ -550,6 +550,167 @@ var _ = Describe("Autoscaler should", framework.LabelAutoscaler, framework.Label }, framework.WaitMedium, pollingInterval).Should(BeTrue(), "Node %s has a deletion taint and it should not", machine.Status.NodeRef.Name) } }) + + // Machines required for test: 2 + // Reason: Needs to verify cordon happens before drain during scale down. Scales 1 -> 2 -> 1. + It("cordons node before draining during scale-down [Slow]", func() { + By("Creating MachineSet with 1 replica") + targetedNodeLabel := fmt.Sprintf("%v-cordon-before-drain", autoscalerWorkerNodeRoleLabel) + machineSetParams := framework.BuildMachineSetParams(ctx, client, 1) + machineSetParams.Labels[targetedNodeLabel] = "" + machineSet, err := framework.CreateMachineSet(client, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create MachineSet with 1 replica") + cleanupObjects[machineSet.GetName()] = machineSet + + By("Waiting for the machineSet to enter Running phase") + framework.WaitForMachineSet(ctx, client, machineSet.GetName()) + + // Enable cordon-before-drain in ClusterAutoscaler + By("Updating ClusterAutoscaler to enable cordon-node-before-terminating") + currentCA, err := framework.GetClusterAutoscaler(client, clusterAutoscaler.GetName()) + Expect(err).ToNot(HaveOccurred(), "Failed to get ClusterAutoscaler") + cordonMode := caov1.CordonNodeBeforeTerminatingModeEnabled + if currentCA.Spec.ScaleDown == nil { + currentCA.Spec.ScaleDown = &caov1.ScaleDownConfig{ + Enabled: true, + } + } + currentCA.Spec.ScaleDown.CordonNodeBeforeTerminating = &cordonMode + Expect(client.Update(ctx, currentCA)).Should(Succeed(), "Failed to update ClusterAutoscaler with cordon-node-before-terminating") + + // Create the autoscaler resource + expectedReplicas := int32(2) + By(fmt.Sprintf("Creating a MachineAutoscaler backed by MachineSet %s - min: 1, max: %d", + machineSet.GetName(), expectedReplicas)) + asr := machineAutoscalerResource(machineSet, 1, expectedReplicas) + Expect(client.Create(ctx, asr)).Should(Succeed(), "Failed to create MachineAutoscaler with min 1/max %d replicas", expectedReplicas) + cleanupObjects[asr.GetName()] = asr + + // Create the workload + jobReplicas := expectedReplicas + uniqueJobName := fmt.Sprintf("%s-cordon-before-drain", workloadJobName) + By(fmt.Sprintf("Creating scale-out workload %s: jobs: %v, memory: %s", + uniqueJobName, jobReplicas, workloadMemRequest.String())) + workload := framework.NewWorkLoad(jobReplicas, workloadMemRequest, uniqueJobName, autoscalingTestLabel, "", corev1.NodeSelectorRequirement{ + Key: targetedNodeLabel, + Operator: corev1.NodeSelectorOpExists, + }) + cleanupObjects[workload.GetName()] = workload + Expect(client.Create(ctx, workload)).Should(Succeed(), "Failed to create scale-out workload %s", uniqueJobName) + + // Wait for the scaleout + By(fmt.Sprintf("Waiting for MachineSet %s replicas to scale out to %d", machineSet.GetName(), expectedReplicas)) + Eventually(func() (int32, error) { + current, err := framework.GetMachineSet(ctx, client, machineSet.GetName()) + if err != nil { + return 0, err + } + + return *current.Spec.Replicas, nil + }, framework.WaitMedium, pollingInterval).Should(BeEquivalentTo(expectedReplicas), "MachineSet %s failed to scale out to %d replicas", machineSet.GetName(), expectedReplicas) + + By("Waiting for all Machines in the MachineSet to enter Running phase") + framework.WaitForMachineSet(ctx, client, machineSet.GetName()) + + By(fmt.Sprintf("Waiting for %d workload pods to be running", jobReplicas)) + framework.WaitForWorkload(ctx, client, machineSet, jobReplicas, workload.GetName()) + + // Remove the workload so we scale down + By("Deleting the workload to trigger scale-down") + Expect(deleteObject(workload.Name, cleanupObjects[workload.Name])).Should(Succeed(), "Failed to delete workload object %s", workload.Name) + delete(cleanupObjects, workload.Name) + + // Make sure one of the nodes get cordoned before deletion, remember which one it is + By("Watching for node to be cordoned BEFORE drain begins") + var cordonedNodeName string + var targetMachine *machinev1.Machine + Eventually(func() (bool, error) { + machines, err := framework.GetMachinesFromMachineSet(ctx, client, machineSet) + if err != nil || len(machines) != 2 { + return false, err + } + + // Check for machines being targeted for deletion (has deletion candidate taint or annotation) + for _, machine := range machines { + node, err := framework.GetNodeForMachine(ctx, client, machine) + if err != nil { + continue + } + + // Check for deletion candidate taint + hasDeletionCandidateTaint := false + for _, taint := range node.Spec.Taints { + if taint.Key == deletionCandidateTaintKey { + hasDeletionCandidateTaint = true + break + } + } + + // If machine has deletion candidate taint, verify it's cordoned (cordon should show up before deletion candidate taint) + if hasDeletionCandidateTaint { + if !node.Spec.Unschedulable { + return false, fmt.Errorf("node %s has deletion candidate taint but is not cordoned", node.Name) + } + + // Node is cordoned! Save for later verification + cordonedNodeName = node.Name + targetMachine = machine + klog.Infof("Node %s successfully cordoned before drain (has deletion candidate taint)", cordonedNodeName) + + return true, nil + } + + // Also check for delete annotation on machine (cordon should show up before delete taint) + if machine.ObjectMeta.Annotations != nil { + if _, hasDeleteAnnotation := machine.ObjectMeta.Annotations[machineDeleteAnnotationKey]; hasDeleteAnnotation { + if !node.Spec.Unschedulable { + return false, fmt.Errorf("machine %s has delete annotation but node %s is not cordoned", machine.Name, node.Name) + } + + cordonedNodeName = node.Name + targetMachine = machine + klog.Infof("Node %s successfully cordoned before drain (has delete annotation)", cordonedNodeName) + + return true, nil + } + } + + // Premature cordon: node is cordoned but has no deletion signal yet — this should not happen + if node.Spec.Unschedulable { + return false, fmt.Errorf("node %s is cordoned but has no deletion signal (no DeletionCandidate taint or delete annotation) — unexpected premature cordon", node.Name) + } + } + + return false, nil + }, framework.WaitLong, pollingInterval).Should(BeTrue(), "Failed to observe node being cordoned before drain") + + By(fmt.Sprintf("Verifying node %s stays cordoned throughout drain until scale-in completes", cordonedNodeName)) + expectedMachineSetSize := 1 + Eventually(func() (int, error) { + // Try to get our node - if it still exists, verify it's still cordoned, if it doesn't + // exist, that's what we want anyway + node, err := framework.GetNodeForMachine(ctx, client, targetMachine) + if err == nil { + // Node still exists, verify it's cordoned + if !node.Spec.Unschedulable { + return 0, fmt.Errorf("node %s lost cordon status during drain", cordonedNodeName) + } + // Any errors aside from not found get returned (not found is okay, we just continue, we're deleting it) + } else if err != nil && !apierrors.IsNotFound(err) { + return 0, err + } + + // Check if scale-in is complete (the event we're waiting for) + machines, err := framework.GetMachinesFromMachineSet(ctx, client, machineSet) + if err != nil { + + return 0, err + } + + return len(machines), nil + }, framework.WaitLong, pollingInterval).Should(BeEquivalentTo(expectedMachineSetSize), "MachineSet %s failed to scale in to %d replica", machineSet.GetName(), expectedMachineSetSize) + + }) }) Context("use a ClusterAutoscaler that has balance similar nodes enabled and 100 maximum total nodes", func() {