stackitmachine_controller_test.go

/*
Copyright 2026.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0
*/

package controller

import (
	"context"
	"fmt"
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"

	infrav1 "github.com/stackitcloud/cluster-api-provider-stackit/api/v1alpha1"
	"github.com/stackitcloud/cluster-api-provider-stackit/cloud"
	cloudfake "github.com/stackitcloud/cluster-api-provider-stackit/cloud/fake"
	"github.com/stackitcloud/cluster-api-provider-stackit/util"
)

var _ = Describe("StackitMachine Controller", func() {
	const namespace = "default"

	var (
		ctx           context.Context
		clusterName   string
		machineName   string
		stackitName   string
		credentials   string
		fakeCloud     *cloudfake.Client
		reconciler    *StackitMachineReconciler
		request       reconcile.Request
		stackitKey    types.NamespacedName
		stackitMach   *infrav1.StackitMachine
		bootstrapName string
	)

	BeforeEach(func() {
		ctx = context.Background()
		suffix := time.Now().UnixNano()
		clusterName = fmt.Sprintf("cluster-%d", suffix)
		machineName = fmt.Sprintf("machine-%d", suffix)
		stackitName = fmt.Sprintf("stackit-machine-%d", suffix)
		credentials = "stackit-credentials-" + fmt.Sprint(suffix)
		bootstrapName = fmt.Sprintf("bootstrap-%d", suffix)
		stackitKey = types.NamespacedName{Namespace: namespace, Name: stackitName}
		request = reconcile.Request{NamespacedName: stackitKey}
		fakeCloud = cloudfake.New(cloud.Network{ID: testNetworkID, Name: "network"})
		reconciler = &StackitMachineReconciler{
			Client: k8sClient,
			Scheme: k8sClient.Scheme(),
			CloudClientFactory: func(context.Context, cloud.Credentials) (cloud.Client, error) {
				return fakeCloud, nil
			},
		}

		createCredentialsSecret(ctx, credentials, namespace, testProjectID)
		createOwnerCluster(ctx, clusterName)
		createReadyStackitCluster(ctx, clusterName, namespace, credentials)
		createOwnerMachine(ctx, machineName, clusterName, stackitName)
		stackitMach = newStackitMachine(stackitName, namespace, machineName)
		Expect(k8sClient.Create(ctx, stackitMach)).To(Succeed())
	})

	AfterEach(func() {
		deleteIfExists(ctx, stackitMach)
		deleteIfExists(ctx, &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: machineName, Namespace: namespace}})
		deleteIfExists(ctx, &infrav1.StackitCluster{ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: namespace}})
		deleteIfExists(ctx, &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: namespace}})
		deleteIfExists(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: credentials, Namespace: namespace}})
		deleteIfExists(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: bootstrapName, Namespace: namespace}})
	})

	It("does not create a VM when Machine.spec.bootstrap.dataSecretName is empty", func() {
		result, err := reconciler.Reconcile(ctx, request)
		Expect(err).NotTo(HaveOccurred())
		Expect(result.RequeueAfter).To(BeNumerically(">", 0))
		Expect(fakeCloud.ServerCount()).To(Equal(0))

		got := &infrav1.StackitMachine{}
		Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
		Expect(got.Status.Ready).To(BeFalse())
		expectCondition(got.Status.Conditions, infrav1.MachineBootstrapReadyCondition, metav1.ConditionFalse, "BootstrapDataSecretMissing")
	})

	It("does not create a VM when the bootstrap Secret is missing", func() {
		updateMachineBootstrapSecret(ctx, machineName, bootstrapName)

		result, err := reconciler.Reconcile(ctx, request)
		Expect(err).NotTo(HaveOccurred())
		Expect(result.RequeueAfter).To(BeNumerically(">", 0))
		Expect(fakeCloud.ServerCount()).To(Equal(0))

		got := &infrav1.StackitMachine{}
		Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
		expectCondition(got.Status.Conditions, infrav1.MachineBootstrapReadyCondition, metav1.ConditionFalse, "BootstrapDataSecretNotFound")
	})

	Context("with bootstrap data", func() {
		BeforeEach(func() {
			updateMachineBootstrapSecret(ctx, machineName, bootstrapName)
			createBootstrapSecret(ctx, bootstrapName)
		})

		It("creates a VM and sets provider status when bootstrap data is available", func() {

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeTrue())
			Expect(got.Status.InstanceID).NotTo(BeEmpty())
			Expect(got.Status.InstanceState).To(Equal("ACTIVE"))
			Expect(got.Status.AvailabilityZone).To(Equal("eu01-1"))
			Expect(got.Status.ProviderID).To(Equal("stackit://" + got.Status.InstanceID))
			Expect(got.Spec.ProviderID).NotTo(BeNil())
			Expect(*got.Spec.ProviderID).To(Equal(got.Status.ProviderID))
			Expect(got.Status.Addresses).To(HaveLen(1))
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionTrue, "Available")
			expectCondition(got.Status.Conditions, infrav1.MachineBootstrapReadyCondition, metav1.ConditionTrue, "Available")
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionTrue, "Available")
		})

		It("does not silently recreate the server of an already-provisioned machine", func() {
			// Recreating it would replay bootstrap data pinned to the previous
			// identity: the replacement either never rejoins or rejoins while Machine
			// and Node still point at the deleted server.
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))
			Expect(fakeCloud.CreateServerCalls).To(Equal(1))

			provisioned := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, provisioned)).To(Succeed())
			Expect(provisioned.Status.Initialization.Provisioned).To(BeTrue())
			instanceID := provisioned.Status.InstanceID
			Expect(instanceID).NotTo(BeEmpty())

			By("removing the server behind the provider's back")
			Expect(fakeCloud.DeleteServer(ctx, instanceID)).To(Succeed())
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			By("reconciling again")
			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))

			Expect(fakeCloud.CreateServerCalls).To(Equal(1),
				"a replacement server was created for an already-provisioned machine")
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			By("reporting a consistent readiness state")
			degraded := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, degraded)).To(Succeed())
			expectCondition(degraded.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InstanceNotFound")
			expectCondition(degraded.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceNotFound")
			Expect(degraded.Status.Ready).To(BeFalse(),
				"legacy status.ready must follow the Ready condition, not contradict it")
		})

		It("reports a stopped server with its own reason", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())

			By("passing through the transitional STOPPING state without failing")
			fakeCloud.SetServerState(got.Status.InstanceID, "STOPPING", "RUNNING")
			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())

			By("marking not ready once the server is stopped")
			fakeCloud.SetServerState(got.Status.InstanceID, "INACTIVE", "STOPPED")
			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result.RequeueAfter).To(Equal(15 * time.Second))

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeFalse())
			Expect(got.Status.InstanceState).To(Equal("INACTIVE"))
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceStopped")
			condition := meta.FindStatusCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition)
			Expect(condition.Message).To(Equal("server is stopped (state INACTIVE, power status STOPPED)"))

			By("reconciling again while the server stays stopped")
			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
		})

		It("reports an active but crashed server as not ready", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			fakeCloud.SetServerState(got.Status.InstanceID, "ACTIVE", "CRASHED")

			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeFalse())
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceCrashed")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InstanceCrashed")
		})

		It("attaches provider-managed node SSH access when bastion is enabled", func() {
			stackitCluster := &infrav1.StackitCluster{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: namespace}, stackitCluster)).To(Succeed())
			stackitCluster.Spec.Bastion = validBastionSpec()
			Expect(k8sClient.Update(ctx, stackitCluster)).To(Succeed())
			stackitCluster.Status.Ready = true
			stackitCluster.Status.Bastion.SecurityGroupID = "bastion-sg"
			Expect(k8sClient.Status().Update(ctx, stackitCluster)).To(Succeed())

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeTrue())
			Expect(fakeCloud.EnsureNodeSSHCalls).To(Equal(1))
			securityGroups, err := fakeCloud.ListSecurityGroupsByTags(ctx, map[string]string{
				util.LabelClusterName:  clusterName,
				util.LabelResourceRole: util.ResourceRoleNodeSSH,
			})
			Expect(err).NotTo(HaveOccurred())
			Expect(securityGroups).To(HaveLen(1))
			Expect(fakeCloud.SecurityGroupRemoteSources(securityGroups[0].ID)).To(ConsistOf("bastion-sg"))
			Expect(fakeCloud.ServerHasSecurityGroup(got.Status.InstanceID, securityGroups[0].ID)).To(BeTrue())
		})

		It("marks conditions not ready when bastion node SSH access fails", func() {
			stackitCluster := &infrav1.StackitCluster{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: namespace}, stackitCluster)).To(Succeed())
			stackitCluster.Spec.Bastion = validBastionSpec()
			Expect(k8sClient.Update(ctx, stackitCluster)).To(Succeed())
			stackitCluster.Status.Ready = true
			stackitCluster.Status.Bastion.SecurityGroupID = "bastion-sg"
			Expect(k8sClient.Status().Update(ctx, stackitCluster)).To(Succeed())

			fakeCloud.FailNextEnsureNodeSSH = fmt.Errorf("node ssh access timeout: %w", cloud.ErrTransient)

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result.RequeueAfter).To(BeNumerically(">", 0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "BastionSSHAccessError")
		})

		It("does not create a VM when availabilityZone is outside published failure domains", func() {
			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			got.Spec.AvailabilityZone = "eu01-9"
			Expect(k8sClient.Update(ctx, got)).To(Succeed())

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeFalse())
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InvalidFailureDomain")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InvalidFailureDomain")
		})

		//nolint:dupl // Failure domain resolution cases test explicit failureDomain vs cluster default.
		It("respects Machine.spec.failureDomain when StackitMachine.spec.availabilityZone is empty", func() {
			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			got.Spec.AvailabilityZone = ""
			Expect(k8sClient.Update(ctx, got)).To(Succeed())

			ownerMachine := &clusterv1.Machine{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: machineName, Namespace: namespace}, ownerMachine)).To(Succeed())
			ownerMachine.Spec.FailureDomain = "eu01-2"
			Expect(k8sClient.Update(ctx, ownerMachine)).To(Succeed())

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(fakeCloud.ServerCount()).To(Equal(1))
			Expect(fakeCloud.LastCreateServerInput.AvailabilityZone).To(Equal("eu01-2"))

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeTrue())
			Expect(got.Status.AvailabilityZone).To(Equal("eu01-2"))
		})

		//nolint:dupl // Failure domain resolution cases test explicit failureDomain vs cluster default.
		It("defaults to the first failure domain when neither failureDomain nor availabilityZone is set", func() {
			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			got.Spec.AvailabilityZone = ""
			Expect(k8sClient.Update(ctx, got)).To(Succeed())

			ownerMachine := &clusterv1.Machine{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: machineName, Namespace: namespace}, ownerMachine)).To(Succeed())
			ownerMachine.Spec.FailureDomain = ""
			Expect(k8sClient.Update(ctx, ownerMachine)).To(Succeed())

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(fakeCloud.ServerCount()).To(Equal(1))
			Expect(fakeCloud.LastCreateServerInput.AvailabilityZone).To(Equal("eu01-1"))

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeTrue())
			Expect(got.Status.AvailabilityZone).To(Equal("eu01-1"))
		})

		It("does not create a VM when Machine.spec.failureDomain is outside published failure domains", func() {
			ownerMachine := &clusterv1.Machine{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: machineName, Namespace: namespace}, ownerMachine)).To(Succeed())
			ownerMachine.Spec.FailureDomain = "eu01-9"
			Expect(k8sClient.Update(ctx, ownerMachine)).To(Succeed())

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(got.Status.Ready).To(BeFalse())
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InvalidFailureDomain")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InvalidFailureDomain")
		})

		It("requeues on unauthorized credentials and recovers once they are corrected", func() {
			reconciler.CloudClientFactory = func(context.Context, cloud.Credentials) (cloud.Client, error) {
				return nil, fmt.Errorf("authenticate: %w", cloud.ErrUnauthorized)
			}

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result.RequeueAfter).To(Equal(credentialsRetryRequeueAfter))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, infrav1.MachineCredentialsReadyCondition, metav1.ConditionFalse, "CredentialsInvalid")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "CredentialsInvalid")

			By("correcting the credentials")
			reconciler.CloudClientFactory = func(context.Context, cloud.Credentials) (cloud.Client, error) {
				return fakeCloud, nil
			}

			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, infrav1.MachineCredentialsReadyCondition, metav1.ConditionTrue, "Available")
		})

		//nolint:dupl // Pause behavior must be verified independently for the machine reconciler.
		It("does not call the cloud API when the owning Cluster is paused", func() {
			cluster := &clusterv1.Cluster{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: namespace}, cluster)).To(Succeed())
			cluster.Spec.Paused = new(true)
			Expect(k8sClient.Update(ctx, cluster)).To(Succeed())

			cloudClientFactoryCalls := 0
			reconciler.CloudClientFactory = func(context.Context, cloud.Credentials) (cloud.Client, error) {
				cloudClientFactoryCalls++
				return fakeCloud, nil
			}

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(cloudClientFactoryCalls).To(Equal(0))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, clusterv1.PausedCondition, metav1.ConditionTrue, clusterv1.PausedReason)
		})

		It("does not call the cloud API when the StackitMachine has the paused annotation", func() {
			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			got.Annotations = map[string]string{clusterv1.PausedAnnotation: ""}
			Expect(k8sClient.Update(ctx, got)).To(Succeed())

			cloudClientFactoryCalls := 0
			reconciler.CloudClientFactory = func(context.Context, cloud.Credentials) (cloud.Client, error) {
				cloudClientFactoryCalls++
				return fakeCloud, nil
			}

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(cloudClientFactoryCalls).To(Equal(0))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, clusterv1.PausedCondition, metav1.ConditionTrue, clusterv1.PausedReason)
		})

		It("requeues when server lookup returns a conflict", func() {
			fakeCloud.FailNextFindServer = fmt.Errorf("multiple matching servers: %w", cloud.ErrConflict)

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result.RequeueAfter).To(BeNumerically(">", 0))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceError")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InstanceError")
		})

		It("requeues when VM creation returns a transient error", func() {
			fakeCloud.FailNextCreateServer = fmt.Errorf("create server timeout: %w", cloud.ErrTransient)

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(result.RequeueAfter).To(BeNumerically(">", 0))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceError")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InstanceError")
		})

		It("returns an error when VM creation is rejected", func() {
			fakeCloud.FailNextCreateServer = fmt.Errorf("create server rejected: %w", cloud.ErrInvalidInput)

			result, err := reconciler.Reconcile(ctx, request)
			Expect(err).To(HaveOccurred())
			Expect(result).To(Equal(reconcile.Result{}))
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceError")
			expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InstanceError")
		})

		It("deletes the VM and removes the finalizer", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(k8sClient.Delete(ctx, got)).To(Succeed())

			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(0))
			Eventually(func() bool {
				err := k8sClient.Get(ctx, stackitKey, &infrav1.StackitMachine{})
				return apierrors.IsNotFound(err)
			}).Should(BeTrue())
		})

		It("returns an error when server deletion fails", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(k8sClient.Delete(ctx, got)).To(Succeed())

			fakeCloud.FailNextDeleteServer = fmt.Errorf("delete server timeout: %w", cloud.ErrTransient)

			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).To(HaveOccurred())
		})

		// The hook observes API server state from inside the cloud call, so it proves
		// the ordering rather than only the end result.
		It("persists the finalizer before creating the server", func() {
			var finalizersAtCreate []string
			fakeCloud.BeforeCreateServer = func() {
				got := &infrav1.StackitMachine{}
				Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
				finalizersAtCreate = got.Finalizers
			}

			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			Expect(finalizersAtCreate).To(ContainElement(infrav1.MachineFinalizer),
				"the server was created while the API server had no finalizer to clean it up")
		})

		It("removes the finalizer when the server is already gone at deletion time", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			provisioned := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, provisioned)).To(Succeed())
			instanceID := provisioned.Status.InstanceID
			Expect(instanceID).NotTo(BeEmpty())

			By("removing the server behind the provider's back")
			Expect(fakeCloud.DeleteServer(ctx, instanceID)).To(Succeed())
			Expect(fakeCloud.ServerCount()).To(Equal(0))

			By("reconciling into the terminal state")
			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())

			degraded := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, degraded)).To(Succeed())
			expectCondition(degraded.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceNotFound")
			Expect(degraded.Status.InstanceID).To(Equal(instanceID))

			By("deleting the object while the cloud reports the server as gone")
			// The fake deletes an unknown ID without an error, so the not-found answer
			// of the cloud must be injected.
			fakeCloud.FailNextDeleteServer = fmt.Errorf("delete server: %w", cloud.ErrNotFound)

			Expect(k8sClient.Delete(ctx, degraded)).To(Succeed())

			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			// The reconcile clears FailNextDeleteServer when it calls DeleteServer,
			// so a nil field proves the call happened.
			Expect(fakeCloud.FailNextDeleteServer).ToNot(HaveOccurred(),
				"the machine deletion did not ask the cloud to remove the server")
			Eventually(func() bool {
				err := k8sClient.Get(ctx, stackitKey, &infrav1.StackitMachine{})
				return apierrors.IsNotFound(err)
			}).Should(BeTrue())
		})

		// A lost status patch leaves a running, tagged server that no field on the
		// object names, so an empty status.instanceID is not proof that none exists.
		It("deletes a tagged server whose instance ID was lost from the status", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			By("losing the persisted instance ID, as if the status patch never landed")
			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			got.Status.InstanceID = ""
			got.Status.ProviderID = ""
			Expect(k8sClient.Status().Update(ctx, got)).To(Succeed())

			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			got.Spec.ProviderID = nil
			Expect(k8sClient.Update(ctx, got)).To(Succeed())

			By("deleting the StackitMachine")
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			Expect(k8sClient.Delete(ctx, got)).To(Succeed())

			_, err = reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(0),
				"the tagged server leaked because deletion trusted the empty status")
			Eventually(func() bool {
				err := k8sClient.Get(ctx, stackitKey, &infrav1.StackitMachine{})
				return apierrors.IsNotFound(err)
			}).Should(BeTrue())
		})

		It("keeps the finalizer when the server deletion fails", func() {
			_, err := reconciler.Reconcile(ctx, request)
			Expect(err).NotTo(HaveOccurred())
			Expect(fakeCloud.ServerCount()).To(Equal(1))

			got := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
			fakeCloud.FailNextDeleteServer = fmt.Errorf("delete server: %w", cloud.ErrTransient)

			Expect(k8sClient.Delete(ctx, got)).To(Succeed())
			_, err = reconciler.Reconcile(ctx, request)

			Expect(err).To(MatchError(cloud.ErrTransient))
			Expect(fakeCloud.ServerCount()).To(Equal(1))
			stillThere := &infrav1.StackitMachine{}
			Expect(k8sClient.Get(ctx, stackitKey, stillThere)).To(Succeed())
			Expect(stillThere.Finalizers).To(ContainElement(infrav1.MachineFinalizer))
		})
	})

	It("maps owning Machine events to StackitMachine reconcile requests", func() {
		machine := &clusterv1.Machine{}
		Expect(k8sClient.Get(ctx, types.NamespacedName{Name: machineName, Namespace: namespace}, machine)).To(Succeed())

		requests := reconciler.stackitMachineRequestsForMachine(ctx, machine)
		Expect(requests).To(Equal([]reconcile.Request{request}))
	})

	It("maps related StackitCluster events to StackitMachine reconcile requests", func() {
		stackitCluster := &infrav1.StackitCluster{}
		Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: namespace}, stackitCluster)).To(Succeed())

		requests := reconciler.stackitMachineRequestsForStackitCluster(ctx, stackitCluster)
		Expect(requests).To(ConsistOf(request))
	})

	// Machine.spec.clusterName names the Cluster, not the StackitCluster, and a
	// ClusterClass-generated infrastructureRef never shares its Cluster's name.
	// Every other spec here uses matching names and would miss that.
	It("maps StackitCluster events when the StackitCluster name differs from the Cluster name", func() {
		suffix := time.Now().UnixNano()
		ownerClusterName := fmt.Sprintf("owner-%d", suffix)
		infraClusterName := ownerClusterName + "-infra"
		otherMachineName := fmt.Sprintf("other-machine-%d", suffix)
		otherStackitName := fmt.Sprintf("other-stackit-machine-%d", suffix)

		By("creating a Cluster whose infrastructureRef points at a differently named StackitCluster")
		ownerCluster := &clusterv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{Name: ownerClusterName, Namespace: namespace},
			Spec: clusterv1.ClusterSpec{
				InfrastructureRef: clusterv1.ContractVersionedObjectReference{
					APIGroup: infrav1.GroupVersion.Group,
					Kind:     "StackitCluster",
					Name:     infraClusterName,
				},
			},
		}
		Expect(k8sClient.Create(ctx, ownerCluster)).To(Succeed())

		infraCluster := newStackitCluster(infraClusterName, namespace, false)
		infraCluster.OwnerReferences[0].Name = ownerClusterName
		Expect(k8sClient.Create(ctx, infraCluster)).To(Succeed())

		createOwnerMachine(ctx, otherMachineName, ownerClusterName, otherStackitName)

		DeferCleanup(func() {
			deleteIfExists(ctx, &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: otherMachineName, Namespace: namespace}})
			deleteIfExists(ctx, infraCluster)
			deleteIfExists(ctx, ownerCluster)
		})

		By("mapping the StackitCluster event onto the Machine owned by its Cluster")
		requests := reconciler.stackitMachineRequestsForStackitCluster(ctx, infraCluster)
		Expect(requests).To(ConsistOf(reconcile.Request{
			NamespacedName: types.NamespacedName{Namespace: namespace, Name: otherStackitName},
		}), "the StackitCluster watch must resolve its owning Cluster instead of matching on its own name")
	})

	It("maps bootstrap Secret events to StackitMachine reconcile requests", func() {
		updateMachineBootstrapSecret(ctx, machineName, bootstrapName)
		secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: bootstrapName, Namespace: namespace}}

		requests := reconciler.stackitMachineRequestsForBootstrapSecret(ctx, secret)
		Expect(requests).To(ConsistOf(request))
	})

	It("ignores Machine events for other infrastructure providers", func() {
		machine := &clusterv1.Machine{
			ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: namespace},
			Spec: clusterv1.MachineSpec{
				InfrastructureRef: clusterv1.ContractVersionedObjectReference{
					APIGroup: "other.infrastructure.example.com",
					Kind:     "OtherMachine",
					Name:     "other",
				},
			},
		}

		requests := reconciler.stackitMachineRequestsForMachine(ctx, machine)
		Expect(requests).To(BeEmpty())
	})
})

func newStackitMachine(name, namespace, machineName string) *infrav1.StackitMachine {
	return &infrav1.StackitMachine{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: namespace,
			OwnerReferences: []metav1.OwnerReference{{
				APIVersion: clusterv1.GroupVersion.String(),
				Kind:       "Machine",
				Name:       machineName,
				UID:        types.UID("machine-" + machineName),
			}},
		},
		Spec: infrav1.StackitMachineSpec{
			ImageID:          testImageID,
			MachineType:      "c2i.2",
			AvailabilityZone: "eu01-1",
			Network: infrav1.StackitMachineNetworkSpec{
				ID: testNetworkID,
			},
		},
	}
}