Skip to content

Commit 9546626

Browse files
HerbaertHerbaert
committed
feat: report server state and power status with dedicated reasons
1 parent 9ccc34a commit 9546626

10 files changed

Lines changed: 284 additions & 37 deletions

‎cloud/fake/client.go‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -250,11 +250,12 @@ func (c *Client) EnsureBastion(_ context.Context, input cloud.BastionInput) (*cl
250250
publicIP := c.publicIPByTags(input.Tags)
251251
securityGroup := c.securityGroupByTags(input.Tags)
252252
return &cloud.Bastion{
253-
ServerID: serverEntry.server.ID,
254-
ServerState: serverEntry.server.State,
255-
PublicIPID: idOfPublicIP(publicIP),
256-
PublicIP: ipOfPublicIP(publicIP),
257-
SecurityGroupID: idOfSecurityGroup(securityGroup),
253+
ServerID: serverEntry.server.ID,
254+
ServerState: serverEntry.server.State,
255+
ServerPowerStatus: serverEntry.server.PowerStatus,
256+
PublicIPID: idOfPublicIP(publicIP),
257+
PublicIP: ipOfPublicIP(publicIP),
258+
SecurityGroupID: idOfSecurityGroup(securityGroup),
258259
}, nil
259260
}
260261
}
@@ -569,6 +570,16 @@ func (c *Client) ServerHasSecurityGroup(serverID, securityGroupID string) bool {
569570
return ok
570571
}
571572

573+
// SetServerState sets the state and power status of a fake server (test helper).
574+
func (c *Client) SetServerState(serverID, state, powerStatus string) {
575+
c.mu.Lock()
576+
defer c.mu.Unlock()
577+
if entry, ok := c.servers[serverID]; ok {
578+
entry.server.State = state
579+
entry.server.PowerStatus = powerStatus
580+
}
581+
}
582+
572583
// ServerUserData returns the user-data stored for a fake server.
573584
func (c *Client) ServerUserData(serverID string) []byte {
574585
c.mu.Lock()

‎cloud/sdk_client.go‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -290,11 +290,12 @@ func (c *SDKClient) EnsureBastion(ctx context.Context, input BastionInput) (*Bas
290290
}
291291

292292
return &Bastion{
293-
ServerID: server.ID,
294-
ServerState: server.State,
295-
PublicIPID: publicIP.ID,
296-
PublicIP: publicIP.IP,
297-
SecurityGroupID: securityGroup.ID,
293+
ServerID: server.ID,
294+
ServerState: server.State,
295+
ServerPowerStatus: server.PowerStatus,
296+
PublicIPID: publicIP.ID,
297+
PublicIP: publicIP.IP,
298+
SecurityGroupID: securityGroup.ID,
298299
}, nil
299300
}
300301

@@ -962,9 +963,10 @@ func (c *SDKClient) serverFromSDK(ctx context.Context, server *iaas.Server) *Ser
962963
return nil
963964
}
964965
out := &Server{
965-
ID: server.GetId(),
966-
Name: server.GetName(),
967-
State: server.GetStatus(),
966+
ID: server.GetId(),
967+
Name: server.GetName(),
968+
State: server.GetStatus(),
969+
PowerStatus: server.GetPowerStatus(),
968970
}
969971
nics, err := c.iaasClient.DefaultAPI.ListServerNICs(ctx, c.projectID, c.region, out.ID).Execute()
970972
if err == nil {

‎cloud/types.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ package cloud
1515

1616
// Server describes a STACKIT compute instance in provider-neutral terms.
1717
type Server struct {
18-
ID string
19-
Name string
20-
State string
21-
ProviderID string
22-
Addresses []Address
18+
ID string
19+
Name string
20+
State string
21+
PowerStatus string
22+
ProviderID string
23+
Addresses []Address
2324
}
2425

2526
// Address is an IP or DNS endpoint of a Server.
@@ -59,11 +60,12 @@ type SecurityGroup struct {
5960

6061
// Bastion describes the provider-managed SSH bastion resources.
6162
type Bastion struct {
62-
ServerID string
63-
ServerState string
64-
PublicIPID string
65-
PublicIP string
66-
SecurityGroupID string
63+
ServerID string
64+
ServerState string
65+
ServerPowerStatus string
66+
PublicIPID string
67+
PublicIP string
68+
SecurityGroupID string
6769
}
6870

6971
// CreateServerInput holds all parameters required to create a new VM.

‎controller/controller_test_helpers_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
corev1 "k8s.io/api/core/v1"
1818
apierrors "k8s.io/apimachinery/pkg/api/errors"
1919
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
20+
"k8s.io/client-go/tools/events"
2021
"sigs.k8s.io/controller-runtime/pkg/client"
2122
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2223

@@ -216,3 +217,16 @@ func deleteIfExists(ctx context.Context, obj client.Object) {
216217
Expect(apierrors.IsNotFound(err)).To(BeTrue())
217218
}
218219
}
220+
221+
// drainEvents returns all events recorded so far.
222+
func drainEvents(recorder *events.FakeRecorder) []string {
223+
var out []string
224+
for {
225+
select {
226+
case event := <-recorder.Events:
227+
out = append(out, event)
228+
default:
229+
return out
230+
}
231+
}
232+
}

‎controller/server_state.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
Copyright 2026.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
*/
10+
11+
package controller
12+
13+
import "fmt"
14+
15+
type serverStateInfo struct {
16+
reason string
17+
description string
18+
// warn marks states that do not resolve on their own. Their reason must
19+
// differ from the transitional state leading there, because warnings are
20+
// deduplicated by reason.
21+
warn bool
22+
}
23+
24+
// serverStates maps the documented STACKIT server states other than ACTIVE.
25+
var serverStates = map[string]serverStateInfo{
26+
"CREATING": {"InstanceStarting", "server is starting", false},
27+
"STARTING": {"InstanceStarting", "server is starting", false},
28+
"REBOOT": {"InstanceBusy", "server is temporarily unavailable", false},
29+
"REBOOTING": {"InstanceBusy", "server is temporarily unavailable", false},
30+
"REBUILD": {"InstanceBusy", "server is temporarily unavailable", false},
31+
"REBUILDING": {"InstanceBusy", "server is temporarily unavailable", false},
32+
"RESIZING": {"InstanceBusy", "server is temporarily unavailable", false},
33+
"MIGRATING": {"InstanceBusy", "server is temporarily unavailable", false},
34+
"UPDATING": {"InstanceBusy", "server is temporarily unavailable", false},
35+
"RESTORING": {"InstanceBusy", "server is temporarily unavailable", false},
36+
"SNAPSHOTTING": {"InstanceBusy", "server is temporarily unavailable", false},
37+
"BACKING-UP": {"InstanceBusy", "server is temporarily unavailable", false},
38+
"STOPPING": {"InstanceStopping", "server is stopping", false},
39+
"INACTIVE": {"InstanceStopped", "server is stopped", true},
40+
"DEALLOCATING": {"InstanceDeallocating", "server is being deallocated", false},
41+
"DEALLOCATED": {"InstanceDeallocated", "server is deallocated", true},
42+
"PAUSED": {"InstancePaused", "server is paused", true},
43+
"RESCUING": {"InstanceEnteringRescue", "server is entering rescue mode", false},
44+
"RESCUE": {"InstanceInRescue", "server is in rescue mode", true},
45+
"UNRESCUING": {"InstanceLeavingRescue", "server is leaving rescue mode", false},
46+
"DELETING": {"InstanceDeleting", "server is being deleted", false},
47+
"DELETED": {"InstanceDeleting", "server is deleted", false},
48+
"ERROR": {"InstanceFailed", "server failed", true},
49+
}
50+
51+
// serverStateCondition maps a server state and power status to a condition
52+
// reason and message. ready is true when the server can be used.
53+
func serverStateCondition(state, powerStatus string) (ready bool, reason, message string, warn bool) {
54+
details := "state " + state
55+
if powerStatus != "" {
56+
details += ", power status " + powerStatus
57+
}
58+
59+
if state == "" || state == "ACTIVE" {
60+
switch powerStatus {
61+
case "CRASHED":
62+
return false, "InstanceCrashed", fmt.Sprintf("server crashed (%s)", details), true
63+
case "ERROR":
64+
return false, "InstancePowerError", fmt.Sprintf("server power error (%s)", details), true
65+
}
66+
return true, "", "", false
67+
}
68+
69+
info, ok := serverStates[state]
70+
if !ok {
71+
info = serverStateInfo{"InstanceNotActive", "server is not active", false}
72+
}
73+
return false, info.reason, fmt.Sprintf("%s (%s)", info.description, details), info.warn
74+
}

‎controller/server_state_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
Copyright 2026.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
*/
10+
11+
package controller
12+
13+
import (
14+
. "github.com/onsi/ginkgo/v2"
15+
. "github.com/onsi/gomega"
16+
)
17+
18+
var _ = Describe("serverStateCondition", func() {
19+
DescribeTable("maps server state and power status",
20+
func(state, powerStatus string, wantReady bool, wantReason, wantMessage string, wantWarn bool) {
21+
ready, reason, message, warn := serverStateCondition(state, powerStatus)
22+
Expect(ready).To(Equal(wantReady))
23+
Expect(reason).To(Equal(wantReason))
24+
Expect(message).To(Equal(wantMessage))
25+
Expect(warn).To(Equal(wantWarn))
26+
},
27+
Entry("active and running", "ACTIVE", "RUNNING", true, "", "", false),
28+
Entry("unknown state", "", "", true, "", "", false),
29+
Entry("active but crashed", "ACTIVE", "CRASHED", false, "InstanceCrashed",
30+
"server crashed (state ACTIVE, power status CRASHED)", true),
31+
Entry("active with power error", "ACTIVE", "ERROR", false, "InstancePowerError",
32+
"server power error (state ACTIVE, power status ERROR)", true),
33+
Entry("creating", "CREATING", "", false, "InstanceStarting", "server is starting (state CREATING)", false),
34+
Entry("rebooting", "REBOOTING", "RUNNING", false, "InstanceBusy",
35+
"server is temporarily unavailable (state REBOOTING, power status RUNNING)", false),
36+
Entry("stopping", "STOPPING", "RUNNING", false, "InstanceStopping",
37+
"server is stopping (state STOPPING, power status RUNNING)", false),
38+
Entry("stopped", "INACTIVE", "STOPPED", false, "InstanceStopped",
39+
"server is stopped (state INACTIVE, power status STOPPED)", true),
40+
Entry("deallocating", "DEALLOCATING", "STOPPED", false, "InstanceDeallocating",
41+
"server is being deallocated (state DEALLOCATING, power status STOPPED)", false),
42+
Entry("deallocated", "DEALLOCATED", "STOPPED", false, "InstanceDeallocated",
43+
"server is deallocated (state DEALLOCATED, power status STOPPED)", true),
44+
Entry("paused", "PAUSED", "", false, "InstancePaused", "server is paused (state PAUSED)", true),
45+
Entry("entering rescue", "RESCUING", "", false, "InstanceEnteringRescue",
46+
"server is entering rescue mode (state RESCUING)", false),
47+
Entry("leaving rescue", "UNRESCUING", "", false, "InstanceLeavingRescue",
48+
"server is leaving rescue mode (state UNRESCUING)", false),
49+
Entry("rescue", "RESCUE", "RUNNING", false, "InstanceInRescue",
50+
"server is in rescue mode (state RESCUE, power status RUNNING)", true),
51+
Entry("deleting", "DELETING", "", false, "InstanceDeleting", "server is being deleted (state DELETING)", false),
52+
Entry("error", "ERROR", "ERROR", false, "InstanceFailed", "server failed (state ERROR, power status ERROR)", true),
53+
Entry("undocumented state", "SOMETHING", "", false, "InstanceNotActive",
54+
"server is not active (state SOMETHING)", false),
55+
)
56+
})

‎controller/stackitcluster_bastion.go‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -130,13 +130,16 @@ func (r *StackitClusterReconciler) reconcileBastion(
130130
stackitCluster, nil, corev1.EventTypeNormal, "BastionCreated", "Create", "Created bastion %s", bastion.ServerID,
131131
)
132132
}
133-
if bastion.ServerState != "" && bastion.ServerState != "ACTIVE" {
134-
clusterScope.SetNotReady(
135-
"Provisioning",
136-
fmt.Sprintf("bastion server state is %s", bastion.ServerState),
137-
infrav1.ClusterBastionReadyCondition,
138-
infrav1.ClusterReadyCondition,
139-
)
133+
if ready, reason, message, warn := serverStateCondition(bastion.ServerState, bastion.ServerPowerStatus); !ready {
134+
previousReason := ""
135+
if previous := meta.FindStatusCondition(stackitCluster.Status.Conditions, infrav1.ClusterBastionReadyCondition); previous != nil {
136+
previousReason = previous.Reason
137+
}
138+
clusterScope.SetNotReady(reason, "bastion "+message, infrav1.ClusterBastionReadyCondition, infrav1.ClusterReadyCondition)
139+
// Only warn on entering the state, not on every requeue.
140+
if warn && r.Recorder != nil && previousReason != reason {
141+
r.Recorder.Eventf(stackitCluster, nil, corev1.EventTypeWarning, reason, "Reconcile", "Bastion server %s: %s", bastion.ServerID, message)
142+
}
140143
return ctrl.Result{RequeueAfter: 15 * time.Second}, false, nil
141144
}
142145
if bastion.PublicIP == "" {

‎controller/stackitcluster_controller_test.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ import (
1919
. "github.com/onsi/gomega"
2020
corev1 "k8s.io/api/core/v1"
2121
apierrors "k8s.io/apimachinery/pkg/api/errors"
22+
"k8s.io/apimachinery/pkg/api/meta"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2324
"k8s.io/apimachinery/pkg/types"
25+
"k8s.io/client-go/tools/events"
2426
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
2527
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2628

@@ -138,6 +140,30 @@ var _ = Describe("StackitCluster Controller", func() {
138140
expectCondition(got.Status.Conditions, infrav1.ClusterBastionReadyCondition, metav1.ConditionTrue, "Available")
139141
})
140142

143+
It("reports a failed bastion server with its own reason and warns", func() {
144+
got := &infrav1.StackitCluster{}
145+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
146+
got.Spec.Bastion = validBastionSpec()
147+
Expect(k8sClient.Update(ctx, got)).To(Succeed())
148+
_, err := reconciler.Reconcile(ctx, request)
149+
Expect(err).NotTo(HaveOccurred())
150+
151+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
152+
fakeCloud.SetServerState(got.Status.Bastion.ServerID, "ERROR", "")
153+
recorder := events.NewFakeRecorder(10)
154+
reconciler.Recorder = recorder
155+
156+
result, err := reconciler.Reconcile(ctx, request)
157+
Expect(err).NotTo(HaveOccurred())
158+
Expect(result.RequeueAfter).To(Equal(15 * time.Second))
159+
160+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
161+
expectCondition(got.Status.Conditions, infrav1.ClusterBastionReadyCondition, metav1.ConditionFalse, "InstanceFailed")
162+
condition := meta.FindStatusCondition(got.Status.Conditions, infrav1.ClusterBastionReadyCondition)
163+
Expect(condition.Message).To(Equal("bastion server failed (state ERROR)"))
164+
Expect(drainEvents(recorder)).To(ContainElement(HavePrefix("Warning InstanceFailed")))
165+
})
166+
141167
It("deletes existing bastion resources when bastion is disabled", func() {
142168
got := &infrav1.StackitCluster{}
143169
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())

‎controller/stackitmachine_controller_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ import (
1919
. "github.com/onsi/gomega"
2020
corev1 "k8s.io/api/core/v1"
2121
apierrors "k8s.io/apimachinery/pkg/api/errors"
22+
"k8s.io/apimachinery/pkg/api/meta"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2324
"k8s.io/apimachinery/pkg/types"
25+
"k8s.io/client-go/tools/events"
2426
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
2527
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2628

@@ -178,6 +180,59 @@ var _ = Describe("StackitMachine Controller", func() {
178180
"legacy status.ready must follow the Ready condition, not contradict it")
179181
})
180182

183+
It("reports a stopped server with its own reason and warns once", func() {
184+
recorder := events.NewFakeRecorder(10)
185+
reconciler.Recorder = recorder
186+
_, err := reconciler.Reconcile(ctx, request)
187+
Expect(err).NotTo(HaveOccurred())
188+
Expect(drainEvents(recorder)).To(ContainElement(HavePrefix("Normal InstanceCreated")))
189+
190+
got := &infrav1.StackitMachine{}
191+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
192+
193+
By("passing through the transitional STOPPING state without a warning")
194+
fakeCloud.SetServerState(got.Status.InstanceID, "STOPPING", "RUNNING")
195+
_, err = reconciler.Reconcile(ctx, request)
196+
Expect(err).NotTo(HaveOccurred())
197+
Expect(drainEvents(recorder)).NotTo(ContainElement(HavePrefix("Warning")))
198+
199+
By("warning once the server is stopped")
200+
fakeCloud.SetServerState(got.Status.InstanceID, "INACTIVE", "STOPPED")
201+
result, err := reconciler.Reconcile(ctx, request)
202+
Expect(err).NotTo(HaveOccurred())
203+
Expect(result.RequeueAfter).To(Equal(15 * time.Second))
204+
205+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
206+
Expect(got.Status.Ready).To(BeFalse())
207+
Expect(got.Status.InstanceState).To(Equal("INACTIVE"))
208+
expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceStopped")
209+
condition := meta.FindStatusCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition)
210+
Expect(condition.Message).To(Equal("server is stopped (state INACTIVE, power status STOPPED)"))
211+
Expect(drainEvents(recorder)).To(ContainElement(HavePrefix("Warning InstanceStopped")))
212+
213+
By("not warning again while the server stays stopped")
214+
_, err = reconciler.Reconcile(ctx, request)
215+
Expect(err).NotTo(HaveOccurred())
216+
Expect(drainEvents(recorder)).NotTo(ContainElement(HavePrefix("Warning")))
217+
})
218+
219+
It("reports an active but crashed server as not ready", func() {
220+
_, err := reconciler.Reconcile(ctx, request)
221+
Expect(err).NotTo(HaveOccurred())
222+
223+
got := &infrav1.StackitMachine{}
224+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
225+
fakeCloud.SetServerState(got.Status.InstanceID, "ACTIVE", "CRASHED")
226+
227+
_, err = reconciler.Reconcile(ctx, request)
228+
Expect(err).NotTo(HaveOccurred())
229+
230+
Expect(k8sClient.Get(ctx, stackitKey, got)).To(Succeed())
231+
Expect(got.Status.Ready).To(BeFalse())
232+
expectCondition(got.Status.Conditions, infrav1.MachineInstanceReadyCondition, metav1.ConditionFalse, "InstanceCrashed")
233+
expectCondition(got.Status.Conditions, infrav1.MachineReadyCondition, metav1.ConditionFalse, "InstanceCrashed")
234+
})
235+
181236
It("attaches provider-managed node SSH access when bastion is enabled", func() {
182237
stackitCluster := &infrav1.StackitCluster{}
183238
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: namespace}, stackitCluster)).To(Succeed())

0 commit comments

Comments
 (0)