Skip to content

fix(systemtest): the Hetzner lane picks its location at run time (ankra-ews9qk) #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Myothas merged 2 commits into from
Oct 14, 2026
Merged

fix(systemtest): the Hetzner lane picks its location at run time (ankra-ews9qk) #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
011842f
fix(systemtest): the Hetzner lane picks its location at run time
Myothas Oct 14, 2026
f20d958
fix(systemtest): an unreadable Hetzner location is never read as lack…
Myothas Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 73 additions & 6 deletions systemtest/create_failure_reason_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,82 @@ func TestCreateFailureReasonReportsThePlatformsRefusal(t *testing.T) {
}
}

// TestHetznerLaneDefaultsToALocationThatOffersItsServerTypes keeps the
// Hetzner lane off nbg1, where Hetzner stopped offering cpx32 in 2026-10 and
// every create was refused (ankra-ews9qk).
func TestHetznerLaneDefaultsToALocationThatOffersItsServerTypes(t *testing.T) {
// TestHetznerLanePicksALocationThatOffersItsTypesNow pins ankra-ews9qk's
// second half: cpx32 left nbg1, then fsn1, then came back to nbg1 within the
// hour on 2026-10-14, so no fixed location keeps the lane creatable. The lane
// picks the first candidate that offers every type it uses right now.
func TestHetznerLanePicksALocationThatOffersItsTypesNow(t *testing.T) {
bash, lookError := exec.LookPath("bash")
if lookError != nil {
t.Skip("bash not available")
}
script, readError := os.ReadFile("lifecycle_systemtest.sh")
if readError != nil {
t.Fatal(readError)
}
if !regexp.MustCompile(`(?m)^HETZNER_LOCATION="\$\{HETZNER_LOCATION:-fsn1\}"$`).Match(script) {
t.Error("HETZNER_LOCATION must default to fsn1 (cpx22 and cpx32 are offered there)")
if !regexp.MustCompile(`(?m)^HETZNER_LOCATION="\$\{HETZNER_LOCATION:-\}"$`).Match(script) {
t.Error("HETZNER_LOCATION must default to empty, so the lane picks its location at run time")
}
extract := func(name string) string {
definition := regexp.MustCompile(`(?ms)^` + name + `\(\) \{.*?^\}`).Find(script)
if definition == nil {
t.Fatalf("%s is not defined in lifecycle_systemtest.sh", name)
}
return string(definition)
}
tableParser := extract("hetzner_types_from_table")
picker := extract("pick_hetzner_location")

rows := " cax11 2 4 GB 40 GB arm 5.99 yes\n" +
" cpx22 2 4 GB 80 GB x86 19.49 yes\n" +
" cpx32 4 8 GB 160 GB x86 35.49 no\n"
header := " NAME CORES MEMORY DISK ARCH PRICE/MO AVAILABLE\n"
for _, table := range []string{"Server types for location fsn1:\n" + header + rows, header + rows, rows} {
parsed, parseError := exec.Command(bash, "-c", tableParser+"\nprintf '%s' \"$1\" | hetzner_types_from_table", "bash", table).Output()
if parseError != nil {
t.Fatal(parseError)
}
if got := strings.Fields(string(parsed)); strings.Join(got, " ") != "cax11 cpx22" {
t.Errorf("hetzner_types_from_table = %v, want [cax11 cpx22] for a table with %d preamble lines", got, strings.Count(table, "\n")-3)
}
}

// The stub plays the minute the run began: fsn1 lacks cpx32, nbg1 has
// everything, hel1 would too.
stub := `hetzner_available_types() { case "$1" in fsn1) printf 'cpx22\ncpx12\n';; nbg1|hel1) printf 'cpx22\ncpx32\n';; broken) return 7;; empty) return 0;; esac; }`
types := "HETZNER_CP_TYPE=cpx32 HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32"
cases := []struct {
name string
candidates string
types string
want string
wantFailed bool
}{
{"skips a location missing a type", "fsn1 nbg1 hel1", types, "nbg1", false},
{"keeps the candidate order", "hel1 nbg1", types, "hel1", false},
{"skips a location it could not read", "broken nbg1", types, "nbg1", false},
{"no candidate offers every type", "fsn1", types,
"no Hetzner location in 'fsn1' offers cpx32, cpx22, cpx22 and cpx32 right now", true},
{"a location it could not read is never read as lacking", "fsn1 broken empty", types,
"no readable Hetzner location offers cpx32, cpx22, cpx22 and cpx32 right now (lacking: fsn1; could not read: broken empty)", true},
{"nothing could be read at all", "broken empty", types,
"could not list Hetzner server types for any of 'broken empty' (a credential or API problem, not Hetzner's stock)", true},
{"an empty type is a configuration error", "nbg1",
"HETZNER_CP_TYPE= HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32",
"configuration error: HETZNER_CP_TYPE, HETZNER_WORKER_TYPE, HETZNER_BASTION_TYPE and HETZNER_BIGGER_TYPE must all be set", true},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
program := "set -euo pipefail\n" + stub + "\n" + picker + "\n" + testCase.types +
"\nHETZNER_LOCATION_CANDIDATES=\"$1\"\npick_hetzner_location"
output, runError := exec.Command(bash, "-c", program, "bash", testCase.candidates).Output()
failed := runError != nil
if failed != testCase.wantFailed {
t.Fatalf("pick_hetzner_location(%q) failed=%v, want %v (output %q)", testCase.candidates, failed, testCase.wantFailed, output)
}
if got := strings.TrimSpace(string(output)); got != testCase.want {
t.Errorf("pick_hetzner_location(%q) = %q, want %q", testCase.candidates, got, testCase.want)
}
})
}
}
83 changes: 79 additions & 4 deletions systemtest/lifecycle_systemtest.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,13 @@ GITOPS_REPOSITORY="${GITOPS_REPOSITORY:-}"
GITOPS_BRANCH="${GITOPS_BRANCH:-master}"

# Regions / zones / locations.
# fsn1, not nbg1: since 2026-10 Hetzner no longer offers cpx32 in nbg1, and
# every create there was refused (ankra-ews9qk). fsn1 and hel1 still carry
# cpx22 and cpx32 at the same price.
HETZNER_LOCATION="${HETZNER_LOCATION:-fsn1}"
# Hetzner's shared-vCPU stock moves by the minute: on 2026-10-14 cpx32 left
# nbg1, then fsn1, then came back to nbg1 within the hour, and a create in a
# location without it is refused (ankra-ews9qk). So the lane picks its
# location at run time: the first of HETZNER_LOCATION_CANDIDATES that offers
# every server type the lane uses right now. Setting HETZNER_LOCATION pins it.
HETZNER_LOCATION="${HETZNER_LOCATION:-}"
HETZNER_LOCATION_CANDIDATES="${HETZNER_LOCATION_CANDIDATES:-fsn1 nbg1 hel1}"
OVH_REGION="${OVH_REGION:-GRA9}"
UPCLOUD_ZONE="${UPCLOUD_ZONE:-de-fra1}"

Expand Down Expand Up @@ -433,6 +436,68 @@ ank_out() {
fi
}

# hetzner_available_types prints the server types Hetzner can provision in a
# location right now, one per line, read from the CLI's server-types table
# (NAME CORES MEMORY GB DISK GB ARCH PRICE AVAILABLE; the last column is yes).
hetzner_available_types() {
local table
table="$(ank_out cluster hetzner server-types --credential-id "$HETZNER_CREDENTIAL_ID" \
--location "$1" --available-only)" || return 1
printf '%s\n' "$table" | hetzner_types_from_table
}

# hetzner_types_from_table reads a server-types table on stdin and prints the
# names whose AVAILABLE column says yes. It matches the row's shape (a
# type-like name first, "yes" last) rather than skipping a fixed number of
# preamble lines, so a missing banner or an extra header cannot shift it.
hetzner_types_from_table() {
awk '$1 ~ /^[a-z]+[0-9]+[a-z]*$/ && $NF == "yes" { print $1 }'
}

# pick_hetzner_location prints the first candidate location that offers every
# type the lane uses and succeeds; otherwise it prints why and fails. A
# location whose types could not be read (the CLI failed, or answered no rows)
# is never read as a location that lacks them: the reason names it apart, and
# when no candidate could be read at all it says so (a credential or API
# problem, not Hetzner's stock). It reads each candidate through
# hetzner_available_types, so a test can stub it.
pick_hetzner_location() {
local location needed available is_complete
local unread="" lacking=""
for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do
if [ -z "$needed" ]; then
echo "configuration error: HETZNER_CP_TYPE, HETZNER_WORKER_TYPE, HETZNER_BASTION_TYPE and HETZNER_BIGGER_TYPE must all be set"
return 1
fi
done
for location in $HETZNER_LOCATION_CANDIDATES; do
if ! available="$(hetzner_available_types "$location")" || [ -z "$available" ]; then
unread="${unread:+$unread }$location"
continue
fi
is_complete=1
for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do

@ankra-platform ankra-platform Bot Oct 14, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] Empty HETZNER_*_TYPE variables silently make every location incomplete

If any of HETZNER_CP_TYPE / HETZNER_WORKER_TYPE / HETZNER_BASTION_TYPE / HETZNER_BIGGER_TYPE is unset or empty, grep -qx "$needed" never matches and every candidate is rejected, surfacing as the misleading 'no location offers the types' failure. The diff does not show where these are defaulted; if they can be empty, validate them up front and fail with a configuration error instead.

Ankra AI review. Read-only and advisory.

if ! printf '%s\n' "$available" | grep -qx "$needed"; then
is_complete=0
break
fi
done
if [ "$is_complete" = 1 ]; then
echo "$location"
return 0
fi
lacking="${lacking:+$lacking }$location"
done
if [ -z "$lacking" ]; then
echo "could not list Hetzner server types for any of '$HETZNER_LOCATION_CANDIDATES' (a credential or API problem, not Hetzner's stock)"
elif [ -n "$unread" ]; then
echo "no readable Hetzner location offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now (lacking: $lacking; could not read: $unread)"
else
echo "no Hetzner location in '$HETZNER_LOCATION_CANDIDATES' offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now"
fi
return 1
}

# create_failure_reason says why a create printed no "Cluster ID:" line. The
# platform refuses a create it cannot run with a 400 whose "detail" names the
# cause (a server type the location no longer offers, a provider account with
Expand Down Expand Up @@ -1891,6 +1956,16 @@ run_provider() {

section "$label :: $name"

if [ "$provider" = hetzner ] && [ -z "$HETZNER_LOCATION" ]; then
local picked
if ! picked="$(pick_hetzner_location)"; then
fail "$label create ($picked)"
return
fi
HETZNER_LOCATION="$picked"

@ankra-platform ankra-platform Bot Oct 14, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] Picked location is pinned for the rest of the run

Assigning HETZNER_LOCATION here mutates the global, so any later hetzner lane in the same script run reuses this pick without re-probing. That is probably desirable (a mid-run location switch would be surprising), but if the intent was per-lane freshness, the assignment should be scoped or re-picked per lane. Worth a one-line comment either way.

Ankra AI review. Read-only and advisory.

log "hetzner location: $HETZNER_LOCATION (offers every type this lane uses right now)"
fi

# 1. Create (capture the printed "Cluster ID: <uuid>")
log "creating $name (distribution=$distribution) ..."
out="$(create_cluster "$provider" "$name" "$distribution")"
Expand Down
Loading