From 011842f0e261c8d2a9096b5a97f0d0f986770af3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20=C3=85sell?= Date: Wed, 14 Oct 2026 15:42:54 +0200 Subject: [PATCH 1/2] fix(systemtest): the Hetzner lane picks its location at run time The fsn1 default from #510 was refused within the hour: Hetzner's shared-vCPU stock moves by the minute (on 2026-10-14 cpx32 left nbg1, then fsn1, then came back to nbg1), so no fixed location keeps the lane creatable. HETZNER_LOCATION now defaults to empty and the lane takes the first of HETZNER_LOCATION_CANDIDATES (fsn1 nbg1 hel1) that offers every type it uses right now (control plane, worker, bastion and the resize type), read from 'ankra cluster hetzner server-types --available-only' with the lane's own credential. When none does, the lane fails saying exactly that. Setting HETZNER_LOCATION still pins it. Bead: ankra-ews9qk Co-Authored-By: Claude Opus 5.5 --- systemtest/create_failure_reason_test.go | 68 +++++++++++++++++++++--- systemtest/lifecycle_systemtest.sh | 55 +++++++++++++++++-- 2 files changed, 113 insertions(+), 10 deletions(-) diff --git a/systemtest/create_failure_reason_test.go b/systemtest/create_failure_reason_test.go index e16b21a0..73d10a1e 100644 --- a/systemtest/create_failure_reason_test.go +++ b/systemtest/create_failure_reason_test.go @@ -71,15 +71,71 @@ func TestCreateFailureReasonReportsThePlatformsRefusal(t *testing.T) { } } -// TestHetznerLaneDefaultsToALocationThatOffersItsServerTypes keeps the -// Hetzner lane off nbg1, where Hetzner stopped offering cpx32 in 2026-10 and -// every create was refused (ankra-ews9qk). -func TestHetznerLaneDefaultsToALocationThatOffersItsServerTypes(t *testing.T) { +// TestHetznerLanePicksALocationThatOffersItsTypesNow pins ankra-ews9qk's +// second half: cpx32 left nbg1, then fsn1, then came back to nbg1 within the +// hour on 2026-10-14, so no fixed location keeps the lane creatable. The lane +// picks the first candidate that offers every type it uses right now. +func TestHetznerLanePicksALocationThatOffersItsTypesNow(t *testing.T) { + bash, lookError := exec.LookPath("bash") + if lookError != nil { + t.Skip("bash not available") + } script, readError := os.ReadFile("lifecycle_systemtest.sh") if readError != nil { t.Fatal(readError) } - if !regexp.MustCompile(`(?m)^HETZNER_LOCATION="\$\{HETZNER_LOCATION:-fsn1\}"$`).Match(script) { - t.Error("HETZNER_LOCATION must default to fsn1 (cpx22 and cpx32 are offered there)") + if !regexp.MustCompile(`(?m)^HETZNER_LOCATION="\$\{HETZNER_LOCATION:-\}"$`).Match(script) { + t.Error("HETZNER_LOCATION must default to empty, so the lane picks its location at run time") + } + extract := func(name string) string { + definition := regexp.MustCompile(`(?ms)^` + name + `\(\) \{.*?^\}`).Find(script) + if definition == nil { + t.Fatalf("%s is not defined in lifecycle_systemtest.sh", name) + } + return string(definition) + } + tableParser := extract("hetzner_types_from_table") + picker := extract("pick_hetzner_location") + + table := "Server types for location fsn1:\n" + + " NAME CORES MEMORY DISK ARCH PRICE/MO AVAILABLE\n" + + " cax11 2 4 GB 40 GB arm 5.99 yes\n" + + " cpx22 2 4 GB 80 GB x86 19.49 yes\n" + + " cpx32 4 8 GB 160 GB x86 35.49 no\n" + parsed, parseError := exec.Command(bash, "-c", tableParser+"\nprintf '%s' \"$1\" | hetzner_types_from_table", "bash", table).Output() + if parseError != nil { + t.Fatal(parseError) + } + if got := strings.Fields(string(parsed)); strings.Join(got, " ") != "cax11 cpx22" { + t.Errorf("hetzner_types_from_table = %v, want [cax11 cpx22] (header skipped, unavailable rows dropped)", got) + } + + // The stub plays the minute the run began: fsn1 lacks cpx32, nbg1 has + // everything, hel1 would too. + stub := `hetzner_available_types() { case "$1" in fsn1) printf 'cpx22\ncpx12\n';; nbg1|hel1) printf 'cpx22\ncpx32\n';; esac; }` + types := "HETZNER_CP_TYPE=cpx32 HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32" + cases := []struct { + name string + candidates string + want string + wantFailed bool + }{ + {"skips a location missing a type", "fsn1 nbg1 hel1", "nbg1", false}, + {"keeps the candidate order", "hel1 nbg1", "hel1", false}, + {"fails when no candidate offers every type", "fsn1", "", true}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + program := "set -euo pipefail\n" + stub + "\n" + picker + "\n" + types + + "\nHETZNER_LOCATION_CANDIDATES=\"$1\"\npick_hetzner_location" + output, runError := exec.Command(bash, "-c", program, "bash", testCase.candidates).Output() + failed := runError != nil + if failed != testCase.wantFailed { + t.Fatalf("pick_hetzner_location(%q) failed=%v, want %v (output %q)", testCase.candidates, failed, testCase.wantFailed, output) + } + if got := strings.TrimSpace(string(output)); got != testCase.want { + t.Errorf("pick_hetzner_location(%q) = %q, want %q", testCase.candidates, got, testCase.want) + } + }) } } diff --git a/systemtest/lifecycle_systemtest.sh b/systemtest/lifecycle_systemtest.sh index 815cc35d..bb37d3a4 100755 --- a/systemtest/lifecycle_systemtest.sh +++ b/systemtest/lifecycle_systemtest.sh @@ -189,10 +189,13 @@ GITOPS_REPOSITORY="${GITOPS_REPOSITORY:-}" GITOPS_BRANCH="${GITOPS_BRANCH:-master}" # Regions / zones / locations. -# fsn1, not nbg1: since 2026-10 Hetzner no longer offers cpx32 in nbg1, and -# every create there was refused (ankra-ews9qk). fsn1 and hel1 still carry -# cpx22 and cpx32 at the same price. -HETZNER_LOCATION="${HETZNER_LOCATION:-fsn1}" +# Hetzner's shared-vCPU stock moves by the minute: on 2026-10-14 cpx32 left +# nbg1, then fsn1, then came back to nbg1 within the hour, and a create in a +# location without it is refused (ankra-ews9qk). So the lane picks its +# location at run time: the first of HETZNER_LOCATION_CANDIDATES that offers +# every server type the lane uses right now. Setting HETZNER_LOCATION pins it. +HETZNER_LOCATION="${HETZNER_LOCATION:-}" +HETZNER_LOCATION_CANDIDATES="${HETZNER_LOCATION_CANDIDATES:-fsn1 nbg1 hel1}" OVH_REGION="${OVH_REGION:-GRA9}" UPCLOUD_ZONE="${UPCLOUD_ZONE:-de-fra1}" @@ -433,6 +436,42 @@ ank_out() { fi } +# hetzner_available_types prints the server types Hetzner can provision in a +# location right now, one per line, read from the CLI's server-types table +# (NAME CORES MEMORY GB DISK GB ARCH PRICE AVAILABLE; the last column is yes). +hetzner_available_types() { + ank_out cluster hetzner server-types --credential-id "$HETZNER_CREDENTIAL_ID" \ + --location "$1" --available-only | hetzner_types_from_table +} + +# hetzner_types_from_table reads a server-types table on stdin and prints the +# names whose AVAILABLE column says yes. +hetzner_types_from_table() { + awk 'NR > 2 && NF >= 9 && $NF == "yes" { print $1 }' +} + +# pick_hetzner_location prints the first candidate location that offers every +# type the lane uses, and fails when none does. It reads the types each +# candidate offers through hetzner_available_types, so a test can stub it. +pick_hetzner_location() { + local location needed available is_complete + for location in $HETZNER_LOCATION_CANDIDATES; do + available="$(hetzner_available_types "$location" || true)" + is_complete=1 + for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do + if ! printf '%s\n' "$available" | grep -qx "$needed"; then + is_complete=0 + break + fi + done + if [ "$is_complete" = 1 ]; then + echo "$location" + return 0 + fi + done + return 1 +} + # create_failure_reason says why a create printed no "Cluster ID:" line. The # platform refuses a create it cannot run with a 400 whose "detail" names the # cause (a server type the location no longer offers, a provider account with @@ -1891,6 +1930,14 @@ run_provider() { section "$label :: $name" + if [ "$provider" = hetzner ] && [ -z "$HETZNER_LOCATION" ]; then + if ! HETZNER_LOCATION="$(pick_hetzner_location)"; then + fail "$label create (no Hetzner location in '$HETZNER_LOCATION_CANDIDATES' offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now)" + return + fi + log "hetzner location: $HETZNER_LOCATION (offers every type this lane uses right now)" + fi + # 1. Create (capture the printed "Cluster ID: ") log "creating $name (distribution=$distribution) ..." out="$(create_cluster "$provider" "$name" "$distribution")" From f20d958b3f94bc48fa4d083537d496ad67a07567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20=C3=85sell?= Date: Wed, 14 Oct 2026 16:14:04 +0200 Subject: [PATCH 2/2] fix(systemtest): an unreadable Hetzner location is never read as lacking a type Review of #511: a failed or empty server-types read was indistinguishable from a location that lacks a type, so a credential or API problem could surface as 'no location offers the types'. pick_hetzner_location now names unread locations apart, says so when none could be read, and reports empty type variables as a configuration error; the table parser matches the row shape instead of skipping a fixed preamble. Bead: ankra-ews9qk Co-Authored-By: Claude Opus 5.5 --- systemtest/create_failure_reason_test.go | 39 ++++++++++++-------- systemtest/lifecycle_systemtest.sh | 46 +++++++++++++++++++----- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/systemtest/create_failure_reason_test.go b/systemtest/create_failure_reason_test.go index 73d10a1e..945d2f44 100644 --- a/systemtest/create_failure_reason_test.go +++ b/systemtest/create_failure_reason_test.go @@ -97,36 +97,47 @@ func TestHetznerLanePicksALocationThatOffersItsTypesNow(t *testing.T) { tableParser := extract("hetzner_types_from_table") picker := extract("pick_hetzner_location") - table := "Server types for location fsn1:\n" + - " NAME CORES MEMORY DISK ARCH PRICE/MO AVAILABLE\n" + - " cax11 2 4 GB 40 GB arm 5.99 yes\n" + + rows := " cax11 2 4 GB 40 GB arm 5.99 yes\n" + " cpx22 2 4 GB 80 GB x86 19.49 yes\n" + " cpx32 4 8 GB 160 GB x86 35.49 no\n" - parsed, parseError := exec.Command(bash, "-c", tableParser+"\nprintf '%s' \"$1\" | hetzner_types_from_table", "bash", table).Output() - if parseError != nil { - t.Fatal(parseError) - } - if got := strings.Fields(string(parsed)); strings.Join(got, " ") != "cax11 cpx22" { - t.Errorf("hetzner_types_from_table = %v, want [cax11 cpx22] (header skipped, unavailable rows dropped)", got) + header := " NAME CORES MEMORY DISK ARCH PRICE/MO AVAILABLE\n" + for _, table := range []string{"Server types for location fsn1:\n" + header + rows, header + rows, rows} { + parsed, parseError := exec.Command(bash, "-c", tableParser+"\nprintf '%s' \"$1\" | hetzner_types_from_table", "bash", table).Output() + if parseError != nil { + t.Fatal(parseError) + } + if got := strings.Fields(string(parsed)); strings.Join(got, " ") != "cax11 cpx22" { + t.Errorf("hetzner_types_from_table = %v, want [cax11 cpx22] for a table with %d preamble lines", got, strings.Count(table, "\n")-3) + } } // The stub plays the minute the run began: fsn1 lacks cpx32, nbg1 has // everything, hel1 would too. - stub := `hetzner_available_types() { case "$1" in fsn1) printf 'cpx22\ncpx12\n';; nbg1|hel1) printf 'cpx22\ncpx32\n';; esac; }` + stub := `hetzner_available_types() { case "$1" in fsn1) printf 'cpx22\ncpx12\n';; nbg1|hel1) printf 'cpx22\ncpx32\n';; broken) return 7;; empty) return 0;; esac; }` types := "HETZNER_CP_TYPE=cpx32 HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32" cases := []struct { name string candidates string + types string want string wantFailed bool }{ - {"skips a location missing a type", "fsn1 nbg1 hel1", "nbg1", false}, - {"keeps the candidate order", "hel1 nbg1", "hel1", false}, - {"fails when no candidate offers every type", "fsn1", "", true}, + {"skips a location missing a type", "fsn1 nbg1 hel1", types, "nbg1", false}, + {"keeps the candidate order", "hel1 nbg1", types, "hel1", false}, + {"skips a location it could not read", "broken nbg1", types, "nbg1", false}, + {"no candidate offers every type", "fsn1", types, + "no Hetzner location in 'fsn1' offers cpx32, cpx22, cpx22 and cpx32 right now", true}, + {"a location it could not read is never read as lacking", "fsn1 broken empty", types, + "no readable Hetzner location offers cpx32, cpx22, cpx22 and cpx32 right now (lacking: fsn1; could not read: broken empty)", true}, + {"nothing could be read at all", "broken empty", types, + "could not list Hetzner server types for any of 'broken empty' (a credential or API problem, not Hetzner's stock)", true}, + {"an empty type is a configuration error", "nbg1", + "HETZNER_CP_TYPE= HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32", + "configuration error: HETZNER_CP_TYPE, HETZNER_WORKER_TYPE, HETZNER_BASTION_TYPE and HETZNER_BIGGER_TYPE must all be set", true}, } for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { - program := "set -euo pipefail\n" + stub + "\n" + picker + "\n" + types + + program := "set -euo pipefail\n" + stub + "\n" + picker + "\n" + testCase.types + "\nHETZNER_LOCATION_CANDIDATES=\"$1\"\npick_hetzner_location" output, runError := exec.Command(bash, "-c", program, "bash", testCase.candidates).Output() failed := runError != nil diff --git a/systemtest/lifecycle_systemtest.sh b/systemtest/lifecycle_systemtest.sh index bb37d3a4..5846ee77 100755 --- a/systemtest/lifecycle_systemtest.sh +++ b/systemtest/lifecycle_systemtest.sh @@ -440,23 +440,41 @@ ank_out() { # location right now, one per line, read from the CLI's server-types table # (NAME CORES MEMORY GB DISK GB ARCH PRICE AVAILABLE; the last column is yes). hetzner_available_types() { - ank_out cluster hetzner server-types --credential-id "$HETZNER_CREDENTIAL_ID" \ - --location "$1" --available-only | hetzner_types_from_table + local table + table="$(ank_out cluster hetzner server-types --credential-id "$HETZNER_CREDENTIAL_ID" \ + --location "$1" --available-only)" || return 1 + printf '%s\n' "$table" | hetzner_types_from_table } # hetzner_types_from_table reads a server-types table on stdin and prints the -# names whose AVAILABLE column says yes. +# names whose AVAILABLE column says yes. It matches the row's shape (a +# type-like name first, "yes" last) rather than skipping a fixed number of +# preamble lines, so a missing banner or an extra header cannot shift it. hetzner_types_from_table() { - awk 'NR > 2 && NF >= 9 && $NF == "yes" { print $1 }' + awk '$1 ~ /^[a-z]+[0-9]+[a-z]*$/ && $NF == "yes" { print $1 }' } # pick_hetzner_location prints the first candidate location that offers every -# type the lane uses, and fails when none does. It reads the types each -# candidate offers through hetzner_available_types, so a test can stub it. +# type the lane uses and succeeds; otherwise it prints why and fails. A +# location whose types could not be read (the CLI failed, or answered no rows) +# is never read as a location that lacks them: the reason names it apart, and +# when no candidate could be read at all it says so (a credential or API +# problem, not Hetzner's stock). It reads each candidate through +# hetzner_available_types, so a test can stub it. pick_hetzner_location() { local location needed available is_complete + local unread="" lacking="" + for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do + if [ -z "$needed" ]; then + echo "configuration error: HETZNER_CP_TYPE, HETZNER_WORKER_TYPE, HETZNER_BASTION_TYPE and HETZNER_BIGGER_TYPE must all be set" + return 1 + fi + done for location in $HETZNER_LOCATION_CANDIDATES; do - available="$(hetzner_available_types "$location" || true)" + if ! available="$(hetzner_available_types "$location")" || [ -z "$available" ]; then + unread="${unread:+$unread }$location" + continue + fi is_complete=1 for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do if ! printf '%s\n' "$available" | grep -qx "$needed"; then @@ -468,7 +486,15 @@ pick_hetzner_location() { echo "$location" return 0 fi + lacking="${lacking:+$lacking }$location" done + if [ -z "$lacking" ]; then + echo "could not list Hetzner server types for any of '$HETZNER_LOCATION_CANDIDATES' (a credential or API problem, not Hetzner's stock)" + elif [ -n "$unread" ]; then + echo "no readable Hetzner location offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now (lacking: $lacking; could not read: $unread)" + else + echo "no Hetzner location in '$HETZNER_LOCATION_CANDIDATES' offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now" + fi return 1 } @@ -1931,10 +1957,12 @@ run_provider() { section "$label :: $name" if [ "$provider" = hetzner ] && [ -z "$HETZNER_LOCATION" ]; then - if ! HETZNER_LOCATION="$(pick_hetzner_location)"; then - fail "$label create (no Hetzner location in '$HETZNER_LOCATION_CANDIDATES' offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now)" + local picked + if ! picked="$(pick_hetzner_location)"; then + fail "$label create ($picked)" return fi + HETZNER_LOCATION="$picked" log "hetzner location: $HETZNER_LOCATION (offers every type this lane uses right now)" fi