diff --git a/systemtest/create_failure_reason_test.go b/systemtest/create_failure_reason_test.go index e16b21a0..945d2f44 100644 --- a/systemtest/create_failure_reason_test.go +++ b/systemtest/create_failure_reason_test.go @@ -71,15 +71,82 @@ func TestCreateFailureReasonReportsThePlatformsRefusal(t *testing.T) { } } -// TestHetznerLaneDefaultsToALocationThatOffersItsServerTypes keeps the -// Hetzner lane off nbg1, where Hetzner stopped offering cpx32 in 2026-10 and -// every create was refused (ankra-ews9qk). -func TestHetznerLaneDefaultsToALocationThatOffersItsServerTypes(t *testing.T) { +// TestHetznerLanePicksALocationThatOffersItsTypesNow pins ankra-ews9qk's +// second half: cpx32 left nbg1, then fsn1, then came back to nbg1 within the +// hour on 2026-10-14, so no fixed location keeps the lane creatable. The lane +// picks the first candidate that offers every type it uses right now. +func TestHetznerLanePicksALocationThatOffersItsTypesNow(t *testing.T) { + bash, lookError := exec.LookPath("bash") + if lookError != nil { + t.Skip("bash not available") + } script, readError := os.ReadFile("lifecycle_systemtest.sh") if readError != nil { t.Fatal(readError) } - if !regexp.MustCompile(`(?m)^HETZNER_LOCATION="\$\{HETZNER_LOCATION:-fsn1\}"$`).Match(script) { - t.Error("HETZNER_LOCATION must default to fsn1 (cpx22 and cpx32 are offered there)") + if !regexp.MustCompile(`(?m)^HETZNER_LOCATION="\$\{HETZNER_LOCATION:-\}"$`).Match(script) { + t.Error("HETZNER_LOCATION must default to empty, so the lane picks its location at run time") + } + extract := func(name string) string { + definition := regexp.MustCompile(`(?ms)^` + name + `\(\) \{.*?^\}`).Find(script) + if definition == nil { + t.Fatalf("%s is not defined in lifecycle_systemtest.sh", name) + } + return string(definition) + } + tableParser := extract("hetzner_types_from_table") + picker := extract("pick_hetzner_location") + + rows := " cax11 2 4 GB 40 GB arm 5.99 yes\n" + + " cpx22 2 4 GB 80 GB x86 19.49 yes\n" + + " cpx32 4 8 GB 160 GB x86 35.49 no\n" + header := " NAME CORES MEMORY DISK ARCH PRICE/MO AVAILABLE\n" + for _, table := range []string{"Server types for location fsn1:\n" + header + rows, header + rows, rows} { + parsed, parseError := exec.Command(bash, "-c", tableParser+"\nprintf '%s' \"$1\" | hetzner_types_from_table", "bash", table).Output() + if parseError != nil { + t.Fatal(parseError) + } + if got := strings.Fields(string(parsed)); strings.Join(got, " ") != "cax11 cpx22" { + t.Errorf("hetzner_types_from_table = %v, want [cax11 cpx22] for a table with %d preamble lines", got, strings.Count(table, "\n")-3) + } + } + + // The stub plays the minute the run began: fsn1 lacks cpx32, nbg1 has + // everything, hel1 would too. + stub := `hetzner_available_types() { case "$1" in fsn1) printf 'cpx22\ncpx12\n';; nbg1|hel1) printf 'cpx22\ncpx32\n';; broken) return 7;; empty) return 0;; esac; }` + types := "HETZNER_CP_TYPE=cpx32 HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32" + cases := []struct { + name string + candidates string + types string + want string + wantFailed bool + }{ + {"skips a location missing a type", "fsn1 nbg1 hel1", types, "nbg1", false}, + {"keeps the candidate order", "hel1 nbg1", types, "hel1", false}, + {"skips a location it could not read", "broken nbg1", types, "nbg1", false}, + {"no candidate offers every type", "fsn1", types, + "no Hetzner location in 'fsn1' offers cpx32, cpx22, cpx22 and cpx32 right now", true}, + {"a location it could not read is never read as lacking", "fsn1 broken empty", types, + "no readable Hetzner location offers cpx32, cpx22, cpx22 and cpx32 right now (lacking: fsn1; could not read: broken empty)", true}, + {"nothing could be read at all", "broken empty", types, + "could not list Hetzner server types for any of 'broken empty' (a credential or API problem, not Hetzner's stock)", true}, + {"an empty type is a configuration error", "nbg1", + "HETZNER_CP_TYPE= HETZNER_WORKER_TYPE=cpx22 HETZNER_BASTION_TYPE=cpx22 HETZNER_BIGGER_TYPE=cpx32", + "configuration error: HETZNER_CP_TYPE, HETZNER_WORKER_TYPE, HETZNER_BASTION_TYPE and HETZNER_BIGGER_TYPE must all be set", true}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + program := "set -euo pipefail\n" + stub + "\n" + picker + "\n" + testCase.types + + "\nHETZNER_LOCATION_CANDIDATES=\"$1\"\npick_hetzner_location" + output, runError := exec.Command(bash, "-c", program, "bash", testCase.candidates).Output() + failed := runError != nil + if failed != testCase.wantFailed { + t.Fatalf("pick_hetzner_location(%q) failed=%v, want %v (output %q)", testCase.candidates, failed, testCase.wantFailed, output) + } + if got := strings.TrimSpace(string(output)); got != testCase.want { + t.Errorf("pick_hetzner_location(%q) = %q, want %q", testCase.candidates, got, testCase.want) + } + }) } } diff --git a/systemtest/lifecycle_systemtest.sh b/systemtest/lifecycle_systemtest.sh index 815cc35d..5846ee77 100755 --- a/systemtest/lifecycle_systemtest.sh +++ b/systemtest/lifecycle_systemtest.sh @@ -189,10 +189,13 @@ GITOPS_REPOSITORY="${GITOPS_REPOSITORY:-}" GITOPS_BRANCH="${GITOPS_BRANCH:-master}" # Regions / zones / locations. -# fsn1, not nbg1: since 2026-10 Hetzner no longer offers cpx32 in nbg1, and -# every create there was refused (ankra-ews9qk). fsn1 and hel1 still carry -# cpx22 and cpx32 at the same price. -HETZNER_LOCATION="${HETZNER_LOCATION:-fsn1}" +# Hetzner's shared-vCPU stock moves by the minute: on 2026-10-14 cpx32 left +# nbg1, then fsn1, then came back to nbg1 within the hour, and a create in a +# location without it is refused (ankra-ews9qk). So the lane picks its +# location at run time: the first of HETZNER_LOCATION_CANDIDATES that offers +# every server type the lane uses right now. Setting HETZNER_LOCATION pins it. +HETZNER_LOCATION="${HETZNER_LOCATION:-}" +HETZNER_LOCATION_CANDIDATES="${HETZNER_LOCATION_CANDIDATES:-fsn1 nbg1 hel1}" OVH_REGION="${OVH_REGION:-GRA9}" UPCLOUD_ZONE="${UPCLOUD_ZONE:-de-fra1}" @@ -433,6 +436,68 @@ ank_out() { fi } +# hetzner_available_types prints the server types Hetzner can provision in a +# location right now, one per line, read from the CLI's server-types table +# (NAME CORES MEMORY GB DISK GB ARCH PRICE AVAILABLE; the last column is yes). +hetzner_available_types() { + local table + table="$(ank_out cluster hetzner server-types --credential-id "$HETZNER_CREDENTIAL_ID" \ + --location "$1" --available-only)" || return 1 + printf '%s\n' "$table" | hetzner_types_from_table +} + +# hetzner_types_from_table reads a server-types table on stdin and prints the +# names whose AVAILABLE column says yes. It matches the row's shape (a +# type-like name first, "yes" last) rather than skipping a fixed number of +# preamble lines, so a missing banner or an extra header cannot shift it. +hetzner_types_from_table() { + awk '$1 ~ /^[a-z]+[0-9]+[a-z]*$/ && $NF == "yes" { print $1 }' +} + +# pick_hetzner_location prints the first candidate location that offers every +# type the lane uses and succeeds; otherwise it prints why and fails. A +# location whose types could not be read (the CLI failed, or answered no rows) +# is never read as a location that lacks them: the reason names it apart, and +# when no candidate could be read at all it says so (a credential or API +# problem, not Hetzner's stock). It reads each candidate through +# hetzner_available_types, so a test can stub it. +pick_hetzner_location() { + local location needed available is_complete + local unread="" lacking="" + for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do + if [ -z "$needed" ]; then + echo "configuration error: HETZNER_CP_TYPE, HETZNER_WORKER_TYPE, HETZNER_BASTION_TYPE and HETZNER_BIGGER_TYPE must all be set" + return 1 + fi + done + for location in $HETZNER_LOCATION_CANDIDATES; do + if ! available="$(hetzner_available_types "$location")" || [ -z "$available" ]; then + unread="${unread:+$unread }$location" + continue + fi + is_complete=1 + for needed in "$HETZNER_CP_TYPE" "$HETZNER_WORKER_TYPE" "$HETZNER_BASTION_TYPE" "$HETZNER_BIGGER_TYPE"; do + if ! printf '%s\n' "$available" | grep -qx "$needed"; then + is_complete=0 + break + fi + done + if [ "$is_complete" = 1 ]; then + echo "$location" + return 0 + fi + lacking="${lacking:+$lacking }$location" + done + if [ -z "$lacking" ]; then + echo "could not list Hetzner server types for any of '$HETZNER_LOCATION_CANDIDATES' (a credential or API problem, not Hetzner's stock)" + elif [ -n "$unread" ]; then + echo "no readable Hetzner location offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now (lacking: $lacking; could not read: $unread)" + else + echo "no Hetzner location in '$HETZNER_LOCATION_CANDIDATES' offers $HETZNER_CP_TYPE, $HETZNER_WORKER_TYPE, $HETZNER_BASTION_TYPE and $HETZNER_BIGGER_TYPE right now" + fi + return 1 +} + # create_failure_reason says why a create printed no "Cluster ID:" line. The # platform refuses a create it cannot run with a 400 whose "detail" names the # cause (a server type the location no longer offers, a provider account with @@ -1891,6 +1956,16 @@ run_provider() { section "$label :: $name" + if [ "$provider" = hetzner ] && [ -z "$HETZNER_LOCATION" ]; then + local picked + if ! picked="$(pick_hetzner_location)"; then + fail "$label create ($picked)" + return + fi + HETZNER_LOCATION="$picked" + log "hetzner location: $HETZNER_LOCATION (offers every type this lane uses right now)" + fi + # 1. Create (capture the printed "Cluster ID: ") log "creating $name (distribution=$distribution) ..." out="$(create_cluster "$provider" "$name" "$distribution")"