From af0e198f3f097f5326e8f45b0e2df9ba6f819371 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 18:32:37 +0100 Subject: [PATCH] feat(storage): reconcile provider-only migration servers --- .../observe-longhorn-migration-headroom.yaml | 22 +++- .../README.md | 8 ++ .../census.go | 121 ++++++++++++++++++ .../census_test.go | 85 ++++++++++++ .../inventory.go | 10 +- .../main.go | 18 ++- .../workflow_test.go | 2 + 7 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 scripts/verify-longhorn-migration-headroom/census.go create mode 100644 scripts/verify-longhorn-migration-headroom/census_test.go diff --git a/.github/workflows/observe-longhorn-migration-headroom.yaml b/.github/workflows/observe-longhorn-migration-headroom.yaml index b304f5defd..18d0c54a13 100644 --- a/.github/workflows/observe-longhorn-migration-headroom.yaml +++ b/.github/workflows/observe-longhorn-migration-headroom.yaml @@ -5,7 +5,7 @@ name: Observe Longhorn Migration Headroom workflow_dispatch: inputs: confirm: - description: "Type exactly: observe-longhorn-migration-headroom" + description: Type observe-longhorn-migration-headroom or observe-longhorn-provider-account required: true type: string @@ -18,7 +18,7 @@ jobs: timeout-minutes: 10 environment: prod permissions: - contents: read + contents: read # Check out and verify the current reviewed source. concurrency: group: prod-deploy cancel-in-progress: false @@ -33,7 +33,8 @@ jobs: run: | set -euo pipefail [[ "${DISPATCH_REF}" == refs/heads/main && "${GITHUB_RUN_ATTEMPT}" == 1 ]] - [[ "${LONGHORN_HEADROOM_CONFIRM}" == observe-longhorn-migration-headroom ]] + [[ "${LONGHORN_HEADROOM_CONFIRM}" == observe-longhorn-migration-headroom || + "${LONGHORN_HEADROOM_CONFIRM}" == observe-longhorn-provider-account ]] - name: Checkout reviewed source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -76,10 +77,21 @@ jobs: env: HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} run: ./scripts/use-prod-stable-api-endpoint.sh - - name: Read migration headroom + - name: Read the confirmed provider inventory + shell: bash env: HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - run: go run ./scripts/verify-longhorn-migration-headroom before-publish + run: | + set -euo pipefail + case "${LONGHORN_HEADROOM_CONFIRM}" in + observe-longhorn-migration-headroom) + go run ./scripts/verify-longhorn-migration-headroom before-publish + ;; + observe-longhorn-provider-account) + go run ./scripts/verify-longhorn-migration-headroom account-census + ;; + *) exit 1 ;; + esac - name: Require unchanged current main after observation env: GH_TOKEN: ${{ github.token }} diff --git a/scripts/verify-longhorn-migration-headroom/README.md b/scripts/verify-longhorn-migration-headroom/README.md index f3b227d8fe..a5fa6e7672 100644 --- a/scripts/verify-longhorn-migration-headroom/README.md +++ b/scripts/verify-longhorn-migration-headroom/README.md @@ -19,6 +19,14 @@ outcomes. A rejected provider envelope identifies which pagination or capacity checks failed, without printing response values or weakening those checks. Credentials and provider/node payloads remain in runner memory. +If the account is full, confirm `observe-longhorn-provider-account` to reconcile +the migration roster with up to ten provider records. This read-only diagnostic +requires two matching provider lists between unchanged Kubernetes Node reads. +It reports aggregate counts and a fixed classification of at most one +unmatched server based on provider labels. Use a separate, reviewed ownership +and cleanup procedure for any provider write. The normal headroom gate still +requires a reserved builder slot. + The later worker activation must repeat this check immediately before mutable publication and before updating the pool, then require the full nine-node roster after Talos readiness. An earlier workflow result alone is insufficient. These diff --git a/scripts/verify-longhorn-migration-headroom/census.go b/scripts/verify-longhorn-migration-headroom/census.go new file mode 100644 index 0000000000..bf8b6232c1 --- /dev/null +++ b/scripts/verify-longhorn-migration-headroom/census.go @@ -0,0 +1,121 @@ +package main + +import ( + "cmp" + "context" + "errors" + "net/http" + "reflect" + "regexp" + "slices" + "strconv" +) + +const censusConfirmation = "observe-longhorn-provider-account" + +var snapshotBuildID = regexp.MustCompile(`^[A-Z2-7]{26}$`) +var errCensusHeld = errors.New("provider account reconciliation is incomplete, ambiguous or changed") + +type accountCensus struct { + Servers, Nodes, Extra int + Classification, State string +} + +func authorizeAccountCensus(getenv func(string) string) bool { + return getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" && + getenv("GITHUB_REF") == "refs/heads/main" && + getenv("GITHUB_WORKFLOW_REF") == "devantler-tech/platform/.github/workflows/observe-longhorn-migration-headroom.yaml@refs/heads/main" && + getenv("GITHUB_RUN_ATTEMPT") == "1" && getenv("RUNNER_ENVIRONMENT") == "github-hosted" && + getenv("LONGHORN_HEADROOM_CONFIRM") == censusConfirmation +} + +func observeAccountCensus(ctx context.Context, client *http.Client, token string, readNodes func(context.Context) ([]byte, error)) (accountCensus, error) { + read := func() ([]node, error) { + raw, err := readNodes(ctx) + if err != nil { + return nil, errCensusHeld + } + return decodeNodes(raw) + } + before, err := read() + if err != nil { + return accountCensus{}, err + } + first, err := collectServerInventory(ctx, client, token, maxAccountServers) + if err != nil { + return accountCensus{}, err + } + result, err := classifyAccountCensus(first, before) + if err != nil { + return accountCensus{}, err + } + second, err := collectServerInventory(ctx, client, token, maxAccountServers) + if err != nil { + return accountCensus{}, err + } + byID := func(a, b server) int { return cmp.Compare(a.ID, b.ID) } + slices.SortFunc(first, byID) + slices.SortFunc(second, byID) + after, err := read() + if err != nil || !reflect.DeepEqual(first, second) || !reflect.DeepEqual(before, after) || active(ctx) != nil { + return accountCensus{}, errCensusHeld + } + return result, nil +} + +// Classify the one provider-only row for investigation. Its labels never +// establish deletion authority or reserve the builder slot. +func classifyAccountCensus(servers []server, nodes []node) (accountCensus, error) { + extra := len(servers) - len(nodes) + if len(servers) > maxAccountServers || extra < 0 || extra > 1 { + return accountCensus{}, errCensusHeld + } + nodeIDs := make(map[string]bool, len(nodes)) + for _, node := range nodes { + nodeIDs[node.Spec.ProviderID] = true + } + ids, names := map[int64]bool{}, map[string]bool{} + joined := make([]server, 0, len(nodes)) + var unmatched server + for _, server := range servers { + if server.ID <= 0 || server.Name == "" || server.Status == "" || ids[server.ID] || names[server.Name] { + return accountCensus{}, errCensusHeld + } + ids[server.ID], names[server.Name] = true, true + if nodeIDs["hcloud://"+strconv.FormatInt(server.ID, 10)] { + joined = append(joined, server) + } else { + unmatched = server + } + } + if len(joined) != len(nodes) || qualify(joined, nodes, false) != nil { + return accountCensus{}, errCensusHeld + } + result := accountCensus{Servers: len(servers), Nodes: len(nodes), Extra: extra, Classification: "none", State: "none"} + if extra == 1 { + result.State = "not-running" + if unmatched.Status == "running" { + result.State = "running" + } + result.Classification = classifyProviderOnly(unmatched, joined[0].PrivateNet[0].Network) + } + return result, nil +} + +func classifyProviderOnly(s server, network int64) string { + cluster := s.Labels["ksail.cluster.name"] + builderCluster := s.Labels["ksail.io/cluster"] + group := s.Labels["hcloud/node-group"] + switch { + case (cluster != "" && cluster != "prod") || (builderCluster != "" && builderCluster != "prod"): + return "other-cluster-labeled" + case snapshotBuildID.MatchString(s.Labels["ksail.io/snapshot-build"]) && s.Labels["ksail.owned"] == "" && group == "": + return "snapshot-build-labeled" + case cluster == "prod" && s.Labels["ksail.owned"] == "true": + return "production-managed-labeled" + case group != "" && len(s.PrivateNet) == 1 && s.PrivateNet[0].Network == network: + return "autoscaler-labeled" + default: + return "unclassified" + } +} diff --git a/scripts/verify-longhorn-migration-headroom/census_test.go b/scripts/verify-longhorn-migration-headroom/census_test.go new file mode 100644 index 0000000000..8e6e766cf7 --- /dev/null +++ b/scripts/verify-longhorn-migration-headroom/census_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "io" + "net/http" + "strings" + "testing" +) + +func TestAccountCensusReportsBuilderWithoutGrantingCapacity(t *testing.T) { + servers, nodes := roster(t, true) + servers = append(servers, server{ID: 999, Name: "temporary-provider-fixture", Status: "running", Labels: map[string]string{ + "ksail.io/cluster": "prod", "ksail.io/snapshot-build": "ABCDEFGHIJKLMNOPQRSTUVWXYZ", + }}) + body := providerResponse(t, func(response map[string]any) { + response["servers"] = servers + response["meta"].(map[string]any)["pagination"].(map[string]any)["total_entries"] = len(servers) + }) + client := providerClient(t, body, 200) + readNodes := func(context.Context) ([]byte, error) { return nodeResponse(t, nodes), nil } + got, err := observeAccountCensus(t.Context(), client, "private-test-token", readNodes) + if err != nil || got != (accountCensus{Servers: 10, Nodes: 9, Extra: 1, Classification: "snapshot-build-labeled", State: "running"}) { + t.Fatalf("provider-only builder was not accounted for: %+v, %v", got, err) + } + if count, err := observe(t.Context(), client, "private-test-token", readNodes, false); err == nil || count != 0 { + t.Fatal("the full account qualified for the reserved builder slot") + } +} + +func TestAccountCensusRefusesAmbiguousIdentityAndNames(t *testing.T) { + servers, nodes := roster(t, true) + servers = append(servers, server{ID: 999, Name: "snapshot-builder-prod", Status: "off"}) + got, err := classifyAccountCensus(servers, nodes) + if err != nil || got.Classification != "unclassified" || got.State != "not-running" { + t.Fatalf("an unowned name was attributed to a builder: %+v, %v", got, err) + } + servers[9].ID = servers[0].ID + if _, err := classifyAccountCensus(servers, nodes); err == nil { + t.Fatal("duplicate provider identity qualified") + } +} + +func TestAccountCensusRejectsProviderDrift(t *testing.T) { + servers, nodes := roster(t, true) + body := func(status string) string { + return providerResponse(t, func(response map[string]any) { + response["servers"] = append(servers, server{ID: 999, Name: "temporary-provider-fixture", Status: status}) + response["meta"].(map[string]any)["pagination"].(map[string]any)["total_entries"] = 10 + }) + } + calls := 0 + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + payload := body("running") + if calls > 0 { + payload = body("stopping") + } + calls++ + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(payload))}, nil + })} + read := func(context.Context) ([]byte, error) { return nodeResponse(t, nodes), nil } + if result, err := observeAccountCensus(t.Context(), client, "private-test-token", read); err == nil || result != (accountCensus{}) || calls != 2 { + t.Fatal("a changing provider account produced a reconciliation receipt") + } +} + +func TestAccountCensusRequiresExactProtectedDispatch(t *testing.T) { + env := map[string]string{ + "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/observe-longhorn-migration-headroom.yaml@refs/heads/main", + "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", + "LONGHORN_HEADROOM_CONFIRM": "observe-longhorn-provider-account", + } + get := func(key string) string { return env[key] } + if !authorizeAccountCensus(get) { + t.Fatal("exact protected account reconciliation was refused") + } + for key, value := range env { + env[key] = "" + if authorizeAccountCensus(get) { + t.Fatalf("account reconciliation authorized without %s", key) + } + env[key] = value + } +} diff --git a/scripts/verify-longhorn-migration-headroom/inventory.go b/scripts/verify-longhorn-migration-headroom/inventory.go index a5dc82d0ee..0f627ac89f 100644 --- a/scripts/verify-longhorn-migration-headroom/inventory.go +++ b/scripts/verify-longhorn-migration-headroom/inventory.go @@ -13,6 +13,8 @@ import ( ) const maxInventoryBytes = 2 * 1024 * 1024 +const maxHeadroomServers = 9 +const maxAccountServers = 10 var serverFields = map[string][]string{ "": {"servers", "meta"}, @@ -37,6 +39,10 @@ var nodeFields = map[string][]string{ } func collectServers(ctx context.Context, client *http.Client, token string) ([]server, error) { + return collectServerInventory(ctx, client, token, maxHeadroomServers) +} + +func collectServerInventory(ctx context.Context, client *http.Client, token string, limit int) ([]server, error) { if token == "" { return nil, errors.New("provider read credential is unavailable") } @@ -75,11 +81,11 @@ func collectServers(ctx context.Context, client *http.Client, token string) ([]s p := page.Meta.Pagination // The account limit is ten, below this fifty-entry request. Anything that // needs another page cannot qualify for a free slot; never treat it as empty. - if page.Servers == nil || len(page.Servers) > 9 || p.Page == nil || *p.Page != 1 || p.PerPage == nil || *p.PerPage != 50 || p.LastPage == nil || *p.LastPage != 1 || !bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")) || p.Total == nil || *p.Total != len(page.Servers) { + if page.Servers == nil || len(page.Servers) > limit || p.Page == nil || *p.Page != 1 || p.PerPage == nil || *p.PerPage != 50 || p.LastPage == nil || *p.LastPage != 1 || !bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")) || p.Total == nil || *p.Total != len(page.Servers) { // Report only the bounded count and guard outcomes, never provider // identities, response values, credentials or a raw payload. return nil, fmt.Errorf("provider inventory is incomplete or exceeds migration capacity: servers=%d servers_present=%t within_capacity=%t page_valid=%t per_page_valid=%t last_page_valid=%t next_page_terminal=%t total_present=%t total_matches=%t", - len(page.Servers), page.Servers != nil, len(page.Servers) <= 9, + len(page.Servers), page.Servers != nil, len(page.Servers) <= limit, p.Page != nil && *p.Page == 1, p.PerPage != nil && *p.PerPage == 50, p.LastPage != nil && *p.LastPage == 1, bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")), p.Total != nil, p.Total != nil && *p.Total == len(page.Servers)) diff --git a/scripts/verify-longhorn-migration-headroom/main.go b/scripts/verify-longhorn-migration-headroom/main.go index 537d222d54..5522044e75 100644 --- a/scripts/verify-longhorn-migration-headroom/main.go +++ b/scripts/verify-longhorn-migration-headroom/main.go @@ -10,6 +10,7 @@ import ( "net/netip" "os" "regexp" + "slices" "strconv" "strings" "time" @@ -161,13 +162,28 @@ func qualify(servers []server, nodes []node, requireV2 bool) error { } func main() { - if len(os.Args) != 2 || (os.Args[1] != "before-publish" && os.Args[1] != "before-update" && os.Args[1] != "after-update") || os.Getenv("GITHUB_ACTIONS") != "true" || os.Getenv("GITHUB_REPOSITORY") != "devantler-tech/platform" { + if len(os.Args) != 2 || !slices.Contains([]string{"before-publish", "before-update", "after-update", "account-census"}, os.Args[1]) || + os.Getenv("GITHUB_ACTIONS") != "true" || os.Getenv("GITHUB_REPOSITORY") != "devantler-tech/platform" { fmt.Fprintln(os.Stderr, "HOLD: migration headroom requires a protected production deployment") os.Exit(1) } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() client := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{TLSHandshakeTimeout: 10 * time.Second, ResponseHeaderTimeout: 15 * time.Second}, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }} + if os.Args[1] == "account-census" { + if !authorizeAccountCensus(os.Getenv) { + fmt.Fprintln(os.Stderr, "HOLD: provider account reconciliation requires a confirmed first attempt on main") + os.Exit(1) + } + result, err := observeAccountCensus(ctx, client, os.Getenv("HCLOUD_TOKEN"), readNativeNodes) + if err != nil { + fmt.Fprintf(os.Stderr, "HOLD: provider account reconciliation: %v\n", err) + os.Exit(1) + } + fmt.Printf("OBSERVED: account_servers=%d joined_nodes=%d provider_only=%d classification=%s state=%s admission=none\n", + result.Servers, result.Nodes, result.Extra, result.Classification, result.State) + return + } n, err := observe(ctx, client, os.Getenv("HCLOUD_TOKEN"), readNativeNodes, os.Args[1] == "after-update") if err != nil { fmt.Fprintf(os.Stderr, "HOLD: migration headroom stage=%s: %v\n", os.Args[1], err) diff --git a/scripts/verify-longhorn-migration-headroom/workflow_test.go b/scripts/verify-longhorn-migration-headroom/workflow_test.go index db77028f41..d5cad33d22 100644 --- a/scripts/verify-longhorn-migration-headroom/workflow_test.go +++ b/scripts/verify-longhorn-migration-headroom/workflow_test.go @@ -32,8 +32,10 @@ func TestDispatchConfirmationRefusesRerunAndOtherRefs(t *testing.T) { ok bool }{ {"confirmed main", "refs/heads/main", "1", "observe-longhorn-migration-headroom", true}, + {"confirmed account reconciliation", "refs/heads/main", "1", "observe-longhorn-provider-account", true}, {"foreign ref", "refs/heads/feature", "1", "observe-longhorn-migration-headroom", false}, {"rerun", "refs/heads/main", "2", "observe-longhorn-migration-headroom", false}, + {"account reconciliation rerun", "refs/heads/main", "2", "observe-longhorn-provider-account", false}, {"missing confirmation", "refs/heads/main", "1", "", false}, } { t.Run(tc.name, func(t *testing.T) {