feat(storage): reconcile provider-only migration servers

.github/workflows/observe-longhorn-migration-headroom.yaml

@@ -5,7 +5,7 @@ name: Observe Longhorn Migration Headroom
   workflow_dispatch:
     inputs:
       confirm:
-        description: "Type exactly: observe-longhorn-migration-headroom"
+        description: Type observe-longhorn-migration-headroom or observe-longhorn-provider-account
         required: true
         type: string
 
@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 10
     environment: prod
     permissions:
-      contents: read
+      contents: read # Check out and verify the current reviewed source.
     concurrency:
       group: prod-deploy
       cancel-in-progress: false
@@ -33,7 +33,8 @@ jobs:
         run: |
           set -euo pipefail
           [[ "${DISPATCH_REF}" == refs/heads/main && "${GITHUB_RUN_ATTEMPT}" == 1 ]]
-          [[ "${LONGHORN_HEADROOM_CONFIRM}" == observe-longhorn-migration-headroom ]]
+          [[ "${LONGHORN_HEADROOM_CONFIRM}" == observe-longhorn-migration-headroom ||
+             "${LONGHORN_HEADROOM_CONFIRM}" == observe-longhorn-provider-account ]]
       - name: Checkout reviewed source
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -76,10 +77,21 @@ jobs:
         env:
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
         run: ./scripts/use-prod-stable-api-endpoint.sh
-      - name: Read migration headroom
+      - name: Read the confirmed provider inventory
+        shell: bash
         env:
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-        run: go run ./scripts/verify-longhorn-migration-headroom before-publish
+        run: |
+          set -euo pipefail
+          case "${LONGHORN_HEADROOM_CONFIRM}" in
+            observe-longhorn-migration-headroom)
+              go run ./scripts/verify-longhorn-migration-headroom before-publish
+              ;;
+            observe-longhorn-provider-account)
+              go run ./scripts/verify-longhorn-migration-headroom account-census
+              ;;
+            *) exit 1 ;;
+          esac
       - name: Require unchanged current main after observation
         env:
           GH_TOKEN: ${{ github.token }}

scripts/verify-longhorn-migration-headroom/README.md

@@ -19,6 +19,14 @@ outcomes. A rejected provider envelope identifies which pagination or capacity
 checks failed, without printing response values or weakening those checks.
 Credentials and provider/node payloads remain in runner memory.
 
+If the account is full, confirm `observe-longhorn-provider-account` to reconcile
+the migration roster with up to ten provider records. This read-only diagnostic
+requires two matching provider lists between unchanged Kubernetes Node reads.
+It reports aggregate counts and a fixed classification of at most one
+unmatched server based on provider labels. Use a separate, reviewed ownership
+and cleanup procedure for any provider write. The normal headroom gate still
+requires a reserved builder slot.
+
 The later worker activation must repeat this check immediately before mutable
 publication and before updating the pool, then require the full nine-node roster
 after Talos readiness. An earlier workflow result alone is insufficient. These

scripts/verify-longhorn-migration-headroom/census.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"cmp"
+	"context"
+	"errors"
+	"net/http"
+	"reflect"
+	"regexp"
+	"slices"
+	"strconv"
+)
+
+const censusConfirmation = "observe-longhorn-provider-account"
+
+var snapshotBuildID = regexp.MustCompile(`^[A-Z2-7]{26}$`)
+var errCensusHeld = errors.New("provider account reconciliation is incomplete, ambiguous or changed")
+
+type accountCensus struct {
+	Servers, Nodes, Extra int
+	Classification, State string
+}
+
+func authorizeAccountCensus(getenv func(string) string) bool {
+	return getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" &&
+		getenv("GITHUB_REF") == "refs/heads/main" &&
+		getenv("GITHUB_WORKFLOW_REF") == "devantler-tech/platform/.github/workflows/observe-longhorn-migration-headroom.yaml@refs/heads/main" &&
+		getenv("GITHUB_RUN_ATTEMPT") == "1" && getenv("RUNNER_ENVIRONMENT") == "github-hosted" &&
+		getenv("LONGHORN_HEADROOM_CONFIRM") == censusConfirmation
+}
+
+func observeAccountCensus(ctx context.Context, client *http.Client, token string, readNodes func(context.Context) ([]byte, error)) (accountCensus, error) {
+	read := func() ([]node, error) {
+		raw, err := readNodes(ctx)
+		if err != nil {
+			return nil, errCensusHeld
+		}
+		return decodeNodes(raw)
+	}
+	before, err := read()
+	if err != nil {
+		return accountCensus{}, err
+	}
+	first, err := collectServerInventory(ctx, client, token, maxAccountServers)
+	if err != nil {
+		return accountCensus{}, err
+	}
+	result, err := classifyAccountCensus(first, before)
+	if err != nil {
+		return accountCensus{}, err
+	}
+	second, err := collectServerInventory(ctx, client, token, maxAccountServers)
+	if err != nil {
+		return accountCensus{}, err
+	}
+	byID := func(a, b server) int { return cmp.Compare(a.ID, b.ID) }
+	slices.SortFunc(first, byID)
+	slices.SortFunc(second, byID)
+	after, err := read()
+	if err != nil || !reflect.DeepEqual(first, second) || !reflect.DeepEqual(before, after) || active(ctx) != nil {
+		return accountCensus{}, errCensusHeld
+	}
+	return result, nil
+}
+
+// Classify the one provider-only row for investigation. Its labels never
+// establish deletion authority or reserve the builder slot.
+func classifyAccountCensus(servers []server, nodes []node) (accountCensus, error) {
+	extra := len(servers) - len(nodes)
+	if len(servers) > maxAccountServers || extra < 0 || extra > 1 {
+		return accountCensus{}, errCensusHeld
+	}
+	nodeIDs := make(map[string]bool, len(nodes))
+	for _, node := range nodes {
+		nodeIDs[node.Spec.ProviderID] = true
+	}
+	ids, names := map[int64]bool{}, map[string]bool{}
+	joined := make([]server, 0, len(nodes))
+	var unmatched server
+	for _, server := range servers {
+		if server.ID <= 0 || server.Name == "" || server.Status == "" || ids[server.ID] || names[server.Name] {
+			return accountCensus{}, errCensusHeld
+		}
+		ids[server.ID], names[server.Name] = true, true
+		if nodeIDs["hcloud://"+strconv.FormatInt(server.ID, 10)] {
+			joined = append(joined, server)
+		} else {
+			unmatched = server
+		}
+	}
+	if len(joined) != len(nodes) || qualify(joined, nodes, false) != nil {
+		return accountCensus{}, errCensusHeld
+	}
+	result := accountCensus{Servers: len(servers), Nodes: len(nodes), Extra: extra, Classification: "none", State: "none"}
+	if extra == 1 {
+		result.State = "not-running"
+		if unmatched.Status == "running" {
+			result.State = "running"
+		}
+		result.Classification = classifyProviderOnly(unmatched, joined[0].PrivateNet[0].Network)
+	}
+	return result, nil
+}
+
+func classifyProviderOnly(s server, network int64) string {
+	cluster := s.Labels["ksail.cluster.name"]
+	builderCluster := s.Labels["ksail.io/cluster"]
+	group := s.Labels["hcloud/node-group"]
+	switch {
+	case (cluster != "" && cluster != "prod") || (builderCluster != "" && builderCluster != "prod"):
+		return "other-cluster-labeled"
+	case snapshotBuildID.MatchString(s.Labels["ksail.io/snapshot-build"]) && s.Labels["ksail.owned"] == "" && group == "":
+		return "snapshot-build-labeled"
+	case cluster == "prod" && s.Labels["ksail.owned"] == "true":
+		return "production-managed-labeled"
+	case group != "" && len(s.PrivateNet) == 1 && s.PrivateNet[0].Network == network:
+		return "autoscaler-labeled"
+	default:
+		return "unclassified"
+	}
+}

scripts/verify-longhorn-migration-headroom/census_test.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestAccountCensusReportsBuilderWithoutGrantingCapacity(t *testing.T) {
+	servers, nodes := roster(t, true)
+	servers = append(servers, server{ID: 999, Name: "temporary-provider-fixture", Status: "running", Labels: map[string]string{
+		"ksail.io/cluster": "prod", "ksail.io/snapshot-build": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+	}})
+	body := providerResponse(t, func(response map[string]any) {
+		response["servers"] = servers
+		response["meta"].(map[string]any)["pagination"].(map[string]any)["total_entries"] = len(servers)
+	})
+	client := providerClient(t, body, 200)
+	readNodes := func(context.Context) ([]byte, error) { return nodeResponse(t, nodes), nil }
+	got, err := observeAccountCensus(t.Context(), client, "private-test-token", readNodes)
+	if err != nil || got != (accountCensus{Servers: 10, Nodes: 9, Extra: 1, Classification: "snapshot-build-labeled", State: "running"}) {
+		t.Fatalf("provider-only builder was not accounted for: %+v, %v", got, err)
+	}
+	if count, err := observe(t.Context(), client, "private-test-token", readNodes, false); err == nil || count != 0 {
+		t.Fatal("the full account qualified for the reserved builder slot")
+	}
+}
+
+func TestAccountCensusRefusesAmbiguousIdentityAndNames(t *testing.T) {
+	servers, nodes := roster(t, true)
+	servers = append(servers, server{ID: 999, Name: "snapshot-builder-prod", Status: "off"})
+	got, err := classifyAccountCensus(servers, nodes)
+	if err != nil || got.Classification != "unclassified" || got.State != "not-running" {
+		t.Fatalf("an unowned name was attributed to a builder: %+v, %v", got, err)
+	}
+	servers[9].ID = servers[0].ID
+	if _, err := classifyAccountCensus(servers, nodes); err == nil {
+		t.Fatal("duplicate provider identity qualified")
+	}
+}
+
+func TestAccountCensusRejectsProviderDrift(t *testing.T) {
+	servers, nodes := roster(t, true)
+	body := func(status string) string {
+		return providerResponse(t, func(response map[string]any) {
+			response["servers"] = append(servers, server{ID: 999, Name: "temporary-provider-fixture", Status: status})
+			response["meta"].(map[string]any)["pagination"].(map[string]any)["total_entries"] = 10
+		})
+	}
+	calls := 0
+	client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+		payload := body("running")
+		if calls > 0 {
+			payload = body("stopping")
+		}
+		calls++
+		return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(payload))}, nil
+	})}
+	read := func(context.Context) ([]byte, error) { return nodeResponse(t, nodes), nil }
+	if result, err := observeAccountCensus(t.Context(), client, "private-test-token", read); err == nil || result != (accountCensus{}) || calls != 2 {
+		t.Fatal("a changing provider account produced a reconciliation receipt")
+	}
+}
+
+func TestAccountCensusRequiresExactProtectedDispatch(t *testing.T) {
+	env := map[string]string{
+		"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main",
+		"GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/observe-longhorn-migration-headroom.yaml@refs/heads/main",
+		"GITHUB_RUN_ATTEMPT":  "1", "RUNNER_ENVIRONMENT": "github-hosted",
+		"LONGHORN_HEADROOM_CONFIRM": "observe-longhorn-provider-account",
+	}
+	get := func(key string) string { return env[key] }
+	if !authorizeAccountCensus(get) {
+		t.Fatal("exact protected account reconciliation was refused")
+	}
+	for key, value := range env {
+		env[key] = ""
+		if authorizeAccountCensus(get) {
+			t.Fatalf("account reconciliation authorized without %s", key)
+		}
+		env[key] = value
+	}
+}

scripts/verify-longhorn-migration-headroom/inventory.go

@@ -13,6 +13,8 @@ import (
 )
 
 const maxInventoryBytes = 2 * 1024 * 1024
+const maxHeadroomServers = 9
+const maxAccountServers = 10
 
 var serverFields = map[string][]string{
 	"":                              {"servers", "meta"},
@@ -37,6 +39,10 @@ var nodeFields = map[string][]string{
 }
 
 func collectServers(ctx context.Context, client *http.Client, token string) ([]server, error) {
+	return collectServerInventory(ctx, client, token, maxHeadroomServers)
+}
+
+func collectServerInventory(ctx context.Context, client *http.Client, token string, limit int) ([]server, error) {
 	if token == "" {
 		return nil, errors.New("provider read credential is unavailable")
 	}
@@ -75,11 +81,11 @@ func collectServers(ctx context.Context, client *http.Client, token string) ([]s
 	p := page.Meta.Pagination
 	// The account limit is ten, below this fifty-entry request. Anything that
 	// needs another page cannot qualify for a free slot; never treat it as empty.
-	if page.Servers == nil || len(page.Servers) > 9 || p.Page == nil || *p.Page != 1 || p.PerPage == nil || *p.PerPage != 50 || p.LastPage == nil || *p.LastPage != 1 || !bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")) || p.Total == nil || *p.Total != len(page.Servers) {
+	if page.Servers == nil || len(page.Servers) > limit || p.Page == nil || *p.Page != 1 || p.PerPage == nil || *p.PerPage != 50 || p.LastPage == nil || *p.LastPage != 1 || !bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")) || p.Total == nil || *p.Total != len(page.Servers) {
 		// Report only the bounded count and guard outcomes, never provider
 		// identities, response values, credentials or a raw payload.
 		return nil, fmt.Errorf("provider inventory is incomplete or exceeds migration capacity: servers=%d servers_present=%t within_capacity=%t page_valid=%t per_page_valid=%t last_page_valid=%t next_page_terminal=%t total_present=%t total_matches=%t",
-			len(page.Servers), page.Servers != nil, len(page.Servers) <= 9,
+			len(page.Servers), page.Servers != nil, len(page.Servers) <= limit,
 			p.Page != nil && *p.Page == 1, p.PerPage != nil && *p.PerPage == 50,
 			p.LastPage != nil && *p.LastPage == 1, bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")),
 			p.Total != nil, p.Total != nil && *p.Total == len(page.Servers))

scripts/verify-longhorn-migration-headroom/main.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"os"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -161,13 +162,28 @@ func qualify(servers []server, nodes []node, requireV2 bool) error {
 }
 
 func main() {
-	if len(os.Args) != 2 || (os.Args[1] != "before-publish" && os.Args[1] != "before-update" && os.Args[1] != "after-update") || os.Getenv("GITHUB_ACTIONS") != "true" || os.Getenv("GITHUB_REPOSITORY") != "devantler-tech/platform" {
+	if len(os.Args) != 2 || !slices.Contains([]string{"before-publish", "before-update", "after-update", "account-census"}, os.Args[1]) ||
+		os.Getenv("GITHUB_ACTIONS") != "true" || os.Getenv("GITHUB_REPOSITORY") != "devantler-tech/platform" {
 		fmt.Fprintln(os.Stderr, "HOLD: migration headroom requires a protected production deployment")
 		os.Exit(1)
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
 	defer cancel()
 	client := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{TLSHandshakeTimeout: 10 * time.Second, ResponseHeaderTimeout: 15 * time.Second}, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }}
+	if os.Args[1] == "account-census" {
+		if !authorizeAccountCensus(os.Getenv) {
+			fmt.Fprintln(os.Stderr, "HOLD: provider account reconciliation requires a confirmed first attempt on main")
+			os.Exit(1)
+		}
+		result, err := observeAccountCensus(ctx, client, os.Getenv("HCLOUD_TOKEN"), readNativeNodes)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "HOLD: provider account reconciliation: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("OBSERVED: account_servers=%d joined_nodes=%d provider_only=%d classification=%s state=%s admission=none\n",
+			result.Servers, result.Nodes, result.Extra, result.Classification, result.State)
+		return
+	}
 	n, err := observe(ctx, client, os.Getenv("HCLOUD_TOKEN"), readNativeNodes, os.Args[1] == "after-update")
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "HOLD: migration headroom stage=%s: %v\n", os.Args[1], err)

scripts/verify-longhorn-migration-headroom/workflow_test.go

@@ -32,8 +32,10 @@ func TestDispatchConfirmationRefusesRerunAndOtherRefs(t *testing.T) {
 		ok                          bool
 	}{
 		{"confirmed main", "refs/heads/main", "1", "observe-longhorn-migration-headroom", true},
+		{"confirmed account reconciliation", "refs/heads/main", "1", "observe-longhorn-provider-account", true},
 		{"foreign ref", "refs/heads/feature", "1", "observe-longhorn-migration-headroom", false},
 		{"rerun", "refs/heads/main", "2", "observe-longhorn-migration-headroom", false},
+		{"account reconciliation rerun", "refs/heads/main", "2", "observe-longhorn-provider-account", false},
 		{"missing confirmation", "refs/heads/main", "1", "", false},
 	} {
 		t.Run(tc.name, func(t *testing.T) {